diff --git a/Cinder/Mitaka/__init__.py b/Cinder/Mitaka/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Mitaka/__init__.py +++ b/Cinder/Mitaka/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Mitaka/constants.py b/Cinder/Mitaka/constants.py index fc7efe5..5fc622a 100644 --- a/Cinder/Mitaka/constants.py +++ b/Cinder/Mitaka/constants.py @@ -34,6 +34,8 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] + PORTGROUP_DESCRIP_PREFIX = "Please do NOT modify this. Engine ID: " FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Mitaka/huawei_driver.py b/Cinder/Mitaka/huawei_driver.py index 6ccb166..0219e55 100644 --- a/Cinder/Mitaka/huawei_driver.py +++ b/Cinder/Mitaka/huawei_driver.py @@ -81,6 +81,7 @@ class HuaweiBaseDriver(driver.VolumeDriver): + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) @@ -2967,7 +2968,6 @@ class HuaweiISCSIDriver(HuaweiBaseDriver, driver.ISCSIDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiISCSIDriver, self).__init__(*args, **kwargs) @@ -3012,7 +3012,8 @@ def _initialize_connection_lock(self, volume, connector): connector, iscsi_info, rmt_iscsi_info) LOG.info('initialize_common_connection_iscsi, ' - 'return data is: %s.', iscsi_info) + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(iscsi_info)) return iscsi_info def _initialize_connection(self, volume, connector, local=True): @@ -3292,7 +3293,6 @@ class HuaweiFCDriver(HuaweiBaseDriver, driver.FibreChannelDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiFCDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Mitaka/huawei_utils.py b/Cinder/Mitaka/huawei_utils.py index 4353955..387aafb 100644 --- a/Cinder/Mitaka/huawei_utils.py +++ b/Cinder/Mitaka/huawei_utils.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import units +from oslo_utils import strutils from cinder import exception from cinder.i18n import _ @@ -348,3 +349,20 @@ def check_group_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) + diff --git a/Cinder/Mitaka/rest_client.py b/Cinder/Mitaka/rest_client.py index e866a9e..590d702 100644 --- a/Cinder/Mitaka/rest_client.py +++ b/Cinder/Mitaka/rest_client.py @@ -2216,19 +2216,19 @@ def get_fc_initiator_on_array(self): return fc_initiators - def get_hyper_domain_id(self, domain_name): - url = "/HyperMetroDomain?range=[0-32]" + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) result = self.call(url, None, "GET") - domain_id = None - if "data" in result: - for item in result['data']: - if domain_name == item['NAME']: - domain_id = item['ID'] - break + self._assert_rest_result(result, "Get hyper domain info error.") + return result.get('data', []) - msg = _('get_hyper_domain_id error.') - self._assert_rest_result(result, msg) - return domain_id + def get_hyper_domain_id(self, domain_name): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get("ID") + return None def create_hypermetro(self, hcp_param): url = "/HyperMetroPair" diff --git a/Cinder/Newton/__init__.py b/Cinder/Newton/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Newton/__init__.py +++ b/Cinder/Newton/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Newton/constants.py b/Cinder/Newton/constants.py index fc7efe5..5fc622a 100644 --- a/Cinder/Newton/constants.py +++ b/Cinder/Newton/constants.py @@ -34,6 +34,8 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] + PORTGROUP_DESCRIP_PREFIX = "Please do NOT modify this. Engine ID: " FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Newton/huawei_driver.py b/Cinder/Newton/huawei_driver.py index 74dfa09..3a0a299 100644 --- a/Cinder/Newton/huawei_driver.py +++ b/Cinder/Newton/huawei_driver.py @@ -81,6 +81,7 @@ class HuaweiBaseDriver(driver.VolumeDriver): + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) @@ -2967,7 +2968,6 @@ class HuaweiISCSIDriver(HuaweiBaseDriver, driver.ISCSIDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiISCSIDriver, self).__init__(*args, **kwargs) @@ -3012,7 +3012,8 @@ def _initialize_connection_lock(self, volume, connector): connector, iscsi_info, rmt_iscsi_info) LOG.info('initialize_common_connection_iscsi, ' - 'return data is: %s.', iscsi_info) + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(iscsi_info)) return iscsi_info def _initialize_connection(self, volume, connector, local=True): @@ -3292,7 +3293,6 @@ class HuaweiFCDriver(HuaweiBaseDriver, driver.FibreChannelDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiFCDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Newton/huawei_utils.py b/Cinder/Newton/huawei_utils.py index 4353955..387aafb 100644 --- a/Cinder/Newton/huawei_utils.py +++ b/Cinder/Newton/huawei_utils.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import units +from oslo_utils import strutils from cinder import exception from cinder.i18n import _ @@ -348,3 +349,20 @@ def check_group_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) + diff --git a/Cinder/Newton/rest_client.py b/Cinder/Newton/rest_client.py index e866a9e..590d702 100644 --- a/Cinder/Newton/rest_client.py +++ b/Cinder/Newton/rest_client.py @@ -2216,19 +2216,19 @@ def get_fc_initiator_on_array(self): return fc_initiators - def get_hyper_domain_id(self, domain_name): - url = "/HyperMetroDomain?range=[0-32]" + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) result = self.call(url, None, "GET") - domain_id = None - if "data" in result: - for item in result['data']: - if domain_name == item['NAME']: - domain_id = item['ID'] - break + self._assert_rest_result(result, "Get hyper domain info error.") + return result.get('data', []) - msg = _('get_hyper_domain_id error.') - self._assert_rest_result(result, msg) - return domain_id + def get_hyper_domain_id(self, domain_name): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get("ID") + return None def create_hypermetro(self, hcp_param): url = "/HyperMetroPair" diff --git a/Cinder/Ocata/__init__.py b/Cinder/Ocata/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Ocata/__init__.py +++ b/Cinder/Ocata/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Ocata/constants.py b/Cinder/Ocata/constants.py index fc7efe5..5fc622a 100644 --- a/Cinder/Ocata/constants.py +++ b/Cinder/Ocata/constants.py @@ -34,6 +34,8 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] + PORTGROUP_DESCRIP_PREFIX = "Please do NOT modify this. Engine ID: " FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Ocata/huawei_driver.py b/Cinder/Ocata/huawei_driver.py index 922f41d..4282918 100644 --- a/Cinder/Ocata/huawei_driver.py +++ b/Cinder/Ocata/huawei_driver.py @@ -81,6 +81,7 @@ class HuaweiBaseDriver(driver.VolumeDriver): + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) @@ -2967,7 +2968,6 @@ class HuaweiISCSIDriver(HuaweiBaseDriver, driver.ISCSIDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiISCSIDriver, self).__init__(*args, **kwargs) @@ -3012,7 +3012,8 @@ def _initialize_connection_lock(self, volume, connector): connector, iscsi_info, rmt_iscsi_info) LOG.info('initialize_common_connection_iscsi, ' - 'return data is: %s.', iscsi_info) + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(iscsi_info)) return iscsi_info def _initialize_connection(self, volume, connector, local=True): @@ -3292,7 +3293,6 @@ class HuaweiFCDriver(HuaweiBaseDriver, driver.FibreChannelDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiFCDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Ocata/huawei_utils.py b/Cinder/Ocata/huawei_utils.py index 4353955..387aafb 100644 --- a/Cinder/Ocata/huawei_utils.py +++ b/Cinder/Ocata/huawei_utils.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import units +from oslo_utils import strutils from cinder import exception from cinder.i18n import _ @@ -348,3 +349,20 @@ def check_group_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) + diff --git a/Cinder/Ocata/rest_client.py b/Cinder/Ocata/rest_client.py index e866a9e..590d702 100644 --- a/Cinder/Ocata/rest_client.py +++ b/Cinder/Ocata/rest_client.py @@ -2216,19 +2216,19 @@ def get_fc_initiator_on_array(self): return fc_initiators - def get_hyper_domain_id(self, domain_name): - url = "/HyperMetroDomain?range=[0-32]" + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) result = self.call(url, None, "GET") - domain_id = None - if "data" in result: - for item in result['data']: - if domain_name == item['NAME']: - domain_id = item['ID'] - break + self._assert_rest_result(result, "Get hyper domain info error.") + return result.get('data', []) - msg = _('get_hyper_domain_id error.') - self._assert_rest_result(result, msg) - return domain_id + def get_hyper_domain_id(self, domain_name): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get("ID") + return None def create_hypermetro(self, hcp_param): url = "/HyperMetroPair" diff --git a/Cinder/Pike/__init__.py b/Cinder/Pike/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Pike/__init__.py +++ b/Cinder/Pike/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Pike/constants.py b/Cinder/Pike/constants.py index fc7efe5..5fc622a 100644 --- a/Cinder/Pike/constants.py +++ b/Cinder/Pike/constants.py @@ -34,6 +34,8 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] + PORTGROUP_DESCRIP_PREFIX = "Please do NOT modify this. Engine ID: " FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Pike/huawei_driver.py b/Cinder/Pike/huawei_driver.py index 01d0f0d..64bd4ac 100644 --- a/Cinder/Pike/huawei_driver.py +++ b/Cinder/Pike/huawei_driver.py @@ -81,6 +81,7 @@ class HuaweiBaseDriver(driver.VolumeDriver): + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) @@ -2967,7 +2968,6 @@ class HuaweiISCSIDriver(HuaweiBaseDriver, driver.ISCSIDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiISCSIDriver, self).__init__(*args, **kwargs) @@ -3012,7 +3012,8 @@ def _initialize_connection_lock(self, volume, connector): connector, iscsi_info, rmt_iscsi_info) LOG.info('initialize_common_connection_iscsi, ' - 'return data is: %s.', iscsi_info) + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(iscsi_info)) return iscsi_info def _initialize_connection(self, volume, connector, local=True): @@ -3292,7 +3293,6 @@ class HuaweiFCDriver(HuaweiBaseDriver, driver.FibreChannelDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiFCDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Pike/huawei_utils.py b/Cinder/Pike/huawei_utils.py index 4353955..387aafb 100644 --- a/Cinder/Pike/huawei_utils.py +++ b/Cinder/Pike/huawei_utils.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import units +from oslo_utils import strutils from cinder import exception from cinder.i18n import _ @@ -348,3 +349,20 @@ def check_group_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) + diff --git a/Cinder/Pike/rest_client.py b/Cinder/Pike/rest_client.py index e866a9e..590d702 100644 --- a/Cinder/Pike/rest_client.py +++ b/Cinder/Pike/rest_client.py @@ -2216,19 +2216,19 @@ def get_fc_initiator_on_array(self): return fc_initiators - def get_hyper_domain_id(self, domain_name): - url = "/HyperMetroDomain?range=[0-32]" + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) result = self.call(url, None, "GET") - domain_id = None - if "data" in result: - for item in result['data']: - if domain_name == item['NAME']: - domain_id = item['ID'] - break + self._assert_rest_result(result, "Get hyper domain info error.") + return result.get('data', []) - msg = _('get_hyper_domain_id error.') - self._assert_rest_result(result, msg) - return domain_id + def get_hyper_domain_id(self, domain_name): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get("ID") + return None def create_hypermetro(self, hcp_param): url = "/HyperMetroPair" diff --git a/Cinder/Queens/__init__.py b/Cinder/Queens/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Queens/__init__.py +++ b/Cinder/Queens/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Queens/constants.py b/Cinder/Queens/constants.py index fc7efe5..5fc622a 100644 --- a/Cinder/Queens/constants.py +++ b/Cinder/Queens/constants.py @@ -34,6 +34,8 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] + PORTGROUP_DESCRIP_PREFIX = "Please do NOT modify this. Engine ID: " FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Queens/huawei_driver.py b/Cinder/Queens/huawei_driver.py index 01d0f0d..64bd4ac 100644 --- a/Cinder/Queens/huawei_driver.py +++ b/Cinder/Queens/huawei_driver.py @@ -81,6 +81,7 @@ class HuaweiBaseDriver(driver.VolumeDriver): + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) @@ -2967,7 +2968,6 @@ class HuaweiISCSIDriver(HuaweiBaseDriver, driver.ISCSIDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiISCSIDriver, self).__init__(*args, **kwargs) @@ -3012,7 +3012,8 @@ def _initialize_connection_lock(self, volume, connector): connector, iscsi_info, rmt_iscsi_info) LOG.info('initialize_common_connection_iscsi, ' - 'return data is: %s.', iscsi_info) + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(iscsi_info)) return iscsi_info def _initialize_connection(self, volume, connector, local=True): @@ -3292,7 +3293,6 @@ class HuaweiFCDriver(HuaweiBaseDriver, driver.FibreChannelDriver): 2.2.RC1 - Add force delete volume """ - VERSION = "2.5.RC4" def __init__(self, *args, **kwargs): super(HuaweiFCDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Queens/huawei_utils.py b/Cinder/Queens/huawei_utils.py index 4353955..387aafb 100644 --- a/Cinder/Queens/huawei_utils.py +++ b/Cinder/Queens/huawei_utils.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import units +from oslo_utils import strutils from cinder import exception from cinder.i18n import _ @@ -348,3 +349,20 @@ def check_group_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) + diff --git a/Cinder/Queens/rest_client.py b/Cinder/Queens/rest_client.py index e866a9e..590d702 100644 --- a/Cinder/Queens/rest_client.py +++ b/Cinder/Queens/rest_client.py @@ -2216,19 +2216,19 @@ def get_fc_initiator_on_array(self): return fc_initiators - def get_hyper_domain_id(self, domain_name): - url = "/HyperMetroDomain?range=[0-32]" + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) result = self.call(url, None, "GET") - domain_id = None - if "data" in result: - for item in result['data']: - if domain_name == item['NAME']: - domain_id = item['ID'] - break + self._assert_rest_result(result, "Get hyper domain info error.") + return result.get('data', []) - msg = _('get_hyper_domain_id error.') - self._assert_rest_result(result, msg) - return domain_id + def get_hyper_domain_id(self, domain_name): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get("ID") + return None def create_hypermetro(self, hcp_param): url = "/HyperMetroPair" diff --git a/Cinder/Rocky/__init__.py b/Cinder/Rocky/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Rocky/__init__.py +++ b/Cinder/Rocky/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Rocky/constants.py b/Cinder/Rocky/constants.py index 471a424..756c839 100644 --- a/Cinder/Rocky/constants.py +++ b/Cinder/Rocky/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Rocky/huawei_base_driver.py b/Cinder/Rocky/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Rocky/huawei_base_driver.py +++ b/Cinder/Rocky/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Rocky/huawei_driver.py b/Cinder/Rocky/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Rocky/huawei_driver.py +++ b/Cinder/Rocky/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Rocky/huawei_flow.py b/Cinder/Rocky/huawei_flow.py index 6c0bb39..6f4f248 100644 --- a/Cinder/Rocky/huawei_flow.py +++ b/Cinder/Rocky/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Rocky/huawei_utils.py b/Cinder/Rocky/huawei_utils.py index 45d1940..3eb2c57 100644 --- a/Cinder/Rocky/huawei_utils.py +++ b/Cinder/Rocky/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Rocky/rest_client.py b/Cinder/Rocky/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Rocky/rest_client.py +++ b/Cinder/Rocky/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Stein/__init__.py b/Cinder/Stein/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Stein/__init__.py +++ b/Cinder/Stein/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Stein/constants.py b/Cinder/Stein/constants.py index 471a424..756c839 100644 --- a/Cinder/Stein/constants.py +++ b/Cinder/Stein/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Stein/huawei_base_driver.py b/Cinder/Stein/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Stein/huawei_base_driver.py +++ b/Cinder/Stein/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Stein/huawei_driver.py b/Cinder/Stein/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Stein/huawei_driver.py +++ b/Cinder/Stein/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Stein/huawei_flow.py b/Cinder/Stein/huawei_flow.py index 6c0bb39..6f4f248 100644 --- a/Cinder/Stein/huawei_flow.py +++ b/Cinder/Stein/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Stein/huawei_utils.py b/Cinder/Stein/huawei_utils.py index 45d1940..3eb2c57 100644 --- a/Cinder/Stein/huawei_utils.py +++ b/Cinder/Stein/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Stein/rest_client.py b/Cinder/Stein/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Stein/rest_client.py +++ b/Cinder/Stein/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Train/__init__.py b/Cinder/Train/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Train/__init__.py +++ b/Cinder/Train/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Train/constants.py b/Cinder/Train/constants.py index 471a424..756c839 100644 --- a/Cinder/Train/constants.py +++ b/Cinder/Train/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Train/huawei_base_driver.py b/Cinder/Train/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Train/huawei_base_driver.py +++ b/Cinder/Train/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Train/huawei_driver.py b/Cinder/Train/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Train/huawei_driver.py +++ b/Cinder/Train/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Train/huawei_flow.py b/Cinder/Train/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Train/huawei_flow.py +++ b/Cinder/Train/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Train/huawei_utils.py b/Cinder/Train/huawei_utils.py index 45d1940..3eb2c57 100644 --- a/Cinder/Train/huawei_utils.py +++ b/Cinder/Train/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Train/rest_client.py b/Cinder/Train/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Train/rest_client.py +++ b/Cinder/Train/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Ussuri/__init__.py b/Cinder/Ussuri/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Ussuri/__init__.py +++ b/Cinder/Ussuri/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Ussuri/constants.py b/Cinder/Ussuri/constants.py index 471a424..756c839 100644 --- a/Cinder/Ussuri/constants.py +++ b/Cinder/Ussuri/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Ussuri/huawei_base_driver.py b/Cinder/Ussuri/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Ussuri/huawei_base_driver.py +++ b/Cinder/Ussuri/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Ussuri/huawei_driver.py b/Cinder/Ussuri/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Ussuri/huawei_driver.py +++ b/Cinder/Ussuri/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Ussuri/huawei_flow.py b/Cinder/Ussuri/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Ussuri/huawei_flow.py +++ b/Cinder/Ussuri/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Ussuri/huawei_utils.py b/Cinder/Ussuri/huawei_utils.py index 45d1940..3eb2c57 100644 --- a/Cinder/Ussuri/huawei_utils.py +++ b/Cinder/Ussuri/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Ussuri/rest_client.py b/Cinder/Ussuri/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Ussuri/rest_client.py +++ b/Cinder/Ussuri/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Victoria/__init__.py b/Cinder/Victoria/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Victoria/__init__.py +++ b/Cinder/Victoria/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Victoria/constants.py b/Cinder/Victoria/constants.py index 471a424..756c839 100644 --- a/Cinder/Victoria/constants.py +++ b/Cinder/Victoria/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Victoria/huawei_base_driver.py b/Cinder/Victoria/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Victoria/huawei_base_driver.py +++ b/Cinder/Victoria/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Victoria/huawei_driver.py b/Cinder/Victoria/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Victoria/huawei_driver.py +++ b/Cinder/Victoria/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Victoria/huawei_flow.py b/Cinder/Victoria/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Victoria/huawei_flow.py +++ b/Cinder/Victoria/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Victoria/huawei_utils.py b/Cinder/Victoria/huawei_utils.py index 6fc3055..f77eb37 100644 --- a/Cinder/Victoria/huawei_utils.py +++ b/Cinder/Victoria/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Victoria/rest_client.py b/Cinder/Victoria/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Victoria/rest_client.py +++ b/Cinder/Victoria/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Wallaby/__init__.py b/Cinder/Wallaby/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Wallaby/__init__.py +++ b/Cinder/Wallaby/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Wallaby/constants.py b/Cinder/Wallaby/constants.py index 471a424..756c839 100644 --- a/Cinder/Wallaby/constants.py +++ b/Cinder/Wallaby/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Wallaby/huawei_base_driver.py b/Cinder/Wallaby/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Wallaby/huawei_base_driver.py +++ b/Cinder/Wallaby/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Wallaby/huawei_driver.py b/Cinder/Wallaby/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Wallaby/huawei_driver.py +++ b/Cinder/Wallaby/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Wallaby/huawei_flow.py b/Cinder/Wallaby/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Wallaby/huawei_flow.py +++ b/Cinder/Wallaby/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Wallaby/huawei_utils.py b/Cinder/Wallaby/huawei_utils.py index 6fc3055..f77eb37 100644 --- a/Cinder/Wallaby/huawei_utils.py +++ b/Cinder/Wallaby/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Wallaby/rest_client.py b/Cinder/Wallaby/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Wallaby/rest_client.py +++ b/Cinder/Wallaby/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Xena/__init__.py b/Cinder/Xena/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Xena/__init__.py +++ b/Cinder/Xena/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Xena/constants.py b/Cinder/Xena/constants.py index 471a424..756c839 100644 --- a/Cinder/Xena/constants.py +++ b/Cinder/Xena/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Xena/huawei_base_driver.py b/Cinder/Xena/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Xena/huawei_base_driver.py +++ b/Cinder/Xena/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Xena/huawei_driver.py b/Cinder/Xena/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Xena/huawei_driver.py +++ b/Cinder/Xena/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Xena/huawei_flow.py b/Cinder/Xena/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Xena/huawei_flow.py +++ b/Cinder/Xena/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Xena/huawei_utils.py b/Cinder/Xena/huawei_utils.py index 6fc3055..f77eb37 100644 --- a/Cinder/Xena/huawei_utils.py +++ b/Cinder/Xena/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Xena/rest_client.py b/Cinder/Xena/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Xena/rest_client.py +++ b/Cinder/Xena/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Yoga/__init__.py b/Cinder/Yoga/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Yoga/__init__.py +++ b/Cinder/Yoga/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Yoga/constants.py b/Cinder/Yoga/constants.py index 471a424..756c839 100644 --- a/Cinder/Yoga/constants.py +++ b/Cinder/Yoga/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Yoga/huawei_base_driver.py b/Cinder/Yoga/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Yoga/huawei_base_driver.py +++ b/Cinder/Yoga/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Yoga/huawei_driver.py b/Cinder/Yoga/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Yoga/huawei_driver.py +++ b/Cinder/Yoga/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Yoga/huawei_flow.py b/Cinder/Yoga/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Yoga/huawei_flow.py +++ b/Cinder/Yoga/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Yoga/huawei_utils.py b/Cinder/Yoga/huawei_utils.py index 6fc3055..f77eb37 100644 --- a/Cinder/Yoga/huawei_utils.py +++ b/Cinder/Yoga/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Yoga/rest_client.py b/Cinder/Yoga/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Yoga/rest_client.py +++ b/Cinder/Yoga/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/Cinder/Zed/__init__.py b/Cinder/Zed/__init__.py index c69c299..c7da93a 100644 --- a/Cinder/Zed/__init__.py +++ b/Cinder/Zed/__init__.py @@ -1 +1 @@ -"""Version: 2.5.RC4""" +"""Version: 2.6.1""" diff --git a/Cinder/Zed/constants.py b/Cinder/Zed/constants.py index 471a424..756c839 100644 --- a/Cinder/Zed/constants.py +++ b/Cinder/Zed/constants.py @@ -25,6 +25,7 @@ MAPPING_VIEW_PREFIX = 'OpenStack_Mapping_View_' PORTGROUP_PREFIX = 'OpenStack_PortGroup_' QOS_NAME_PREFIX = 'OpenStack_' +SENSITIVE_KEYS = ['auth_password'] FC_PORT_CONNECTED = '10' FC_INIT_ONLINE = '27' diff --git a/Cinder/Zed/huawei_base_driver.py b/Cinder/Zed/huawei_base_driver.py index 719f981..b6c3002 100644 --- a/Cinder/Zed/huawei_base_driver.py +++ b/Cinder/Zed/huawei_base_driver.py @@ -55,7 +55,7 @@ class HuaweiBaseDriver(object): - VERSION = "2.5.RC4" + VERSION = "2.6.1" def __init__(self, *args, **kwargs): super(HuaweiBaseDriver, self).__init__(*args, **kwargs) diff --git a/Cinder/Zed/huawei_driver.py b/Cinder/Zed/huawei_driver.py index 28dc74e..8c09a2e 100644 --- a/Cinder/Zed/huawei_driver.py +++ b/Cinder/Zed/huawei_driver.py @@ -89,7 +89,9 @@ def initialize_connection(self, volume, connector): mapping_info.pop('aval_host_lun_ids', None) conn = {'driver_volume_type': 'iscsi', 'data': mapping_info} - LOG.info('Initialize iscsi connection successfully: %s.', conn) + LOG.info('Initialize iscsi connection successfully,' + 'return data is: %s.', + huawei_utils.mask_dict_sensitive_info(conn)) return conn def terminate_connection(self, volume, connector, **kwargs): diff --git a/Cinder/Zed/huawei_flow.py b/Cinder/Zed/huawei_flow.py index f909343..00d649c 100644 --- a/Cinder/Zed/huawei_flow.py +++ b/Cinder/Zed/huawei_flow.py @@ -388,7 +388,7 @@ def execute(self, opts, volume, lun_info): if not metadata.get('hypermetro'): add_hypermetro = True else: - if not metadata.get('hypermetro'): + if metadata.get('hypermetro'): delete_hypermetro = True if (add_hypermetro or delete_hypermetro) and in_use_lun: @@ -1671,10 +1671,11 @@ def execute(self, connector, lun_id): class ClearLunMappingTask(task.Task): default_provides = 'ini_tgt_map' - def __init__(self, client, configuration, fc_san=None, *args, **kwargs): + def __init__(self, client, configuration, fc_san=None, is_fc=False, *args, **kwargs): super(ClearLunMappingTask, self).__init__(*args, **kwargs) self.client = client self.fc_san = fc_san + self.is_fc = is_fc self.configuration = configuration def _get_obj_count_of_lungroup(self, lungroup_id): @@ -1748,7 +1749,7 @@ def execute(self, connector, lun_id, lun_type, host_id, mappingview_id, if mappingview_id and portgroup_id: self._delete_portgroup(mappingview_id, portgroup_id) - if mappingview_id and not self.fc_san: + if mappingview_id and not self.is_fc: self.client.update_iscsi_initiator_chap( connector.get('initiator'), chap_info=None) if mappingview_id and lungroup_id: @@ -2686,7 +2687,7 @@ def terminate_fc_connection(lun, lun_type, connector, fc_san, client, work_flow.add( GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san), + ClearLunMappingTask(client, configuration, fc_san, is_fc=True), ) engine = taskflow.engines.load(work_flow, store=store_spec) @@ -2703,7 +2704,7 @@ def terminate_remote_fc_connection(hypermetro_id, connector, fc_san, client, work_flow.add( GetHyperMetroRemoteLunTask(client, hypermetro_id), GetLunMappingTask(client), - ClearLunMappingTask(client, configuration, fc_san, + ClearLunMappingTask(client, configuration, fc_san, is_fc=True, inject={'lun_type': constants.LUN_TYPE}), ) diff --git a/Cinder/Zed/huawei_utils.py b/Cinder/Zed/huawei_utils.py index 6fc3055..f77eb37 100644 --- a/Cinder/Zed/huawei_utils.py +++ b/Cinder/Zed/huawei_utils.py @@ -650,3 +650,19 @@ def check_volume_type_valid(opt): "specified at the same volume_type.") LOG.error(msg) raise exception.VolumeBackendAPIException(data=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Cinder/Zed/rest_client.py b/Cinder/Zed/rest_client.py index b24c1b8..3555e6c 100644 --- a/Cinder/Zed/rest_client.py +++ b/Cinder/Zed/rest_client.py @@ -84,6 +84,20 @@ def delete(self, url, **kwargs): def get(self, url, **kwargs): return self.client.get(url, **kwargs) + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.GET_PATCH_NUM + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.GET_PATCH_NUM: + break + + range_start += constants.GET_PATCH_NUM + return info_list + def _assert_result(result, msg_format, *args): if _error_code(result) != 0: @@ -227,17 +241,6 @@ def is_host_associate_inband_lun(self, host_id): return False - def is_lun_associated_to_lungroup(self, lungroup_id, lun_info): - url = ("/associate?&ASSOCIATEOBJTYPE=256&ASSOCIATEOBJID=%s" - "&filter=NAME::%s&selectFields=ID,NAME,ASSOCIATEMETADATA,WWN" - % (lungroup_id, lun_info['NAME'])) - result = self.get(url) - _assert_result(result, 'Check lungroup associate error.') - for item in result.get('data', []): - if item.get('ID') == lun_info.get('ID'): - return True - return False - class StoragePool(CommonObject): _obj_url = '/storagepool' @@ -651,8 +654,10 @@ def update_iscsi_initiator_chap(self, initiator, chap_info): data = {"USECHAP": "false", "MULTIPATHTYPE": "0"} - result = self.put('/%(ini)s', data=data, ini=initiator) + result = self.put('/%(ini)s', data=data, ini=initiator, log_filter=True) _assert_result(result, 'Update initiator %s chap error.', initiator) + LOG.info("Update initiator chap info successfully, " + "url is /iscsi_initiator/%s, method is %s", initiator, 'put') def remove_iscsi_initiator_from_host(self, initiator): data = {"ID": initiator} @@ -1062,11 +1067,18 @@ class HyperMetroDomain(CommonObject): _obj_url = '/HyperMetroDomain' def get_hypermetro_domain_id(self, domain_name): - result = self.get('?range=[0-32]') - _assert_result(result, 'Get hyper metro domains info error.') - for item in result.get('data', []): - if domain_name == item['NAME']: - return item['ID'] + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: + if domain_name == item.get('NAME'): + return item.get('ID') + return None + + def _get_hypermetro_domain(self, start, end, params): + url = ("?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.get(url) + _assert_result(result, "Get hyper metro domains info error.") + return result.get('data', []) class HyperMetroPair(CommonObject): diff --git a/ConfigDoc/en/OpenStack Cinder Driver Configuration Guide.pdf b/ConfigDoc/en/OpenStack Cinder Driver Configuration Guide.pdf index 5f05b75..df987b3 100644 Binary files a/ConfigDoc/en/OpenStack Cinder Driver Configuration Guide.pdf and b/ConfigDoc/en/OpenStack Cinder Driver Configuration Guide.pdf differ diff --git a/ConfigDoc/en/OpenStack Manila Driver Configuration Guide.pdf b/ConfigDoc/en/OpenStack Manila Driver Configuration Guide.pdf index 9b0225e..909cbda 100644 Binary files a/ConfigDoc/en/OpenStack Manila Driver Configuration Guide.pdf and b/ConfigDoc/en/OpenStack Manila Driver Configuration Guide.pdf differ diff --git "a/ConfigDoc/zh/OpenStack Cinder Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" "b/ConfigDoc/zh/OpenStack Cinder Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" index 4b82964..0c9844d 100644 Binary files "a/ConfigDoc/zh/OpenStack Cinder Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" and "b/ConfigDoc/zh/OpenStack Cinder Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" differ diff --git "a/ConfigDoc/zh/OpenStack Manila Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" "b/ConfigDoc/zh/OpenStack Manila Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" index 0c9a5e7..8401a8d 100644 Binary files "a/ConfigDoc/zh/OpenStack Manila Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" and "b/ConfigDoc/zh/OpenStack Manila Driver\351\205\215\347\275\256\346\214\207\345\215\227.pdf" differ diff --git a/Manila/Mitaka/__init__.py b/Manila/Mitaka/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Mitaka/__init__.py +++ b/Manila/Mitaka/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Mitaka/constants.py b/Manila/Mitaka/constants.py index 693935e..619c017 100644 --- a/Manila/Mitaka/constants.py +++ b/Manila/Mitaka/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Mitaka/helper.py b/Manila/Mitaka/helper.py index 86ce66f..f055044 100644 --- a/Manila/Mitaka/helper.py +++ b/Manila/Mitaka/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Mitaka/huawei_nas.py b/Manila/Mitaka/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Mitaka/huawei_nas.py +++ b/Manila/Mitaka/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Mitaka/huawei_utils.py b/Manila/Mitaka/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Mitaka/huawei_utils.py +++ b/Manila/Mitaka/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Newton/__init__.py b/Manila/Newton/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Newton/__init__.py +++ b/Manila/Newton/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Newton/constants.py b/Manila/Newton/constants.py index 693935e..619c017 100644 --- a/Manila/Newton/constants.py +++ b/Manila/Newton/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Newton/helper.py b/Manila/Newton/helper.py index 86ce66f..f055044 100644 --- a/Manila/Newton/helper.py +++ b/Manila/Newton/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Newton/huawei_nas.py b/Manila/Newton/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Newton/huawei_nas.py +++ b/Manila/Newton/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Newton/huawei_utils.py b/Manila/Newton/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Newton/huawei_utils.py +++ b/Manila/Newton/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Ocata/__init__.py b/Manila/Ocata/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Ocata/__init__.py +++ b/Manila/Ocata/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Ocata/constants.py b/Manila/Ocata/constants.py index 693935e..619c017 100644 --- a/Manila/Ocata/constants.py +++ b/Manila/Ocata/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Ocata/helper.py b/Manila/Ocata/helper.py index 86ce66f..f055044 100644 --- a/Manila/Ocata/helper.py +++ b/Manila/Ocata/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Ocata/huawei_nas.py b/Manila/Ocata/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Ocata/huawei_nas.py +++ b/Manila/Ocata/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Ocata/huawei_utils.py b/Manila/Ocata/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Ocata/huawei_utils.py +++ b/Manila/Ocata/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Pike/__init__.py b/Manila/Pike/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Pike/__init__.py +++ b/Manila/Pike/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Pike/constants.py b/Manila/Pike/constants.py index 693935e..619c017 100644 --- a/Manila/Pike/constants.py +++ b/Manila/Pike/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Pike/helper.py b/Manila/Pike/helper.py index 86ce66f..f055044 100644 --- a/Manila/Pike/helper.py +++ b/Manila/Pike/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Pike/huawei_nas.py b/Manila/Pike/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Pike/huawei_nas.py +++ b/Manila/Pike/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Pike/huawei_utils.py b/Manila/Pike/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Pike/huawei_utils.py +++ b/Manila/Pike/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Queens/__init__.py b/Manila/Queens/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Queens/__init__.py +++ b/Manila/Queens/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Queens/constants.py b/Manila/Queens/constants.py index 693935e..619c017 100644 --- a/Manila/Queens/constants.py +++ b/Manila/Queens/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Queens/helper.py b/Manila/Queens/helper.py index 86ce66f..f055044 100644 --- a/Manila/Queens/helper.py +++ b/Manila/Queens/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Queens/huawei_nas.py b/Manila/Queens/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Queens/huawei_nas.py +++ b/Manila/Queens/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Queens/huawei_utils.py b/Manila/Queens/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Queens/huawei_utils.py +++ b/Manila/Queens/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Rocky/__init__.py b/Manila/Rocky/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Rocky/__init__.py +++ b/Manila/Rocky/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Rocky/constants.py b/Manila/Rocky/constants.py index 693935e..619c017 100644 --- a/Manila/Rocky/constants.py +++ b/Manila/Rocky/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Rocky/helper.py b/Manila/Rocky/helper.py index 86ce66f..f055044 100644 --- a/Manila/Rocky/helper.py +++ b/Manila/Rocky/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Rocky/huawei_nas.py b/Manila/Rocky/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Rocky/huawei_nas.py +++ b/Manila/Rocky/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Rocky/huawei_utils.py b/Manila/Rocky/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Rocky/huawei_utils.py +++ b/Manila/Rocky/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Stein/__init__.py b/Manila/Stein/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Stein/__init__.py +++ b/Manila/Stein/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Stein/constants.py b/Manila/Stein/constants.py index 693935e..619c017 100644 --- a/Manila/Stein/constants.py +++ b/Manila/Stein/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Stein/helper.py b/Manila/Stein/helper.py index 86ce66f..f055044 100644 --- a/Manila/Stein/helper.py +++ b/Manila/Stein/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Stein/huawei_nas.py b/Manila/Stein/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Stein/huawei_nas.py +++ b/Manila/Stein/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Stein/huawei_utils.py b/Manila/Stein/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Stein/huawei_utils.py +++ b/Manila/Stein/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Train/__init__.py b/Manila/Train/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Train/__init__.py +++ b/Manila/Train/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Train/constants.py b/Manila/Train/constants.py index 693935e..619c017 100644 --- a/Manila/Train/constants.py +++ b/Manila/Train/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Train/helper.py b/Manila/Train/helper.py index 86ce66f..f055044 100644 --- a/Manila/Train/helper.py +++ b/Manila/Train/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Train/huawei_nas.py b/Manila/Train/huawei_nas.py index 02ce82d..35f82d9 100644 --- a/Manila/Train/huawei_nas.py +++ b/Manila/Train/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -951,6 +954,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -962,6 +966,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Train/huawei_utils.py b/Manila/Train/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Train/huawei_utils.py +++ b/Manila/Train/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Ussuri/__init__.py b/Manila/Ussuri/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Ussuri/__init__.py +++ b/Manila/Ussuri/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Ussuri/constants.py b/Manila/Ussuri/constants.py index 693935e..619c017 100644 --- a/Manila/Ussuri/constants.py +++ b/Manila/Ussuri/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Ussuri/helper.py b/Manila/Ussuri/helper.py index 86ce66f..f055044 100644 --- a/Manila/Ussuri/helper.py +++ b/Manila/Ussuri/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Ussuri/huawei_nas.py b/Manila/Ussuri/huawei_nas.py index 9b91dbb..42a1244 100644 --- a/Manila/Ussuri/huawei_nas.py +++ b/Manila/Ussuri/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -952,6 +955,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -963,6 +967,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Ussuri/huawei_utils.py b/Manila/Ussuri/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Ussuri/huawei_utils.py +++ b/Manila/Ussuri/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Victoria/__init__.py b/Manila/Victoria/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Victoria/__init__.py +++ b/Manila/Victoria/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Victoria/constants.py b/Manila/Victoria/constants.py index 693935e..619c017 100644 --- a/Manila/Victoria/constants.py +++ b/Manila/Victoria/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Victoria/helper.py b/Manila/Victoria/helper.py index 86ce66f..f055044 100644 --- a/Manila/Victoria/helper.py +++ b/Manila/Victoria/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Victoria/huawei_nas.py b/Manila/Victoria/huawei_nas.py index 9b91dbb..42a1244 100644 --- a/Manila/Victoria/huawei_nas.py +++ b/Manila/Victoria/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -952,6 +955,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -963,6 +967,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Victoria/huawei_utils.py b/Manila/Victoria/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Victoria/huawei_utils.py +++ b/Manila/Victoria/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Wallaby/__init__.py b/Manila/Wallaby/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Wallaby/__init__.py +++ b/Manila/Wallaby/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Wallaby/constants.py b/Manila/Wallaby/constants.py index 693935e..619c017 100644 --- a/Manila/Wallaby/constants.py +++ b/Manila/Wallaby/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Wallaby/helper.py b/Manila/Wallaby/helper.py index 86ce66f..f055044 100644 --- a/Manila/Wallaby/helper.py +++ b/Manila/Wallaby/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Wallaby/huawei_nas.py b/Manila/Wallaby/huawei_nas.py index 9b91dbb..42a1244 100644 --- a/Manila/Wallaby/huawei_nas.py +++ b/Manila/Wallaby/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -952,6 +955,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -963,6 +967,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Wallaby/huawei_utils.py b/Manila/Wallaby/huawei_utils.py index 9068a5d..734ff03 100644 --- a/Manila/Wallaby/huawei_utils.py +++ b/Manila/Wallaby/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Xena/__init__.py b/Manila/Xena/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Xena/__init__.py +++ b/Manila/Xena/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Xena/constants.py b/Manila/Xena/constants.py index 693935e..619c017 100644 --- a/Manila/Xena/constants.py +++ b/Manila/Xena/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Xena/helper.py b/Manila/Xena/helper.py index 86ce66f..f055044 100644 --- a/Manila/Xena/helper.py +++ b/Manila/Xena/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Xena/huawei_nas.py b/Manila/Xena/huawei_nas.py index 9b91dbb..42a1244 100644 --- a/Manila/Xena/huawei_nas.py +++ b/Manila/Xena/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -952,6 +955,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -963,6 +967,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Xena/huawei_utils.py b/Manila/Xena/huawei_utils.py index c99f2a8..97ea907 100644 --- a/Manila/Xena/huawei_utils.py +++ b/Manila/Xena/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Yoga/__init__.py b/Manila/Yoga/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Yoga/__init__.py +++ b/Manila/Yoga/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Yoga/constants.py b/Manila/Yoga/constants.py index 693935e..619c017 100644 --- a/Manila/Yoga/constants.py +++ b/Manila/Yoga/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Yoga/helper.py b/Manila/Yoga/helper.py index 86ce66f..f055044 100644 --- a/Manila/Yoga/helper.py +++ b/Manila/Yoga/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Yoga/huawei_nas.py b/Manila/Yoga/huawei_nas.py index 9b91dbb..42a1244 100644 --- a/Manila/Yoga/huawei_nas.py +++ b/Manila/Yoga/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -952,6 +955,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -963,6 +967,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Yoga/huawei_utils.py b/Manila/Yoga/huawei_utils.py index c99f2a8..97ea907 100644 --- a/Manila/Yoga/huawei_utils.py +++ b/Manila/Yoga/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out) diff --git a/Manila/Zed/__init__.py b/Manila/Zed/__init__.py index f06b276..c7da93a 100644 --- a/Manila/Zed/__init__.py +++ b/Manila/Zed/__init__.py @@ -1 +1 @@ -"""Version: 2.5.0""" +"""Version: 2.6.1""" diff --git a/Manila/Zed/constants.py b/Manila/Zed/constants.py index 693935e..619c017 100644 --- a/Manila/Zed/constants.py +++ b/Manila/Zed/constants.py @@ -25,6 +25,8 @@ QOS_ACTIVE = '2' QOS_INACTIVATED = '45' MAX_FS_NUM_IN_QOS = 64 +MAX_QUERY_COUNT = 100 +SENSITIVE_KEYS = ['access_password'] CAPACITY_UNIT = 1024 * 1024 * 2 DEFAULT_WAIT_INTERVAL = 3 @@ -137,6 +139,8 @@ AVAILABLE_FEATURE_STATUS = (1, 2) VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None) SUPPORT_CLONE_PAIR_VERSION = "V600R003C00" +SUPPORT_CLONE_FS_SPLIT_VERSION = "615" +ACTION_START_SPLIT = 1 SNAPSHOT_ROLLBACK_RATE = "100" SNAPSHOT_ROLLBACK_TIMEOUT = 60 * 60 * 24 SNAPSHOT_ROLLING_BACK = "1" diff --git a/Manila/Zed/helper.py b/Manila/Zed/helper.py index 86ce66f..f055044 100644 --- a/Manila/Zed/helper.py +++ b/Manila/Zed/helper.py @@ -15,6 +15,7 @@ import base64 import json +import re import netaddr import requests @@ -237,6 +238,20 @@ def call(self, url, method, data=None, **kwargs): LOG.error('Relogin failed, no need to send again.') return result + @staticmethod + def _get_info_by_range(func, params=None): + range_start = 0 + info_list = [] + while True: + range_end = range_start + constants.MAX_QUERY_COUNT + info = func(range_start, range_end, params) + info_list += info + if len(info) < constants.MAX_QUERY_COUNT: + break + + range_start += constants.MAX_QUERY_COUNT + return info_list + def create_filesystem(self, fs_param): url = "/filesystem" result = self.call(url, 'POST', fs_param) @@ -997,12 +1012,35 @@ def get_controller_id(self, controller_name): if con.get('LOCATION') == controller_name: return con.get("ID") - def split_clone_fs(self, fs_id): + def _get_filesystem_split_url_data(self, fs_id): + """ + Since 6.1.5, the oceanstor storage begin + support filesystem-split-clone. + """ + array_info = self.get_array_info() + point_release = array_info.get("pointRelease", "0") + LOG.info("get array release_version success," + " release_version is %s" % point_release) + # Extracts numbers from point_release + release_version = "".join(re.findall(r"\d+", point_release)) + url = "/filesystem_split_switch" data = {"ID": fs_id, "SPLITENABLE": True, "SPLITSPEED": 4, } - result = self.call("/filesystem_split_switch", "PUT", data) + if release_version >= constants.SUPPORT_CLONE_FS_SPLIT_VERSION: + url = "/clone_fs_split" + data = { + "ID": fs_id, + "action": constants.ACTION_START_SPLIT, + "splitSpeed": 4, + } + + return url, data + + def split_clone_fs(self, fs_id): + (url, data) = self._get_filesystem_split_url_data(fs_id) + result = self.call(url, "PUT", data) _assert_result(result, 'Split clone fs %s error.', fs_id) def create_hypermetro_pair(self, params): @@ -1038,12 +1076,18 @@ def delete_hypermetro_pair(self, pair_id): _assert_result(result, 'Delete HyperMetro pair %s error.', pair_id) def get_hypermetro_domain_id(self, domain_name): - result = self.call("/HyperMetroDomain?range=[0-100]", "GET") - _assert_result(result, "Get HyperMetro domains info error.") - for item in result.get("data", []): + domain_list = self._get_info_by_range(self._get_hypermetro_domain) + for item in domain_list: if item.get("NAME") == domain_name: return item.get("ID") + def _get_hypermetro_domain(self, start, end, params): + url = ("/HyperMetroDomain?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get HyperMetro domains info error.") + return result.get('data', []) + def get_hypermetro_domain_info(self, domain_name): result = self.call("/FsHyperMetroDomain?RUNNINGSTATUS=0", "GET", log_filter=True) @@ -1054,20 +1098,25 @@ def get_hypermetro_domain_info(self, domain_name): def get_hypermetro_vstore_id(self, domain_name, local_vstore_name, remote_vstore_name): - result = self.call("/vstore_pair?range=[0-100]", "GET", - data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair id error.") - for item in result.get("data", []): + vstore_list = self._get_info_by_range(self._get_hypermetro_vstore) + for item in vstore_list: if item.get("DOMAINNAME") == domain_name and item.get( "LOCALVSTORENAME") == local_vstore_name and item.get( "REMOTEVSTORENAME") == remote_vstore_name: return item.get("ID") return None + def _get_hypermetro_vstore(self, start, end, params): + url = ("/vstore_pair?range=[%(start)s-%(end)s]" + % {"start": str(start), "end": str(end)}) + result = self.call(url, "GET") + _assert_result(result, "Get vstore_pair id error.") + return result.get('data', []) + def get_hypermetro_vstore_by_pair_id(self, vstore_pair_id): url = "/vstore_pair/%s" % vstore_pair_id result = self.call(url, 'GET', data=None, log_filter=True) - _assert_result(result, "Get HyperMetro vstore_pair info by id error.") + _assert_result(result, "Get vstore_pair info by id error.") return result["data"] def get_array_info(self): diff --git a/Manila/Zed/huawei_nas.py b/Manila/Zed/huawei_nas.py index 9b91dbb..42a1244 100644 --- a/Manila/Zed/huawei_nas.py +++ b/Manila/Zed/huawei_nas.py @@ -75,6 +75,8 @@ class HuaweiNasDriver(driver.ShareDriver): + VERSION = "2.6.1" + def __init__(self, *args, **kwargs): super(HuaweiNasDriver, self).__init__((True, False), *args, **kwargs) self.configuration.append_config_values(huawei_opts) @@ -839,7 +841,7 @@ def _update_share_stats(self, date=None): data = { 'share_backend_name': backend_name or 'HUAWEI_NAS_Driver', 'vendor_name': 'Huawei', - 'driver_version': '2.5.RC1', + 'driver_version': self.VERSION, 'storage_protocol': 'NFS_CIFS', 'snapshot_support': (self.feature_supports['HyperSnap'] and self.configuration.snapshot_support), @@ -904,7 +906,8 @@ def _get_access_for_share_copy(self, share): access['access_type'] = 'user' LOG.info("Get access %(access)s for share %(share)s copy.", - {'access': access, 'share': share['name']}) + {'access': huawei_utils.mask_dict_sensitive_info(access), + 'share': share['name']}) return access def rpc_create_share_from_hypermetro_snapshot(self, context, share, @@ -952,6 +955,7 @@ def create_share_from_snapshot(self, context, share, snapshot_proto = self._get_share_proto(snapshot) if snapshot_proto == share['share_proto']: + LOG.info("Try to create share by clone") try: location = self._create_from_snapshot_by_clone( context, share, share_fs_id, snapshot_id, share_server) @@ -963,6 +967,7 @@ def create_share_from_snapshot(self, context, share, LOG.warning('Share protocol is inconsistent, will use host copy.') try: + LOG.info("Try to create share by host copy") location = self._create_from_snapshot_by_host( context, share, snapshot, share_server) return location diff --git a/Manila/Zed/huawei_utils.py b/Manila/Zed/huawei_utils.py index c99f2a8..97ea907 100644 --- a/Manila/Zed/huawei_utils.py +++ b/Manila/Zed/huawei_utils.py @@ -337,3 +337,19 @@ def cidr_to_prefixlen(cidr): msg = _("Invalid cidr supplied, reason is %s") % err LOG.error(msg) raise exception.InvalidInput(reason=msg) + + +def mask_dict_sensitive_info(data, secret="***"): + # mask sensitive data in the dictionary + if not isinstance(data, dict): + return data + + out = {} + for key, value in data.items(): + if isinstance(value, dict): + value = mask_dict_sensitive_info(value, secret=secret) + elif key in constants.SENSITIVE_KEYS: + value = secret + out[key] = value + + return strutils.mask_dict_password(out)