Bugfixes

src/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2016 iNuron NV
+# Copyright (C) 2017 iNuron NV
 #
 # This file is part of Open vStorage Open Source Edition (OSE),
 # as available from
@@ -15,5 +15,5 @@
 # but WITHOUT ANY WARRANTY of any kind.
 
 """
-This package contains various extensions
+This package contains stores, and the required factories to load them
 """

src/api/client.py

@@ -24,9 +24,7 @@
 import logging
 import requests
 from requests.packages.urllib3 import disable_warnings
-from requests.packages.urllib3.exceptions import InsecurePlatformWarning
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-from requests.packages.urllib3.exceptions import SNIMissingWarning
+from requests.packages.urllib3.exceptions import InsecurePlatformWarning, InsecureRequestWarning, SNIMissingWarning
 
 logging.getLogger('urllib3').setLevel(logging.WARNING)
 logger = logging.getLogger(__name__)
@@ -82,6 +80,7 @@ def __init__(self, ip, port, credentials=None, verify=False, version='*', raw_re
         self._version = version
         self._raw_response = raw_response
         try:
+            # noinspection PyUnresolvedReferences
             from ovs_extensions.storage.volatilefactory import VolatileFactory
             self._volatile_client = VolatileFactory.get_client()
         except ImportError:

src/db/arakoon/arakooninstaller.py

@@ -24,12 +24,12 @@
 import json
 import logging
 from ConfigParser import RawConfigParser
+from StringIO import StringIO
 from ovs_extensions.generic.configuration import Configuration
 from ovs_extensions.generic.sshclient import CalledProcessError, SSHClient
 from ovs_extensions.generic.system import System
 from ovs_extensions.generic.volatilemutex import volatile_mutex
 from ovs_extensions.services.servicefactory import ServiceFactory
-from StringIO import StringIO
 
 logger = logging.getLogger(__name__)
 ARAKOON_CLUSTER_TYPES = ['ABM', 'FWK', 'NSM', 'SD', 'CFG']
@@ -983,7 +983,7 @@ def build_client(config):
         :rtype: PyrakoonClient
         """
         if os.environ.get('RUNNING_UNITTESTS') == 'True':
-            from ovs_extensions.db.arakoon.tests.client import MockPyrakoonClient
+            from ovs.extensions.db.arakoon.tests.client import MockPyrakoonClient
             return MockPyrakoonClient(config.cluster_id, None)
 
         from ovs_extensions.db.arakoon.pyrakoon.client import PyrakoonClient

src/db/arakoon/configuration.py

@@ -60,7 +60,7 @@ def dir_exists(self, key):
         :rtype: bool
         """
         key = ArakoonConfiguration._clean_key(key)
-        client = self._get_client()
+        client = self.get_client()
         for entry in list(client.prefix(key)):
             parts = entry.split('/')
             for index in range(len(parts)):
@@ -79,7 +79,7 @@ def list(self, key):
         from ovs_extensions.generic.toolbox import ExtensionsToolbox
 
         key = ArakoonConfiguration._clean_key(key)
-        client = self._get_client()
+        client = self.get_client()
         entries = []
         for entry in client.prefix(key):
             if key == '' or entry.startswith(key.rstrip('/') + '/'):
@@ -98,23 +98,24 @@ def delete(self, key, recursive):
         :return: None
         """
         key = ArakoonConfiguration._clean_key(key)
-        client = self._get_client()
+        client = self.get_client()
         if recursive is True:
             client.delete_prefix(key)
         else:
             client.delete(key)
 
-    def get(self, key):
+    def get(self, key, **kwargs):
         """
         Retrieve the value for specified key
         :param key: Key to retrieve
         :type key: str
         :return: Value of key
         :rtype: str
         """
+        print 'get function {0}'.format(kwargs)
         key = ArakoonConfiguration._clean_key(key)
-        client = self._get_client()
-        return client.get(key)
+        client = self.get_client()
+        return client.get(key, **kwargs)
 
     def set(self, key, value):
         """
@@ -128,10 +129,10 @@ def set(self, key, value):
         if isinstance(value, basestring):
             value = str(value)
         key = ArakoonConfiguration._clean_key(key)
-        client = self._get_client()
+        client = self.get_client()
         client.set(key, value)
 
-    def _get_client(self):
+    def get_client(self):
         """
         Builds a PyrakoonClient
         :return: A PyrakoonClient instance

src/generic/configuration.py

@@ -100,9 +100,10 @@ def get(cls, key, raw=False, **kwargs):
         :param raw: Raw data if True else json format
         :return: Value for key
         """
+        print 'public {0}'.format(kwargs)
         try:
             key_entries = key.split('|')
-            data = cls._get(key_entries[0], raw)
+            data = cls._get(key_entries[0], raw, **kwargs)
             if len(key_entries) == 1:
                 return data
             try:
@@ -205,6 +206,13 @@ def list(cls, key):
         """
         return cls._list(key)
 
+    @classmethod
+    def get_client(cls):
+        """
+        Retrieve a configuration store client
+        """
+        return cls._passthrough(method='get_client')
+
     @classmethod
     def _dir_exists(cls, key):
         # Unittests
@@ -265,7 +273,7 @@ def _delete(cls, key, recursive):
                                 key=key, recursive=recursive)
 
     @classmethod
-    def _get(cls, key, raw):
+    def _get(cls, key, raw, **kwargs):
         # Unittests
         if os.environ.get('RUNNING_UNITTESTS') == 'True':
             if key in ['', '/']:
@@ -284,8 +292,10 @@ def _get(cls, key, raw):
                 data = None
         else:
             # Forward call to used configuration store
+            print 'private {0}'.format(kwargs)
             data = cls._passthrough(method='get',
-                                    key=key)
+                                    key=key,
+                                    **kwargs)
         if raw is True:
             return data
         return json.loads(data)
@@ -328,7 +338,7 @@ def _passthrough(cls, method, *args, **kwargs):
     def get_store_info(cls):
         """
         Retrieve the configuration store method. This can currently only be 'arakoon'
-        :return: A tuple containing the store and params that can be bassed to the configuration implementation instance
+        :return: A tuple containing the store and params that can be passed to the configuration implementation instance
         :rtype: tuple(str, dict)
         """
         raise NotImplementedError()

src/generic/filemutex.py

@@ -80,6 +80,7 @@ def acquire(self, wait=None):
         self._start = time.time()
         if wait is None:
             wait = self._wait
+        passed = 0
         if wait is None:
             fcntl.flock(self._handle, fcntl.LOCK_EX)
             passed = time.time() - self._start

src/generic/remote.py

@@ -19,9 +19,9 @@
 
 import os
 import sys
-from subprocess import check_output
-from rpyc.utils.zerodeploy import DeployedServer
 from plumbum import SshMachine
+from rpyc.utils.zerodeploy import DeployedServer
+from subprocess import check_output
 
 
 class remote(object):

src/generic/timer.py

@@ -40,6 +40,7 @@ def __enter__(self):
         self.start = time.time()
 
     def __exit__(self, *args):
+        _ = args
         duration = time.time() - self.start
         if duration > 2 and self.force_ms is not True:
             logger.debug('{0} took {1:.5f}s'.format(self.identification, duration))

src/generic/toolbox.py

@@ -17,12 +17,20 @@
 """
 ExtensionsToolbox module
 """
+import re
+import sys
 
 
 class ExtensionsToolbox(object):
     """
     Generic class for various methods
     """
+    regex_ip = re.compile('^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))$')
+    regex_guid = re.compile('^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$')
+    regex_vpool = re.compile('^[0-9a-z][\-a-z0-9]{1,20}[a-z0-9]$')
+    regex_preset = re.compile('^[0-9a-zA-Z][a-zA-Z0-9-_]{1,18}[a-zA-Z0-9]$')
+    compiled_regex_type = type(re.compile('some_regex'))
+
     @staticmethod
     def remove_prefix(string, prefix):
         """
@@ -67,6 +75,110 @@ def edit_version_file(client, package_name, old_service_name, new_service_name=N
                 client.file_write(filename=run_file, contents=contents)
                 client.file_chown(filenames=[run_file], user='ovs', group='ovs')
 
+    @staticmethod
+    def check_type(value, required_type):
+        """
+        Validates whether a certain value is of a given type. Some types are treated as special
+        case:
+          - A 'str' type accepts 'str', 'unicode' and 'basestring'
+          - A 'float' type accepts 'float', 'int'
+          - A list instance acts like an enum
+        :param value: Value to check
+        :param required_type: Expected type for value
+        """
+        given_type = type(value)
+        if required_type is str:
+            correct = isinstance(value, basestring)
+            allowed_types = ['str', 'unicode', 'basestring']
+        elif required_type is float:
+            correct = isinstance(value, float) or isinstance(value, int)
+            allowed_types = ['float', 'int']
+        elif required_type is int:
+            correct = isinstance(value, int) or isinstance(value, long)
+            allowed_types = ['int', 'long']
+        elif isinstance(required_type, list):
+            # We're in an enum scenario. Field_type isn't a real type, but a list containing
+            # all possible enum values. Here as well, we need to do some str/unicode/basestring
+            # checking.
+            if isinstance(required_type[0], basestring):
+                value = str(value)
+            correct = value in required_type
+            allowed_types = required_type
+            given_type = value
+        else:
+            correct = isinstance(value, required_type)
+            allowed_types = [required_type.__name__]
+
+        return correct, allowed_types, given_type
+
+    @staticmethod
+    def verify_required_params(required_params, actual_params, verify_keys=False):
+        """
+        Verify whether the actual parameters match the required parameters
+        :param required_params: Required parameters which actual parameters have to meet
+        :type required_params: dict
+        :param actual_params: Actual parameters to check for validity
+        :type actual_params: dict
+        :param verify_keys: Verify whether the passed in keys are actually part of the required keys
+        :type verify_keys: bool
+        :return: None
+        :rtype: NoneType
+        """
+        if not isinstance(required_params, dict) or not isinstance(actual_params, dict):
+            raise RuntimeError('Required and actual parameters must be of type dictionary')
+
+        error_messages = []
+        if verify_keys is True:
+            for key in actual_params:
+                if key not in required_params:
+                    error_messages.append('Specified parameter "{0}" is not valid'.format(key))
+
+        for required_key, key_info in required_params.iteritems():
+            expected_type = key_info[0]
+            expected_value = key_info[1]
+            optional = len(key_info) == 3 and key_info[2] is False
+
+            if optional is True and (required_key not in actual_params or actual_params[required_key] in ('', None)):
+                continue
+
+            if required_key not in actual_params:
+                error_messages.append('Missing required param "{0}" in actual parameters'.format(required_key))
+                continue
+
+            mandatory_or_optional = 'Optional' if optional is True else 'Mandatory'
+            actual_value = actual_params[required_key]
+            if ExtensionsToolbox.check_type(actual_value, expected_type)[0] is False:
+                error_messages.append('{0} param "{1}" is of type "{2}" but we expected type "{3}"'.format(mandatory_or_optional, required_key, type(actual_value), expected_type))
+                continue
+
+            if expected_value is None:
+                continue
+
+            if expected_type == list:
+                if type(expected_value) == ExtensionsToolbox.compiled_regex_type:  # List of strings which need to match regex
+                    for item in actual_value:
+                        if not re.match(expected_value, item):
+                            error_messages.append('{0} param "{1}" has an item "{2}" which does not match regex "{3}"'.format(mandatory_or_optional, required_key, item, expected_value.pattern))
+            elif expected_type == dict:
+                ExtensionsToolbox.verify_required_params(expected_value, actual_params[required_key])
+            elif expected_type == int or expected_type == float:
+                if isinstance(expected_value, list) and actual_value not in expected_value:
+                    error_messages.append('{0} param "{1}" with value "{2}" should be 1 of the following: {3}'.format(mandatory_or_optional, required_key, actual_value, expected_value))
+                if isinstance(expected_value, dict):
+                    minimum = expected_value.get('min', sys.maxint * -1)
+                    maximum = expected_value.get('max', sys.maxint)
+                    if not minimum <= actual_value <= maximum:
+                        error_messages.append('{0} param "{1}" with value "{2}" should be in range: {3} - {4}'.format(mandatory_or_optional, required_key, actual_value, minimum, maximum))
+                    if actual_value in expected_value.get('exclude', []):
+                        error_messages.append('{0} param "{1}" cannot have value {2}'.format(mandatory_or_optional, required_key, actual_value))
+            else:
+                if ExtensionsToolbox.check_type(expected_value, list)[0] is True and actual_value not in expected_value:
+                    error_messages.append('{0} param "{1}" with value "{2}" should be 1 of the following: {3}'.format(mandatory_or_optional, required_key, actual_value, expected_value))
+                elif ExtensionsToolbox.check_type(expected_value, ExtensionsToolbox.compiled_regex_type)[0] is True and not re.match(expected_value, actual_value):
+                    error_messages.append('{0} param "{1}" with value "{2}" does not match regex "{3}"'.format(mandatory_or_optional, required_key, actual_value, expected_value.pattern))
+        if error_messages:
+            raise RuntimeError('\n' + '\n'.join(error_messages))
+
     @staticmethod
     def advanced_sort(element, separator):
         """

src/os/interfaces/centos.py

@@ -18,8 +18,7 @@
 Centos OS module
 """
 
-from subprocess import CalledProcessError
-from subprocess import check_output
+from subprocess import CalledProcessError, check_output
 from ovs_extensions.generic.configuration import Configuration
 from ovs_extensions.generic.system import System
 

src/os/interfaces/ubuntu.py

@@ -18,8 +18,7 @@
 Ubuntu OS module
 """
 
-from subprocess import CalledProcessError
-from subprocess import check_output
+from subprocess import CalledProcessError, check_output
 from ovs_extensions.generic.configuration import Configuration
 from ovs_extensions.generic.system import System
 

src/os/osfactory.py

@@ -19,8 +19,8 @@
 """
 import logging
 from subprocess import check_output
-from ovs_extensions.os.ubuntu import Ubuntu
-from ovs_extensions.os.centos import Centos
+from ovs_extensions.os.interfaces.centos import Centos
+from ovs_extensions.os.interfaces.ubuntu import Ubuntu
 
 logger = logging.getLogger(__name__)
 

src/services/interfaces/systemd.py

@@ -21,6 +21,7 @@
 import time
 import logging
 from subprocess import CalledProcessError, check_output
+from ovs_extensions.generic.configuration import Configuration
 from ovs_extensions.generic.system import System
 from ovs_extensions.generic.toolbox import ExtensionsToolbox
 from ovs_extensions.services.interfaces.manager import Manager

src/services/mockups/systemd.py

@@ -21,13 +21,13 @@
 from ovs_extensions.generic.configuration import Configuration
 from ovs_extensions.generic.system import System
 from ovs_extensions.generic.toolbox import ExtensionsToolbox
+from ovs_extensions.services.interfaces.manager import Manager
 
 
-class SystemdMock(object):
+class SystemdMock(Manager):
     """
     Contains all logic related to Systemd Mock services
     """
-    SERVICE_CONFIG_KEY = '/ovs/framework/hosts/{0}/services/{1}'
     services = {}
 
     @staticmethod