Merge branch 'develop' into merge-develop

doc/topics/jinja/index.rst

@@ -1255,8 +1255,52 @@ Returns:
 
   'default'
 
+
+.. jinja_ref:: json_query
+
+``json_query``
+--------------
+
+.. versionadded:: Neon
+
+A port of Ansible ``json_query`` Jinja filter to make queries against JSON data using `JMESPath language`_.
+Could be used to filter ``pillar`` data, ``yaml`` maps, and together with :jinja_ref:`http_query`.
+Depends on the `jmespath`_ Python module.
+
+Examples:
+
+.. code-block:: jinja
+
+  Example 1: {{ [1, 2, 3, 4, [5, 6]] | json_query('[]') }}
+
+  Example 2: {{
+  {"machines": [
+    {"name": "a", "state": "running"},
+    {"name": "b", "state": "stopped"},
+    {"name": "b", "state": "running"}
+  ]} | json_query("machines[?state=='running'].name") }}
+
+  Example 3: {{
+  {"services": [
+    {"name": "http", "host": "1.2.3.4", "port": 80},
+    {"name": "smtp", "host": "1.2.3.5", "port": 25},
+    {"name": "ssh",  "host": "1.2.3.6", "port": 22},
+  ]} | json_query("services[].port") }}
+
+Returns:
+
+.. code-block:: text
+
+  Example 1: [1, 2, 3, 4, 5, 6]
+
+  Example 2: ['a', 'b']
+
+  Example 3: [80, 25, 22]
+
 .. _`builtin filters`: http://jinja.pocoo.org/docs/templates/#builtin-filters
 .. _`timelib`: https://github.com/pediapress/timelib/
+.. _`JMESPath language`: http://jmespath.org/
+.. _`jmespath`: https://github.com/jmespath/jmespath.py
 
 Networking Filters
 ------------------

requirements/tests.txt

@@ -40,3 +40,4 @@ kazoo
 ansible; sys.platform != 'win32' and sys.platform != 'darwin' and python_version >= '3.5'
 ansible; sys.platform != 'win32' and sys.platform != 'darwin' and python_version == '2.7'
 pylxd>=2.2.5; sys.platform != 'win32' and sys.platform != 'darwin'
+jmespath

salt/modules/chocolatey.py

@@ -430,7 +430,8 @@ def install(name,
             .. versionadded:: 2017.7.0
 
         execution_timeout (str):
-        Chocolatey execution timeout value you want to pass to the installation process. Default is None.
+            Chocolatey execution timeout value you want to pass to the
+            installation process. Default is None.
 
             .. versionadded:: 2018.3.0
 

salt/modules/service.py

@@ -41,6 +41,7 @@ def __virtual__():
         'elementary OS',
         'McAfee  OS Server',
         'Raspbian',
+        'SUSE',
     ))
     if __grains__.get('os') in disable:
         return (False, 'Your OS is on the disabled list')

salt/modules/virt.py

@@ -1034,7 +1034,8 @@ def _fill_disk_filename(vm_name, disk, hypervisor, **kwargs):
         else:
             if not base_dir.startswith('/'):
                 # The pool seems not to be a path, lookup for pool infos
-                pool = pool_info(base_dir, **kwargs)
+                infos = pool_info(base_dir, **kwargs)
+                pool = infos[base_dir] if base_dir in infos else None
                 if not pool or not pool['target_path'] or pool['target_path'].startswith('/dev'):
                     raise CommandExecutionError(
                                 'Unable to create new disk {0}, specified pool {1} does not exist '
@@ -1518,9 +1519,9 @@ def init(name,
     .. _graphics element: https://libvirt.org/formatdomain.html#elementsGraphics
     '''
     caps = capabilities(**kwargs)
-    os_types = sorted(set([guest['os_type'] for guest in caps['guests']]))
-    arches = sorted(set([guest['arch']['name'] for guest in caps['guests']]))
-    hypervisors = sorted(set([x for y in [guest['arch']['domains'].keys() for guest in caps['guests']] for x in y]))
+    os_types = sorted({guest['os_type'] for guest in caps['guests']})
+    arches = sorted({guest['arch']['name'] for guest in caps['guests']})
+    hypervisors = sorted({x for y in [guest['arch']['domains'].keys() for guest in caps['guests']] for x in y})
     hypervisor = __salt__['config.get']('libvirt:hypervisor', hypervisor)
     if hypervisor is not None:
         salt.utils.versions.warn_until(
@@ -1790,7 +1791,7 @@ def _diff_disk_lists(old, new):
     :param old: list of ElementTree nodes representing the old disks
     :param new: list of ElementTree nodes representing the new disks
     '''
-    # Fix the target device to avoid duplicates before diffing: this may lead
+    # Change the target device to avoid duplicates before diffing: this may lead
     # to additional changes. Think of unchanged disk 'hda' and another disk listed
     # before it becoming 'hda' too... the unchanged need to turn into 'hdb'.
     targets = []
@@ -1919,10 +1920,11 @@ def update(name,
 
     # Compute the XML to get the disks, interfaces and graphics
     hypervisor = desc.get('type')
+    all_disks = _disk_profile(disk_profile, hypervisor, disks, name, **kwargs)
     new_desc = ElementTree.fromstring(_gen_xml(name,
                                                cpu,
                                                mem,
-                                               _disk_profile(disk_profile, hypervisor, disks, name, **kwargs),
+                                               all_disks,
                                                _get_merged_nics(hypervisor, nic_profile, interfaces),
                                                hypervisor,
                                                domain.OSType(),
@@ -1965,6 +1967,13 @@ def update(name,
 
     # Set the new definition
     if need_update:
+        # Create missing disks if needed
+        if changes['disk']:
+            for idx, item in enumerate(changes['disk']['sorted']):
+                source_file = all_disks[idx]['source_file']
+                if item in changes['disk']['new'] and source_file and not os.path.isfile(source_file):
+                    _qemu_image_create(all_disks[idx])
+
         try:
             conn.defineXML(ElementTree.tostring(desc))
             status['definition'] = True
@@ -2638,7 +2647,7 @@ def get_profiles(hypervisor=None, **kwargs):
     ret = {}
 
     caps = capabilities(**kwargs)
-    hypervisors = sorted(set([x for y in [guest['arch']['domains'].keys() for guest in caps['guests']] for x in y]))
+    hypervisors = sorted({x for y in [guest['arch']['domains'].keys() for guest in caps['guests']] for x in y})
     default_hypervisor = 'kvm' if 'kvm' in hypervisors else hypervisors[0]
 
     if not hypervisor:

salt/states/chocolatey.py

@@ -29,7 +29,7 @@ def __virtual__():
 
 def installed(name, version=None, source=None, force=False, pre_versions=False,
               install_args=None, override_args=False, force_x86=False,
-              package_args=None, allow_multiple=False):
+              package_args=None, allow_multiple=False, execution_timeout=None):
     '''
     Installs a package if not already installed
 
@@ -76,6 +76,10 @@ def installed(name, version=None, source=None, force=False, pre_versions=False,
 
             .. versionadded:: 2017.7.0
 
+        execution_timeout (str):
+            Chocolatey execution timeout value you want to pass to the
+            installation process. Default is None.
+
     .. code-block:: yaml
 
         Installsomepackage:
@@ -176,7 +180,8 @@ def installed(name, version=None, source=None, force=False, pre_versions=False,
                                             override_args=override_args,
                                             force_x86=force_x86,
                                             package_args=package_args,
-                                            allow_multiple=allow_multiple)
+                                            allow_multiple=allow_multiple,
+                                            execution_timeout=execution_timeout)
 
     if 'Running chocolatey failed' not in result:
         ret['result'] = True

salt/utils/data.py

@@ -29,6 +29,11 @@
 from salt.ext import six
 from salt.ext.six.moves import range  # pylint: disable=redefined-builtin
 
+try:
+    import jmespath
+except ImportError:
+    jmespath = None
+
 log = logging.getLogger(__name__)
 
 
@@ -894,3 +899,15 @@ def stringify(data):
             item = six.text_type(item)
         ret.append(item)
     return ret
+
+
+@jinja_filter('json_query')
+def json_query(data, expr):
+    '''
+    Query data using JMESPath language (http://jmespath.org).
+    '''
+    if jmespath is None:
+        err = 'json_query requires jmespath module installed'
+        log.error(err)
+        raise RuntimeError(err)
+    return jmespath.search(expr, data)

salt/utils/mac_utils.py

@@ -354,7 +354,7 @@ def _available_services(refresh=False):
                 # Follow symbolic links of files in _launchd_paths
                 file_path = os.path.join(root, file_name)
                 true_path = os.path.realpath(file_path)
-
+                log.trace('Gathering service info for {}'.format(true_path))
                 # ignore broken symlinks
                 if not os.path.exists(true_path):
                     continue
@@ -397,10 +397,16 @@ def _available_services(refresh=False):
                         logging.warning(msg, true_path)
                         continue
 
-                _available_services[plist['Label'].lower()] = {
-                    'file_name': file_name,
-                    'file_path': true_path,
-                    'plist': plist}
+                try:
+                    # not all launchd plists contain a Label key
+                    _available_services[plist['Label'].lower()] = {
+                        'file_name': file_name,
+                        'file_path': true_path,
+                        'plist': plist}
+                except KeyError:
+                    log.debug('Service {} does not contain a'
+                              ' Label key. Skipping.'.format(true_path))
+                    continue
 
     # put this in __context__ as this is a time consuming function.
     # a fix for this issue. https://github.com/saltstack/salt/issues/48414

tests/unit/modules/test_virt.py

@@ -698,32 +698,32 @@ def test_gen_xml_for_kvm_custom_profile(self):
             self.assertTrue(len(root.findall('.//interface')) == 2)
 
     @patch('salt.modules.virt.pool_info',
-           return_value={'target_path': os.path.join(salt.syspaths.ROOT_DIR,
-                                                     'pools',
-                                                     'default')})
+           return_value={'mypool': {'target_path': os.path.join(salt.syspaths.ROOT_DIR, 'pools', 'mypool')}})
     def test_disk_profile_kvm_disk_pool(self, mock_poolinfo):
         '''
         Test virt._gen_xml(), KVM case with pools defined.
         '''
         disks = {
             'noeffect': [
-                {'first': {'size': 8192, 'pool': 'default'}},
+                {'first': {'size': 8192, 'pool': 'mypool'}},
                 {'second': {'size': 4096}}
             ]
         }
+
+        # pylint: disable=no-member
         with patch.dict(virt.__salt__, {'config.get': MagicMock(side_effect=[
                 disks,
                 os.path.join(salt.syspaths.ROOT_DIR, 'default', 'path')])}):
+
             diskp = virt._disk_profile('noeffect', 'kvm', [], 'hello')
 
-            pools_path = os.path.join(
-                salt.syspaths.ROOT_DIR, 'pools', 'default') + os.sep
-            default_path = os.path.join(
-                salt.syspaths.ROOT_DIR, 'default', 'path') + os.sep
+            pools_path = os.path.join(salt.syspaths.ROOT_DIR, 'pools', 'mypool') + os.sep
+            default_path = os.path.join(salt.syspaths.ROOT_DIR, 'default', 'path') + os.sep
 
             self.assertEqual(len(diskp), 2)
             self.assertTrue(diskp[0]['source_file'].startswith(pools_path))
             self.assertTrue(diskp[1]['source_file'].startswith(default_path))
+        # pylint: enable=no-member
 
     def test_disk_profile_kvm_disk_external_image(self):
         '''
@@ -1185,16 +1185,28 @@ def test_update(self):
         devdetach_mock = MagicMock(return_value=0)
         domain_mock.attachDevice = devattach_mock
         domain_mock.detachDevice = devdetach_mock
-        ret = virt.update('myvm', disk_profile='default', disks=[{'name': 'added', 'size': 2048}])
-        self.assertListEqual(
-            [os.path.join(root_dir, 'myvm_added.qcow2')],
-            [ET.fromstring(disk).find('source').get('file') for disk in ret['disk']['attached']])
-
-        self.assertListEqual(
-            [os.path.join(root_dir, 'myvm_data.qcow2')],
-            [ET.fromstring(disk).find('source').get('file') for disk in ret['disk']['detached']])
-        devattach_mock.assert_called_once()
-        devdetach_mock.assert_called_once()
+        mock_chmod = MagicMock()
+        mock_run = MagicMock()
+        with patch.dict(os.__dict__, {'chmod': mock_chmod}):  # pylint: disable=no-member
+            with patch.dict(virt.__salt__, {'cmd.run': mock_run}):  # pylint: disable=no-member
+                ret = virt.update('myvm', disk_profile='default', disks=[
+                    {'name': 'cddrive', 'device': 'cdrom', 'source_file': None, 'model': 'ide'},
+                    {'name': 'added', 'size': 2048}])
+                added_disk_path = os.path.join(
+                        virt.__salt__['config.get']('virt:images'), 'myvm_added.qcow2')  # pylint: disable=no-member
+                self.assertEqual(mock_run.call_args[0][0],
+                                 'qemu-img create -f qcow2 {0} 2048M'.format(added_disk_path))
+                self.assertEqual(mock_chmod.call_args[0][0], added_disk_path)
+                self.assertListEqual(
+                    [None, os.path.join(root_dir, 'myvm_added.qcow2')],
+                    [ET.fromstring(disk).find('source').get('file') if str(disk).find('<source') > -1 else None
+                     for disk in ret['disk']['attached']])
+
+                self.assertListEqual(
+                    [os.path.join(root_dir, 'myvm_data.qcow2')],
+                    [ET.fromstring(disk).find('source').get('file') for disk in ret['disk']['detached']])
+                self.assertEqual(devattach_mock.call_count, 2)
+                devdetach_mock.assert_called_once()
 
         # Update nics case
         yaml_config = '''
@@ -2182,7 +2194,7 @@ def test_all_capabilities(self):
 
         caps = virt.all_capabilities()
         self.assertEqual('44454c4c-3400-105a-8033-b3c04f4b344a', caps['host']['host']['uuid'])
-        self.assertEqual(set(['qemu', 'kvm']), set([domainCaps['domain'] for domainCaps in caps['domains']]))
+        self.assertEqual(set(['qemu', 'kvm']), {domainCaps['domain'] for domainCaps in caps['domains']})
 
     def test_network_tag(self):
         '''

tests/unit/utils/test_data.py

@@ -598,3 +598,24 @@ def test_stringify(self):
             salt.utils.data.stringify(['one', 'two', str('three'), 4, 5]),  # future lint: disable=blacklisted-function
             ['one', 'two', 'three', '4', '5']
         )
+
+    def test_json_query(self):
+        # Raises exception if jmespath module is not found
+        with patch('salt.utils.data.jmespath', None):
+            self.assertRaisesRegex(
+                RuntimeError, 'requires jmespath',
+                salt.utils.data.json_query, {}, '@'
+            )
+
+        # Test search
+        user_groups = {
+            'user1': {'groups': ['group1', 'group2', 'group3']},
+            'user2': {'groups': ['group1', 'group2']},
+            'user3': {'groups': ['group3']},
+        }
+        expression = '*.groups[0]'
+        primary_groups = ['group1', 'group1', 'group3']
+        self.assertEqual(
+            sorted(salt.utils.data.json_query(user_groups, expression)),
+            primary_groups
+        )

tests/unit/utils/test_jinja.py

@@ -1312,6 +1312,16 @@ def test_base64_decode(self):
                                      dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
         self.assertEqual(rendered, 'random')
 
+    def test_json_query(self):
+        '''
+        Test the `json_query` Jinja filter.
+        '''
+        rendered = render_jinja_tmpl(
+            "{{ [1, 2, 3] | json_query('[1]')}}",
+            dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)
+        )
+        self.assertEqual(rendered, '2')
+
     # def test_print(self):
     #     env = Environment(extensions=[SerializerExtension])
     #     source = '{% import_yaml "toto.foo" as docu %}'