Merge pull request #3612 from StackStorm/more_v2.3.2_changes

More changes for v2.3.2

CHANGELOG.rst

@@ -18,7 +18,11 @@ Added
   Contributed by Nick Maludy. #3508
 * Update ``st2`` CLI so it also displays "there are more results" note when ``-n`` flag is
   used and there are more items available. (improvement) #3552
-* Add ability to explicitly set ``stream_url`` in st2client and CLI. (improvement) #3432
+* Add ability to explicitly set ``stream_url`` in st2client. (improvement) #3432
+* Add support for handling arrays of dictionaries to ``st2 config`` CLI command. (improvement)
+  #3594
+
+  Contributed by Hiroyasu OHYAMA.
 
 Fixed
 ~~~~~
@@ -42,6 +46,10 @@ Fixed
   Reported by sibirajal.
 * Add a check to make sure action exists in the POST of the action execution API. (bug fix)
 * Fix api key generation, to use system user, when auth is disabled. (bug fix) #3578 #3593
+* Fix invocation of Mistral workflow from Action Chain with jinja in params. (bug fix) #3440
+* Fix st2client API bug, a backward incompatible change in `query()` method, introduced in note
+  implementation (#3514) in 2.3.1. The `query()` method is now backward compatible (pre 2.3) and
+  `query_with_count()` method is used for results pagination and note. #3616
 
 2.3.1 - July 07, 2017
 ---------------------

contrib/examples/actions/chains/invoke_mistral_with_jinja.yaml

@@ -0,0 +1,6 @@
+chain:
+    -
+        name: task1
+        ref: examples.mistral-basic
+        params:
+            cmd: "{{cmd}}"

contrib/examples/actions/invoke_mistral_with_jinja.yaml

@@ -0,0 +1,10 @@
+---
+name: invoke-mistral-with-jinja
+description: Example to invoke a Mistral workflow from an Action Chain
+runner_type: action-chain
+entry_point: chains/invoke_mistral_with_jinja.yaml
+enabled: true
+parameters:
+    cmd:
+        type: string
+        required: true

contrib/runners/mistral_v2/mistral_v2.py

@@ -29,6 +29,7 @@
 from st2common.persistence.execution import ActionExecution
 from st2common.persistence.liveaction import LiveAction
 from st2common.services import action as action_service
+from st2common.util import jinja
 from st2common.util.workflow import mistral as utils
 from st2common.util.url import get_url_without_trailing_slash
 from st2common.util.api import get_full_public_api_url
@@ -161,9 +162,20 @@ def _construct_workflow_execution_options(self):
         parent_context = {
             'execution_id': self.execution_id
         }
+
         if getattr(self.liveaction, 'context', None):
             parent_context.update(self.liveaction.context)
 
+        # Convert jinja expressions in the params of Action Chain under the parent context
+        # into raw block. If there is any jinja expressions, Mistral will try to evaulate
+        # the expression. If there is a local context reference, the evaluation will fail
+        # because the local context reference is out of scope.
+        chain_ctx = parent_context.get('chain') or {}
+        chain_params_ctx = chain_ctx.get('params') or {}
+
+        for k, v in six.iteritems(chain_params_ctx):
+            parent_context['chain']['params'][k] = jinja.convert_jinja_to_raw_block(v)
+
         st2_execution_context = {
             'api_url': api_url,
             'endpoint': endpoint,

contrib/runners/mistral_v2/tests/unit/test_mistral_v2.py

@@ -270,6 +270,193 @@ def test_launch_workflow(self):
         executions.ExecutionManager.create.assert_called_with(
             WF1_NAME, workflow_input=workflow_input, env=env)
 
+    @mock.patch.object(
+        workflows.WorkflowManager, 'list',
+        mock.MagicMock(return_value=[]))
+    @mock.patch.object(
+        workflows.WorkflowManager, 'get',
+        mock.MagicMock(return_value=WF1))
+    @mock.patch.object(
+        workflows.WorkflowManager, 'create',
+        mock.MagicMock(return_value=[WF1]))
+    @mock.patch.object(
+        executions.ExecutionManager, 'create',
+        mock.MagicMock(return_value=executions.Execution(None, WF1_EXEC)))
+    def test_launch_workflow_under_parent_chain_with_jinja_params(self):
+        ac_ctx = {
+            'chain': {
+                'params': {
+                    'var1': 'foobar',
+                    'var2': '{{foobar}}',
+                    'var3': ['{{foo}}', '{{bar}}'],
+                    'var4': {
+                        'foobar': '{{foobar}}'
+                    },
+                    'var5': {
+                        'foobar': '{% for item in items %}foobar{% end for %}'
+                    }
+                }
+            }
+        }
+
+        liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ac_ctx)
+        liveaction, execution = action_service.request(liveaction)
+        liveaction = LiveAction.get_by_id(str(liveaction.id))
+        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
+
+        mistral_context = liveaction.context.get('mistral', None)
+        self.assertIsNotNone(mistral_context)
+        self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
+        self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
+
+        workflow_input = copy.deepcopy(ACTION_PARAMS)
+        workflow_input.update({'count': '3'})
+
+        env = {
+            'st2_execution_id': str(execution.id),
+            'st2_liveaction_id': str(liveaction.id),
+            'st2_action_api_url': 'http://0.0.0.0:9101/v1',
+            '__actions': {
+                'st2.action': {
+                    'st2_context': {
+                        'api_url': 'http://0.0.0.0:9101/v1',
+                        'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
+                        'parent': {
+                            'execution_id': str(execution.id),
+                            'chain': {
+                                'params': {
+                                    'var1': 'foobar',
+                                    'var2': '{% raw %}{{foobar}}{% endraw %}',
+                                    'var3': [
+                                        '{% raw %}{{foo}}{% endraw %}',
+                                        '{% raw %}{{bar}}{% endraw %}'
+                                    ],
+                                    'var4': {
+                                        'foobar': '{% raw %}{{foobar}}{% endraw %}'
+                                    },
+                                    'var5': {
+                                        'foobar': (
+                                            '{% raw %}{% for item in items %}'
+                                            'foobar{% end for %}{% endraw %}'
+                                        )
+                                    }
+                                }
+                            }
+                        },
+                        'notify': {},
+                        'skip_notify_tasks': []
+                    }
+                }
+            }
+        }
+
+        executions.ExecutionManager.create.assert_called_with(
+            WF1_NAME, workflow_input=workflow_input, env=env)
+
+    @mock.patch.object(
+        workflows.WorkflowManager, 'list',
+        mock.MagicMock(return_value=[]))
+    @mock.patch.object(
+        workflows.WorkflowManager, 'get',
+        mock.MagicMock(return_value=WF1))
+    @mock.patch.object(
+        workflows.WorkflowManager, 'create',
+        mock.MagicMock(return_value=[WF1]))
+    @mock.patch.object(
+        executions.ExecutionManager, 'create',
+        mock.MagicMock(return_value=executions.Execution(None, WF1_EXEC)))
+    def test_launch_workflow_under_parent_chain_with_nonetype_in_chain_context(self):
+        ac_ctx = {'chain': None}
+
+        liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ac_ctx)
+        liveaction, execution = action_service.request(liveaction)
+        liveaction = LiveAction.get_by_id(str(liveaction.id))
+        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
+
+        mistral_context = liveaction.context.get('mistral', None)
+        self.assertIsNotNone(mistral_context)
+        self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
+        self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
+
+        workflow_input = copy.deepcopy(ACTION_PARAMS)
+        workflow_input.update({'count': '3'})
+
+        env = {
+            'st2_execution_id': str(execution.id),
+            'st2_liveaction_id': str(liveaction.id),
+            'st2_action_api_url': 'http://0.0.0.0:9101/v1',
+            '__actions': {
+                'st2.action': {
+                    'st2_context': {
+                        'api_url': 'http://0.0.0.0:9101/v1',
+                        'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
+                        'parent': {
+                            'execution_id': str(execution.id),
+                            'chain': None
+                        },
+                        'notify': {},
+                        'skip_notify_tasks': []
+                    }
+                }
+            }
+        }
+
+        executions.ExecutionManager.create.assert_called_with(
+            WF1_NAME, workflow_input=workflow_input, env=env)
+
+    @mock.patch.object(
+        workflows.WorkflowManager, 'list',
+        mock.MagicMock(return_value=[]))
+    @mock.patch.object(
+        workflows.WorkflowManager, 'get',
+        mock.MagicMock(return_value=WF1))
+    @mock.patch.object(
+        workflows.WorkflowManager, 'create',
+        mock.MagicMock(return_value=[WF1]))
+    @mock.patch.object(
+        executions.ExecutionManager, 'create',
+        mock.MagicMock(return_value=executions.Execution(None, WF1_EXEC)))
+    def test_launch_workflow_under_parent_chain_with_nonetype_in_params_context(self):
+        ac_ctx = {'chain': {'params': None}}
+
+        liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS, context=ac_ctx)
+        liveaction, execution = action_service.request(liveaction)
+        liveaction = LiveAction.get_by_id(str(liveaction.id))
+        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_RUNNING)
+
+        mistral_context = liveaction.context.get('mistral', None)
+        self.assertIsNotNone(mistral_context)
+        self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
+        self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))
+
+        workflow_input = copy.deepcopy(ACTION_PARAMS)
+        workflow_input.update({'count': '3'})
+
+        env = {
+            'st2_execution_id': str(execution.id),
+            'st2_liveaction_id': str(liveaction.id),
+            'st2_action_api_url': 'http://0.0.0.0:9101/v1',
+            '__actions': {
+                'st2.action': {
+                    'st2_context': {
+                        'api_url': 'http://0.0.0.0:9101/v1',
+                        'endpoint': 'http://0.0.0.0:9101/v1/actionexecutions',
+                        'parent': {
+                            'execution_id': str(execution.id),
+                            'chain': {
+                                'params': None
+                            }
+                        },
+                        'notify': {},
+                        'skip_notify_tasks': []
+                    }
+                }
+            }
+        }
+
+        executions.ExecutionManager.create.assert_called_with(
+            WF1_NAME, workflow_input=workflow_input, env=env)
+
     @mock.patch.object(
         workflows.WorkflowManager, 'list',
         mock.MagicMock(return_value=[]))

st2api/st2api/controllers/v1/timers.py

@@ -87,7 +87,8 @@ def __init__(self):
 
     def get_all(self, timer_type=None):
         if timer_type and timer_type not in self._allowed_timer_types:
-            msg = 'Timer type %s not in supported types - %s.' % self._allowed_timer_types
+            msg = 'Timer type %s not in supported types - %s.' % (timer_type,
+                                                                  self._allowed_timer_types)
             abort(http_client.BAD_REQUEST, msg)
 
         t_all = self._timers.get_all(timer_type=timer_type)

st2client/st2client/commands/action.py

@@ -1062,8 +1062,7 @@ def run(self, args, **kwargs):
         exclude_attributes = ','.join(exclude_attributes)
         kwargs['exclude_attributes'] = exclude_attributes
 
-        result, count = self.manager.query(limit=args.last, **kwargs)
-        return (result, count)
+        return self.manager.query_with_count(limit=args.last, **kwargs)
 
     def run_and_print(self, args, **kwargs):
 
@@ -1084,7 +1083,7 @@ def run_and_print(self, args, **kwargs):
                               attributes=args.attr, widths=args.width,
                               attribute_transform_functions=self.attribute_transform_functions)
 
-            if args.last and count and int(count) > args.last:
+            if args.last and count and count > args.last:
                 table.SingleRowTable.note_box(self.resource_name, args.last)
 
 

st2client/st2client/commands/policy.py

@@ -49,7 +49,7 @@ def run(self, args, **kwargs):
         if args.resource_type:
             filters = {'resource_type': args.resource_type}
             filters.update(**kwargs)
-            instances, _ = self.manager.query(**filters)
+            instances = self.manager.query(**filters)
             return instances
         else:
             return self.manager.get_all(**kwargs)

st2client/st2client/commands/rbac.py

@@ -72,7 +72,7 @@ def run(self, args, **kwargs):
             kwargs['system'] = args.system
 
         if args.system:
-            result, _ = self.manager.query(**kwargs)
+            result = self.manager.query(**kwargs)
         else:
             result = self.manager.get_all(**kwargs)
 
@@ -143,7 +143,7 @@ def run(self, args, **kwargs):
             kwargs['remote'] = args.remote
 
         if args.role or args.user or args.remote:
-            result, _ = self.manager.query(**kwargs)
+            result = self.manager.query(**kwargs)
         else:
             result = self.manager.get_all(**kwargs)
 

st2client/st2client/commands/rule.py

@@ -83,8 +83,7 @@ def run(self, args, **kwargs):
             # switch attr to display the trigger and action
             args.attr = self.display_attributes_iftt
 
-        result, count = self.manager.query(limit=args.last, **kwargs)
-        return (result, count)
+        return self.manager.query_with_count(limit=args.last, **kwargs)
 
     def run_and_print(self, args, **kwargs):
         instances, count = self.run(args, **kwargs)
@@ -96,7 +95,7 @@ def run_and_print(self, args, **kwargs):
             self.print_output(instances, table.MultiColumnTable,
                               attributes=args.attr, widths=args.width)
 
-            if args.last and count and int(count) > args.last:
+            if args.last and count and count > args.last:
                 table.SingleRowTable.note_box(self.resource_name, args.last)
 
 

st2client/st2client/commands/rule_enforcement.py

@@ -110,8 +110,7 @@ def run(self, args, **kwargs):
         if args.timestamp_lt:
             kwargs['enforced_at_lt'] = args.timestamp_lt
 
-        result, count = self.manager.query(limit=args.last, **kwargs)
-        return (result, count)
+        return self.manager.query_with_count(limit=args.last, **kwargs)
 
     def run_and_print(self, args, **kwargs):
         instances, count = self.run(args, **kwargs)
@@ -123,5 +122,5 @@ def run_and_print(self, args, **kwargs):
         else:
             self.print_output(instances, table.MultiColumnTable,
                               attributes=args.attr, widths=args.width)
-            if args.last and count and int(count) > args.last:
+            if args.last and count and count > args.last:
                 table.SingleRowTable.note_box(self.resource_name, args.last)

st2client/st2client/commands/timer.py

@@ -46,8 +46,7 @@ def run(self, args, **kwargs):
             kwargs['timer_type'] = args.timer_type
 
         if kwargs:
-            result, _ = self.manager.query(**kwargs)
-            return result
+            return self.manager.query(**kwargs)
         else:
             return self.manager.get_all(**kwargs)
 

st2client/st2client/commands/trace.py

@@ -163,8 +163,7 @@ def run(self, args, **kwargs):
                 kwargs['sort_asc'] = True
             elif args.sort_order in ['desc', 'descending']:
                 kwargs['sort_desc'] = True
-        result, count = self.manager.query(limit=args.last, **kwargs)
-        return (result, count)
+        return self.manager.query_with_count(limit=args.last, **kwargs)
 
     def run_and_print(self, args, **kwargs):
         instances, count = self.run(args, **kwargs)
@@ -177,7 +176,7 @@ def run_and_print(self, args, **kwargs):
             self.print_trace_details(trace=instances[0], args=args)
 
             if not args.json and not args.yaml:
-                if args.last and count and int(count) > args.last:
+                if args.last and count and count > args.last:
                         table.SingleRowTable.note_box(self.resource_name, 1)
         else:
             if args.json or args.yaml:
@@ -190,7 +189,7 @@ def run_and_print(self, args, **kwargs):
                                   attributes=args.attr, widths=args.width,
                                   attribute_transform_functions=self.attribute_transform_functions)
 
-                if args.last and count and int(count) > args.last:
+                if args.last and count and count > args.last:
                     table.SingleRowTable.note_box(self.resource_name, args.last)