From 289debf93d1a0e5ae2d38c62b6bd3e86704b654c Mon Sep 17 00:00:00 2001 From: Alec Clowes Date: Thu, 29 Jun 2017 12:09:08 -0400 Subject: [PATCH 1/3] - add docstring - fix link - banner fiddling --- frontend/src/TaskDetail.js | 4 ++-- yawn/manage.py | 7 +++---- yawn/management/commands/exec.py | 17 +++++++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/frontend/src/TaskDetail.js b/frontend/src/TaskDetail.js index f6ded38..42ca446 100644 --- a/frontend/src/TaskDetail.js +++ b/frontend/src/TaskDetail.js @@ -63,7 +63,7 @@ export default class TaskDetail extends React.Component {
{execution.status}
Worker
- + {execution.worker.name}
@@ -130,4 +130,4 @@ export default class TaskDetail extends React.Component { ) } } -} \ No newline at end of file +} diff --git a/yawn/manage.py b/yawn/manage.py index 0508904..45eb688 100755 --- a/yawn/manage.py +++ b/yawn/manage.py @@ -12,16 +12,15 @@ def main(): # check if yawn is in installed apps, and bail if it is not if 'yawn' not in settings.INSTALLED_APPS: print("Please check your DJANGO_SETTINGS_MODULE environment variable.\n" - "Make sure 'yawn' must be in your INSTALLED_APPS.\n" + "Make sure 'yawn' is in your INSTALLED_APPS.\n" "Generally, your settings file should start with 'from yawn.settings.base import *'") sys.exit(1) - print('\nYAWN workflow management tool') - + print('YAWN workflow management tool') if os.environ['DJANGO_SETTINGS_MODULE'] == 'yawn.settings.debug': print(' Running in DEBUG mode') - # run the django manage.py commandline + # run the django manage.py command line execute_from_command_line(sys.argv) diff --git a/yawn/management/commands/exec.py b/yawn/management/commands/exec.py index a826190..56d71ff 100644 --- a/yawn/management/commands/exec.py +++ b/yawn/management/commands/exec.py @@ -1,4 +1,9 @@ -import argparse +""" +This helper command will import any python callable on your python path and +call it with the supplied arguments. + +Use `yawn.task.decorators.make_task` to +""" import importlib from django.core.management.base import BaseCommand @@ -8,15 +13,15 @@ class Command(BaseCommand): help = 'Execute a python callable' def add_arguments(self, parser): - parser.add_argument('module') - parser.add_argument('callable') - parser.add_argument('arguments', nargs=argparse.REMAINDER) + parser.add_argument('module', help='The python module to import, i.e. animal.bird') + parser.add_argument('callable', help='The python callable to invoke, i.e. Swallow') + parser.add_argument('argument', nargs='+', help='Arguments to pass to the callable') def handle(self, *args, **options): self.stdout.write('Importing module %s' % options['module']) module_ = importlib.import_module(options['module']) - self.stdout.write('Calling %s("%s")' % (options['callable'], '", "'.join(options['arguments']))) - getattr(module_, options['callable'])(*options['arguments']) + self.stdout.write('Calling %s("%s")' % (options['callable'], '", "'.join(options['argument']))) + getattr(module_, options['callable'])(*options['argument']) self.stdout.write('Execution complete') From d978b2acfd687565939951ca74db2594f9a33656 Mon Sep 17 00:00:00 2001 From: Alec Clowes Date: Thu, 29 Jun 2017 12:23:27 -0400 Subject: [PATCH 2/3] also send /api-auth/ to the application --- yawn/management/commands/webserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yawn/management/commands/webserver.py b/yawn/management/commands/webserver.py index 30d4dc6..195d0ec 100644 --- a/yawn/management/commands/webserver.py +++ b/yawn/management/commands/webserver.py @@ -15,7 +15,7 @@ def __call__(self, environ, start_response): and we return index.html, and then react-router interprets the path. """ path = decode_path_info(environ['PATH_INFO']) - if path.startswith('/api/'): + if path.startswith('/api'): return self.application(environ, start_response) static_file = self.files.get(path) if static_file is None: From a3ad536aeec7ddec992dccbe3ac5038106153ee6 Mon Sep 17 00:00:00 2001 From: Alec Clowes Date: Thu, 29 Jun 2017 15:13:18 -0400 Subject: [PATCH 3/3] allow tasks without an associated workflow --- frontend/src/App.css | 8 +++- frontend/src/ExecutionTable.js | 4 +- frontend/src/TaskDetail.js | 17 ++++++-- .../__snapshots__/TaskDetail.test.js.snap | 4 +- yawn/conftest.py | 4 +- yawn/management/commands/exec.py | 8 +++- .../0002_tasks_without_workflows.py | 42 +++++++++++++++++++ yawn/task/helpers.py | 31 ++++++++++++++ yawn/task/models.py | 9 ++-- yawn/task/serializers.py | 4 +- yawn/task/tests/test_helpers.py | 35 ++++++++++++++++ yawn/task/tests/test_models.py | 14 ++++++- yawn/worker/main.py | 2 +- yawn/worker/models.py | 2 +- yawn/worker/serializers.py | 2 +- yawn/worker/views.py | 2 +- yawn/workflow/models.py | 2 +- yawn/workflow/serializers.py | 5 +-- yawn/workflow/tests/test_views.py | 4 +- 19 files changed, 170 insertions(+), 29 deletions(-) create mode 100644 yawn/migrations/0002_tasks_without_workflows.py create mode 100644 yawn/task/helpers.py create mode 100644 yawn/task/tests/test_helpers.py diff --git a/frontend/src/App.css b/frontend/src/App.css index 0a92972..2218d79 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -8,6 +8,12 @@ margin: 0; } +pre { + margin: 0; + padding: 5px; + display: inline-block; +} + /* Status Colors */ .waiting { @@ -41,4 +47,4 @@ g.edgePath path { stroke: #333; fill: #333; stroke-width: 1px; -} \ No newline at end of file +} diff --git a/frontend/src/ExecutionTable.js b/frontend/src/ExecutionTable.js index 305b92c..e7f38ae 100644 --- a/frontend/src/ExecutionTable.js +++ b/frontend/src/ExecutionTable.js @@ -16,7 +16,7 @@ export default class ExecutionTable extends React.Component { return this.props.executions.map((execution) => ( {execution.worker.name} - {execution.task.workflow.name} + {execution.task.workflow && execution.task.workflow.name} {execution.task.name} {execution.status} {execution.start_timestamp} @@ -45,4 +45,4 @@ export default class ExecutionTable extends React.Component { ) } -} \ No newline at end of file +} diff --git a/frontend/src/TaskDetail.js b/frontend/src/TaskDetail.js index 42ca446..56b947c 100644 --- a/frontend/src/TaskDetail.js +++ b/frontend/src/TaskDetail.js @@ -42,6 +42,15 @@ export default class TaskDetail extends React.Component { )) } + renderWorkflowLink() { + const task = this.state.task; + if (task.workflow) return ( + + {task.workflow.name} - v{task.workflow.version} + + ) + } + renderExecution() { if (this.state.execution === 0) { return
No executions
@@ -102,14 +111,14 @@ export default class TaskDetail extends React.Component {
Workflow
- - {task.workflow.name} - v{task.workflow.version} - + {this.renderWorkflowLink()}
Task Name
{task.name}
Command
-
{JSON.stringify(task.command)}
+
+ 
{task.command}
+
Max Retries
{task.max_retries}
Timeout
diff --git a/frontend/src/tests/__snapshots__/TaskDetail.test.js.snap b/frontend/src/tests/__snapshots__/TaskDetail.test.js.snap index 55c0437..6bda5c2 100644 --- a/frontend/src/tests/__snapshots__/TaskDetail.test.js.snap +++ b/frontend/src/tests/__snapshots__/TaskDetail.test.js.snap @@ -61,7 +61,9 @@ exports[`TaskDetail success 1`] = ` Command
- "echo Starting..." + 
+              echo Starting...
+
Max Retries diff --git a/yawn/conftest.py b/yawn/conftest.py index b593230..e8e60c4 100644 --- a/yawn/conftest.py +++ b/yawn/conftest.py @@ -71,8 +71,8 @@ def run(): name = WorkflowName.objects.create(name='workflow1') workflow = name.new_version(parameters={'parent': True, 'child': False}) - task1 = Template.objects.create(workflow=workflow, name='task1', command=['']) - task2 = Template.objects.create(workflow=workflow, name='task2', command=['']) + task1 = Template.objects.create(workflow=workflow, name='task1', command='') + task2 = Template.objects.create(workflow=workflow, name='task2', command='') task2.upstream.add(task1) return workflow.submit_run(parameters={'child': True}) diff --git a/yawn/management/commands/exec.py b/yawn/management/commands/exec.py index 56d71ff..e817509 100644 --- a/yawn/management/commands/exec.py +++ b/yawn/management/commands/exec.py @@ -15,13 +15,17 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('module', help='The python module to import, i.e. animal.bird') parser.add_argument('callable', help='The python callable to invoke, i.e. Swallow') - parser.add_argument('argument', nargs='+', help='Arguments to pass to the callable') + parser.add_argument('argument', nargs='*', help='Arguments to pass to the callable') def handle(self, *args, **options): self.stdout.write('Importing module %s' % options['module']) module_ = importlib.import_module(options['module']) - self.stdout.write('Calling %s("%s")' % (options['callable'], '", "'.join(options['argument']))) + arguments = '' + if options['argument']: + arguments = "'{}'".format("', '".join(options['argument'])) + + self.stdout.write('Calling %s(%s)' % (options['callable'], arguments)) getattr(module_, options['callable'])(*options['argument']) self.stdout.write('Execution complete') diff --git a/yawn/migrations/0002_tasks_without_workflows.py b/yawn/migrations/0002_tasks_without_workflows.py new file mode 100644 index 0000000..66a2a3a --- /dev/null +++ b/yawn/migrations/0002_tasks_without_workflows.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.2 on 2017-06-29 16:44 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('yawn', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='queue', + name='name', + field=models.TextField(unique=True), + ), + migrations.AlterField( + model_name='task', + name='run', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='yawn.Run'), + ), + migrations.AlterField( + model_name='template', + name='name', + field=models.TextField(), + ), + migrations.AlterField( + model_name='template', + name='workflow', + field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, + to='yawn.Workflow'), + ), + migrations.AlterField( + model_name='workflowname', + name='name', + field=models.TextField(unique=True), + ), + ] diff --git a/yawn/task/helpers.py b/yawn/task/helpers.py new file mode 100644 index 0000000..e9b6e50 --- /dev/null +++ b/yawn/task/helpers.py @@ -0,0 +1,31 @@ +import shlex + +from yawn.task.models import Template, Task +from yawn.workflow.models import Workflow # noqa - needed to load the foreign key! +from yawn.worker.models import Queue + + +def delay(func, *args, timeout=None, max_retries=0, queue=None): + arguments = [shlex.quote(arg) for arg in args] + command = 'yawn exec {0.__module__} {0.__name__} {1}'.format( + func, ' '.join(arguments)).strip() + task_name = '{0.__module__}.{0.__name__}({1})'.format( + func, ', '.join(arguments)) + + if queue: + queue_obj, _ = Queue.objects.get_or_create(name=queue) + else: + queue_obj = Queue.get_default_queue() + + template, _ = Template.objects.get_or_create( + name=task_name, + command=command, + queue=queue_obj, + max_retries=max_retries, + timeout=timeout + ) + task = Task.objects.create( + template=template + ) + task.enqueue() + return task diff --git a/yawn/task/models.py b/yawn/task/models.py index c92ec18..058234b 100644 --- a/yawn/task/models.py +++ b/yawn/task/models.py @@ -7,9 +7,9 @@ class Template(models.Model): - workflow = models.ForeignKey('Workflow', models.PROTECT, editable=False) + workflow = models.ForeignKey('Workflow', models.PROTECT, editable=False, null=True) queue = models.ForeignKey(Queue, models.PROTECT) - name = models.SlugField(allow_unicode=True, db_index=False) + name = models.TextField() command = models.TextField() max_retries = models.IntegerField(default=0) @@ -41,7 +41,7 @@ class Task(models.Model): UPSTREAM_FAILED = 'upstream_failed' STATUS_CHOICES = [(x, x) for x in (WAITING, QUEUED, RUNNING, SUCCEEDED, FAILED, UPSTREAM_FAILED)] - run = models.ForeignKey('Run', models.PROTECT) + run = models.ForeignKey('Run', models.PROTECT, null=True) template = models.ForeignKey(Template, models.PROTECT) status = models.TextField(choices=STATUS_CHOICES, default=WAITING) @@ -177,7 +177,8 @@ def mark_finished(self, exit_code=None, lost=False): self.task.save() with transaction.atomic(): self.task.update_downstream() - self.task.run.update_status() + if self.task.run: + self.task.run.update_status() self.stop_timestamp = functions.Now() # need to be careful not to overwrite stdout/stderr diff --git a/yawn/task/serializers.py b/yawn/task/serializers.py index a870039..968246d 100644 --- a/yawn/task/serializers.py +++ b/yawn/task/serializers.py @@ -6,7 +6,7 @@ class SimpleWorkflowSerializer(serializers.ModelSerializer): - name = serializers.SlugField(source='name.name', read_only=True) + name = serializers.CharField(source='name.name', read_only=True) class Meta: model = Workflow @@ -14,7 +14,7 @@ class Meta: class TaskSerializer(serializers.ModelSerializer): - name = serializers.SlugField(source='template.name', read_only=True) + name = serializers.CharField(source='template.name', read_only=True) workflow = SimpleWorkflowSerializer(source='template.workflow', read_only=True) class Meta: diff --git a/yawn/task/tests/test_helpers.py b/yawn/task/tests/test_helpers.py new file mode 100644 index 0000000..143625d --- /dev/null +++ b/yawn/task/tests/test_helpers.py @@ -0,0 +1,35 @@ +from yawn.task.helpers import delay + + +def some_function(*args): + pass + + +class SomeClass: + pass + + +def test_quoted_arg(): + task = delay(SomeClass, 'A small "taste" of chaos') + assert task.template.command == 'yawn exec yawn.task.tests.test_helpers ' \ + 'SomeClass \'A small "taste" of chaos\'' + + +def test_many_args(): + task = delay(some_function, 'something', '1') + assert task.template.command == "yawn exec yawn.task.tests.test_helpers " \ + "some_function something 1" + + +def test_deduplication(): + task1 = delay(some_function) + task2 = delay(some_function) + assert task1.template_id == task2.template_id + + +def test_queued(): + task = delay(SomeClass, queue='queue', max_retries=2, timeout=10) + assert task.message_set.count() == 1 + assert task.message_set.first().queue.name == 'queue' + assert task.template.max_retries == 2 + assert task.template.timeout == 10 diff --git a/yawn/task/tests/test_models.py b/yawn/task/tests/test_models.py index 7d76e64..9e33b63 100644 --- a/yawn/task/tests/test_models.py +++ b/yawn/task/tests/test_models.py @@ -14,7 +14,7 @@ A succeeded but B failed, mark C and D upstream_failed """ from yawn.worker.models import Queue, Worker -from yawn.task.models import Task, Execution +from yawn.task.models import Task, Execution, Template def test_first_queued(run): @@ -91,3 +91,15 @@ def test_execution_output(run): execution.refresh_from_db() assert execution.stdout == 'foobar' assert execution.stderr == 'blah' + + +def test_task_without_workflow(): + template = Template.objects.create(name='task1', command='') + task = Task.objects.create(template=template) + worker = Worker.objects.create(name='worker1') + + # asserts that a task can be run without having a workflow associated + execution = task.start_execution(worker) + execution.mark_finished(exit_code=0) + task.refresh_from_db() + assert task.status == Task.SUCCEEDED diff --git a/yawn/worker/main.py b/yawn/worker/main.py index e2e4244..f10ba54 100644 --- a/yawn/worker/main.py +++ b/yawn/worker/main.py @@ -135,7 +135,7 @@ def start_tasks(self): self.executor.start_subprocess( execution_id=execution.id, command=execution.task.template.command, - environment=execution.task.run.parameters, + environment=execution.task.run.parameters if execution.task.run else {}, timeout=execution.task.template.timeout ) diff --git a/yawn/worker/models.py b/yawn/worker/models.py index 49962fc..d5edf4b 100644 --- a/yawn/worker/models.py +++ b/yawn/worker/models.py @@ -51,7 +51,7 @@ def __str__(self): class Queue(models.Model): """Arbitrary tag defining where tasks run.""" - name = models.SlugField(unique=True, allow_unicode=True) + name = models.TextField(unique=True) _default = None diff --git a/yawn/worker/serializers.py b/yawn/worker/serializers.py index d2eb64a..e72b445 100644 --- a/yawn/worker/serializers.py +++ b/yawn/worker/serializers.py @@ -25,7 +25,7 @@ def update(self, instance, validated_data): class MessageSerializer(serializers.ModelSerializer): - queue = serializers.SlugField(source='queue.name') + queue = serializers.CharField(source='queue.name') class Meta: model = Message diff --git a/yawn/worker/views.py b/yawn/worker/views.py index f25d539..9423ff4 100644 --- a/yawn/worker/views.py +++ b/yawn/worker/views.py @@ -12,7 +12,7 @@ class WorkerViewSet(viewsets.GenericViewSet, """ Worker endpoint, GET(list) """ - queryset = Worker.objects.all().order_by('id') + queryset = Worker.objects.all().order_by('-id') serializer_class = WorkerSerializer diff --git a/yawn/workflow/models.py b/yawn/workflow/models.py index 162f752..79afddf 100644 --- a/yawn/workflow/models.py +++ b/yawn/workflow/models.py @@ -6,7 +6,7 @@ class WorkflowName(models.Model): - name = models.SlugField(allow_unicode=True, unique=True) + name = models.TextField(unique=True) current_version = models.OneToOneField('Workflow', null=True, related_name='is_current') def new_version(self, **kwargs): diff --git a/yawn/workflow/serializers.py b/yawn/workflow/serializers.py index 1529751..e97a97a 100644 --- a/yawn/workflow/serializers.py +++ b/yawn/workflow/serializers.py @@ -1,7 +1,6 @@ import re from operator import itemgetter -from django.core.validators import slug_unicode_re from django.db import transaction from django.db.models import NOT_PROVIDED from rest_framework import serializers @@ -33,7 +32,7 @@ def get_versions(self, obj): class TemplateSerializer(serializers.ModelSerializer): queue = serializers.SlugRelatedField(slug_field='name', queryset=Queue.objects.all()) upstream = serializers.ListField( - child=serializers.SlugField(), default=[], source='upstream.all') + child=serializers.CharField(), default=[], source='upstream.all') class Meta: model = Template @@ -41,7 +40,7 @@ class Meta: class WorkflowSerializer(serializers.ModelSerializer): - name = serializers.RegexField(source='name.name', regex=slug_unicode_re) + name = serializers.CharField(source='name.name') name_id = serializers.IntegerField(read_only=True) tasks = TemplateSerializer(many=True, allow_empty=False, source='template_set') diff --git a/yawn/workflow/tests/test_views.py b/yawn/workflow/tests/test_views.py index 7c0de9b..cee5a13 100644 --- a/yawn/workflow/tests/test_views.py +++ b/yawn/workflow/tests/test_views.py @@ -47,12 +47,12 @@ def test_create_workflow_versions(client, data): def test_invalid_fields(client, data): - data['name'] = 'not a slug' + data['name'] = '' data['parameters'] = 'not a dict' data['tasks'][1]['upstream'].append('invalid_task') response = client.post('/api/workflows/', data) assert response.status_code == 400, response.data - assert 'required pattern' in response.data['name'][0] + assert 'This field may not be blank' in response.data['name'][0] assert 'must be a dictionary' in response.data['parameters'][0] assert 'upstream task(s) invalid_task' in response.data['tasks'][0]