From c7f5543b91d24c370e41d504416bb8a7c1d30b09 Mon Sep 17 00:00:00 2001 From: Alec Clowes Date: Thu, 9 Feb 2017 10:17:38 -0700 Subject: [PATCH 1/2] add flake8 and editorconfig --- .editorconfig | 13 +++++++++++++ .flake8 | 2 ++ README.rst | 27 ++++++++++++--------------- circle.yml | 5 ++++- setup.py | 4 ++-- yawn/conftest.py | 21 ++++++++++++++++++--- yawn/migrations/0001_initial.py | 21 +++++++++++++++------ yawn/settings/debug.py | 2 +- yawn/settings/test.py | 2 +- yawn/task/tests/test_models.py | 18 +----------------- yawn/task/tests/test_views.py | 2 -- yawn/worker/main.py | 4 ++-- yawn/worker/tests/test_models.py | 2 +- yawn/workflow/tests/test_models.py | 1 - yawn/workflow/tests/test_views.py | 2 -- 15 files changed, 72 insertions(+), 54 deletions(-) create mode 100644 .editorconfig create mode 100644 .flake8 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e5274cc --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +[*.py] +indent_style = space +indent_size = 4 + +[*.{js,jsx,less,html,json,css,yaml,yml}] +indent_style = space +indent_size = 2 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..6deafc2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/README.rst b/README.rst index d131e47..572ae2a 100644 --- a/README.rst +++ b/README.rst @@ -33,9 +33,14 @@ Broker Components ---------- -- Web server provides a user interface. -- Worker schedules and executes tasks. -- Postgres 9.5+ database stores state. +Web Servers + The website provides a user interface to view the workflows and tasks running within them. + It allows you to run an existing workflow or re-run a failed task. The web server also provides + a REST API to programatically create and run workflows. + +Workers + The worker schedules and executes tasks. The worker uses ``subprocess.Popen`` to run tasks and + capture stdout and stderr. Concepts -------- @@ -45,13 +50,13 @@ Workflow acyclic graph (DAG). Workflows can be scheduled to run on a regular basis and they are versioned so they can change over time. -Task - A shell command that specifies the upstream tasks it depends on, the number times to retry, and a - timeout. - Run A manually triggered or scheduled run of a Workflow. +Task + A shell command that specifies the upstream tasks it depends on, the number times to retry, and a + timeout. The task is given environment variables configured in the workflow and run. + Execution A single execution of a Task's command, capturing the exit code and standard output and error. @@ -186,12 +191,4 @@ Load some examples and run the worker to process them:: .. _create-react-app: https://github.com/facebookincubator/create-react-app .. _Django: https://airflow.incubator.apache.org/ -TODO ----- -- WSGI + static file server wrapped in a ``yawn webserver`` command -- Config file for database connection, etc -- Python API / wrapper for creating workflows, submitting tasks -- submit run in UI -- edit parameters on run? -- show env given to the execution? diff --git a/circle.yml b/circle.yml index 8abcb55..1535d90 100644 --- a/circle.yml +++ b/circle.yml @@ -3,10 +3,13 @@ machine: version: 3.5.2 dependencies: + cache_directories: + - frontend/node_modules override: - pip install -e .[test] - cd frontend && npm install test: override: - - pytest + - pytest yawn + - flake8 yawn diff --git a/setup.py b/setup.py index 342e559..61b9e9a 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ """ # Always prefer setuptools over distutils -from setuptools import setup, find_packages +from setuptools import setup # To use a consistent encoding from codecs import open from os import path @@ -113,4 +113,4 @@ 'yawn=yawn.manage:main', ], }, -) \ No newline at end of file +) diff --git a/yawn/conftest.py b/yawn/conftest.py index d494542..b34f221 100644 --- a/yawn/conftest.py +++ b/yawn/conftest.py @@ -57,6 +57,21 @@ def client(): from rest_framework.test import APIClient user = User.objects.create_user('test_user') - client = APIClient() - client.force_authenticate(user) - return client + api_client = APIClient() + api_client.force_authenticate(user) + return api_client + + +@pytest.fixture() +def run(): + """Setup a workflow and run to test with""" + from yawn.workflow.models import WorkflowName + from yawn.task.models import Template + + name = WorkflowName.objects.create(name='workflow1') + workflow = name.new_version(parameters={'parent': True, 'child': False}) + task1 = Template.objects.create(workflow=workflow, name='task1', command=['']) + task2 = Template.objects.create(workflow=workflow, name='task2', command=['']) + task2.upstream.add(task1) + + return workflow.submit_run(parameters={'child': True}) diff --git a/yawn/migrations/0001_initial.py b/yawn/migrations/0001_initial.py index e9149ef..dc00edd 100644 --- a/yawn/migrations/0001_initial.py +++ b/yawn/migrations/0001_initial.py @@ -10,7 +10,6 @@ class Migration(migrations.Migration): - initial = True dependencies = [ @@ -21,7 +20,9 @@ class Migration(migrations.Migration): name='Execution', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.TextField(choices=[('running', 'running'), ('succeeded', 'succeeded'), ('failed', 'failed'), ('killed', 'killed'), ('lost', 'lost')], default='running')), + ('status', models.TextField( + choices=[('running', 'running'), ('succeeded', 'succeeded'), ('failed', 'failed'), + ('killed', 'killed'), ('lost', 'lost')], default='running')), ('start_timestamp', models.DateTimeField(default=django.db.models.functions.base.Now)), ('stop_timestamp', models.DateTimeField(null=True)), ('exit_code', models.IntegerField(null=True)), @@ -48,7 +49,9 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('submitted_time', models.DateTimeField()), ('scheduled_time', models.DateTimeField(null=True)), - ('status', models.TextField(choices=[('running', 'running'), ('succeeded', 'succeeded'), ('failed', 'failed')], default='running')), + ('status', + models.TextField(choices=[('running', 'running'), ('succeeded', 'succeeded'), ('failed', 'failed')], + default='running')), ('parameters', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), ], ), @@ -56,7 +59,10 @@ class Migration(migrations.Migration): name='Task', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.TextField(choices=[('waiting', 'waiting'), ('queued', 'queued'), ('running', 'running'), ('succeeded', 'succeeded'), ('failed', 'failed'), ('upstream_failed', 'upstream_failed')], default='waiting')), + ('status', models.TextField( + choices=[('waiting', 'waiting'), ('queued', 'queued'), ('running', 'running'), + ('succeeded', 'succeeded'), ('failed', 'failed'), ('upstream_failed', 'upstream_failed')], + default='waiting')), ('run', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='yawn.Run')), ], ), @@ -77,7 +83,8 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.TextField()), - ('status', models.TextField(choices=[('active', 'active'), ('exited', 'exited'), ('lost', 'lost')], default='active')), + ('status', models.TextField(choices=[('active', 'active'), ('exited', 'exited'), ('lost', 'lost')], + default='active')), ('start_timestamp', models.DateTimeField(default=django.db.models.functions.base.Now)), ('last_heartbeat', models.DateTimeField(default=django.db.models.functions.base.Now)), ], @@ -98,7 +105,9 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.SlugField(allow_unicode=True, unique=True)), - ('current_version', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='is_current', to='yawn.Workflow')), + ('current_version', + models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='is_current', + to='yawn.Workflow')), ], ), migrations.AddField( diff --git a/yawn/settings/debug.py b/yawn/settings/debug.py index 6a87f23..c827f1c 100644 --- a/yawn/settings/debug.py +++ b/yawn/settings/debug.py @@ -1,4 +1,4 @@ -from yawn.settings.base import * +from yawn.settings.base import * # NOQA SECRET_KEY = 'example secret key, change me' DEBUG = True diff --git a/yawn/settings/test.py b/yawn/settings/test.py index ac802ef..6443b5d 100644 --- a/yawn/settings/test.py +++ b/yawn/settings/test.py @@ -1,4 +1,4 @@ -from .base import * +from .base import * # NOQA SECRET_KEY = 'secret key for tests' diff --git a/yawn/task/tests/test_models.py b/yawn/task/tests/test_models.py index cb7d915..7d76e64 100644 --- a/yawn/task/tests/test_models.py +++ b/yawn/task/tests/test_models.py @@ -13,24 +13,8 @@ A & B succeed, don't submit C(failed) A succeeded but B failed, mark C and D upstream_failed """ -import pytest - from yawn.worker.models import Queue, Worker -from yawn.task.models import Template, Task, Execution -from yawn.workflow.models import WorkflowName - - -@pytest.fixture() -def run(): - """Setup a workflow and run to test with""" - name = WorkflowName.objects.create(name='workflow1') - workflow = name.new_version(parameters={'parent': True, 'child': False}) - task1 = Template.objects.create(workflow=workflow, name='task1', command=['']) - task2 = Template.objects.create(workflow=workflow, name='task2', command=['']) - task2.upstream.add(task1) - - run = workflow.submit_run(parameters={'child': True}) - return run +from yawn.task.models import Task, Execution def test_first_queued(run): diff --git a/yawn/task/tests/test_views.py b/yawn/task/tests/test_views.py index 2552472..13004f0 100644 --- a/yawn/task/tests/test_views.py +++ b/yawn/task/tests/test_views.py @@ -1,6 +1,4 @@ from yawn.worker.models import Worker -# to make the fixture available: -from yawn.task.tests.test_models import run def test_get_task(client, run): diff --git a/yawn/worker/main.py b/yawn/worker/main.py index 65f374b..e2e4244 100644 --- a/yawn/worker/main.py +++ b/yawn/worker/main.py @@ -1,6 +1,6 @@ import enum import signal -import typing +import typing # NOQA import collections from django.db import transaction @@ -8,7 +8,7 @@ from yawn.task.models import Task, Execution from yawn.worker.models import Worker, Queue -from yawn.worker.executor import Manager, Result +from yawn.worker.executor import Manager, Result # NOQA from yawn.workflow.models import Workflow from yawn.utilities import logger from yawn.utilities.cron import Crontab diff --git a/yawn/worker/tests/test_models.py b/yawn/worker/tests/test_models.py index 60369a0..dd52dbd 100644 --- a/yawn/worker/tests/test_models.py +++ b/yawn/worker/tests/test_models.py @@ -1,4 +1,4 @@ -from yawn.task.models import Template, Task, Execution +from yawn.task.models import Template, Task, Execution # NOQA from yawn.worker.models import Worker from yawn.workflow.models import WorkflowName diff --git a/yawn/workflow/tests/test_models.py b/yawn/workflow/tests/test_models.py index 1b780be..6540ea9 100644 --- a/yawn/workflow/tests/test_models.py +++ b/yawn/workflow/tests/test_models.py @@ -96,4 +96,3 @@ def test_statuses(): task2.save() run.update_status() assert run.status == Run.SUCCEEDED - diff --git a/yawn/workflow/tests/test_views.py b/yawn/workflow/tests/test_views.py index 5e68d79..7c0de9b 100644 --- a/yawn/workflow/tests/test_views.py +++ b/yawn/workflow/tests/test_views.py @@ -6,8 +6,6 @@ from django.utils.dateparse import parse_datetime from yawn.workflow.models import WorkflowName -# to make the fixture available: -from yawn.task.tests.test_models import run @pytest.fixture() From 6554ae55e9ace2b7d8f9934521867ec6adbda659 Mon Sep 17 00:00:00 2001 From: Alec Clowes Date: Thu, 9 Feb 2017 10:33:14 -0700 Subject: [PATCH 2/2] add codecov --- .gitignore | 2 +- circle.yml | 10 ++++++---- setup.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 2dbb74d..929e4a5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ # build *.egg-info/ *build/ -*coverage/ +.coverage # node packages *node_modules/ diff --git a/circle.yml b/circle.yml index 1535d90..ec9aff9 100644 --- a/circle.yml +++ b/circle.yml @@ -6,10 +6,12 @@ dependencies: cache_directories: - frontend/node_modules override: - - pip install -e .[test] - - cd frontend && npm install + - pip install -e .[test] + - cd frontend && npm install test: override: - - pytest yawn - - flake8 yawn + - pytest --cov=yawn yawn + - flake8 yawn + post: + - bash <(curl -s https://codecov.io/bash) diff --git a/setup.py b/setup.py index 61b9e9a..ee1ed19 100755 --- a/setup.py +++ b/setup.py @@ -85,8 +85,8 @@ extras_require={ # 'dev': ['check-manifest'], 'test': [ - 'coverage', 'pytest', + 'pytest-cov', 'flake8', 'pyyaml', ],