diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d0b6ce3..ab934b52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,6 +122,10 @@ py34-django110: <<: *tests_template py34-django111: <<: *tests_template +py35-django111: + <<: *tests_template + image: onegreyonewhite/tox:ubuntu + pep8_checks: stage: code_standarts diff --git a/.gitlab/issue_templates/Ask.md b/.gitlab/issue_templates/Ask.md new file mode 100644 index 00000000..d706060f --- /dev/null +++ b/.gitlab/issue_templates/Ask.md @@ -0,0 +1,10 @@ +#ASK + +Replace all comments to answers. +Before adding a new issue check the tracker for similar issues. + +#### What's the core of the question? +what's the core of the (your) question about...... + +#### Details +Files, code, etc.... diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 00000000..33eef698 --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,25 @@ +#BUG + +Replace all comments to answers. +Before adding a new issue check the tracker for similar issues. + +#### What's the core of the bug? +what's the core of the bug in...... + +#### Exception output +...if exists. + +#### How to reproduce? +How bug we could reproduce? + +#### Polemarch version +`sudo -u polemarch /opt/polemarch/bin/polemarchctl webserver --version` + +#### Database type +Default, Mysql, etc... + +#### RPC type +Default, RabbitMQ, etc... + +#### Additional info +Other info that could help. diff --git a/.gitlab/issue_templates/Feature request.md b/.gitlab/issue_templates/Feature request.md new file mode 100644 index 00000000..072a3d45 --- /dev/null +++ b/.gitlab/issue_templates/Feature request.md @@ -0,0 +1,13 @@ +# Feature request + +Replace all comments to answers. +Before adding a new issue check the tracker for similar issues. + +#### What feature do you suggest? +Description of features. + +#### How do you imagine it? +Describe the changes in functionality after adding this feature + +#### Why do you think that this feature should be implemented in the application? +Describe shortly how this feature will be useful for users diff --git a/Makefile b/Makefile index e5e4091e..1ed02b62 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ all: compile docs: -rm -rf doc/_build mkdir -p doc/_static - $(PY) setup.py build_sphinx --build-dir doc/_build + $(PY) setup.py build_sphinx --build-dir doc/_build -W test: tox -e $(ENVS) $(TESTS) diff --git a/README.rst b/README.rst index fe9f4a9e..657f48ad 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,11 @@ Polemarch **Polemarch** is service for orchestration infrastructure by ansible. Simply WEB gui for orchestration infrastructure by ansible playbooks. -Official site: https://gitlab.com/vstconsulting/polemarch +Official site: +https://gitlab.com/vstconsulting/polemarch + +For any questions you could use issues tracker: +https://gitlab.com/vstconsulting/polemarch/issues .. image:: https://raw.githubusercontent.com/vstconsulting/polemarch/master/doc/screencast.gif :alt: interface of Polemarch @@ -33,7 +37,7 @@ Red Hat/CentOS installation .. sourcecode:: bash - sudo yum localinstall polemarch-0.0.2-0.x86_64.rpm. + sudo yum localinstall polemarch-0.0.X-0.x86_64.rpm. 3. Run services with commands @@ -62,7 +66,7 @@ Ubuntu/Debian installation .. sourcecode:: bash - sudo dpkg -i polemarch_0.0.2-1_amd64.deb || sudo apt-get install -f + sudo dpkg -i polemarch_0.0.X-1_amd64.deb || sudo apt-get install -f 3. Run services with commands diff --git a/deb.mk b/deb.mk index 475be88a..bfb20b4c 100644 --- a/deb.mk +++ b/deb.mk @@ -91,7 +91,7 @@ export DEBIAN_RULES define DEBIAN_PREINST #!/bin/bash # making sure user created -id -u $(USER) &>/dev/null || useradd -M $(USER) +id -u $(USER) &>/dev/null || useradd -m $(USER) id -g $(USER) &>/dev/null || groupadd $(USER) endef export DEBIAN_PREINST diff --git a/doc/config.rst b/doc/config.rst index eb1f478b..b5f9c0c5 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -85,13 +85,13 @@ example than general howto) you must do such steps: location = 127.0.0.1:11211 4. Setup some network filesystem. NFS for example. Mount it in same directory - to all worker-intended nodes. Write that directory in :ref:`worker`. + to all worker-intended nodes. Write that directory in :ref:`main`. Example: .. sourcecode:: ini - [worker] - exchange_dir = /mnt/mynfs + [main] + projects_dir = /mnt/mynfs 5. Setup some http-balancer. HAProxy for example. Point it to web-intended nodes. @@ -129,6 +129,8 @@ example than general howto) you must do such steps: That's it. +.. _main: + Main settings ------------- @@ -211,15 +213,3 @@ Section ``[web]``. Here placed settings related to web-server. It is settings like: allowed hosts, static files directory or pagination limit. - -.. _worker: - -Worker settings ---------------- - -Section ``[worker]``. - -Section for worker-related settings. Now here just one - directory to store -files, which must be accessible by all workers. It have meaning only if you -have cluster - more than one workers. In such case you must use some kind of -network filesystem to share data between workers. Like NFS, Samba or something. \ No newline at end of file diff --git a/doc/quickstart.rst b/doc/quickstart.rst index effcc2a5..e60be169 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -12,7 +12,7 @@ Red Hat/CentOS installation .. sourcecode:: bash - sudo yum localinstall polemarch-0.0.2-0.x86_64.rpm. + sudo yum localinstall polemarch-0.0.X-0.x86_64.rpm. 3. Run services with commands @@ -41,7 +41,7 @@ Ubuntu/Debian installation .. sourcecode:: bash - sudo dpkg -i polemarch_0.0.2-1_amd64.deb || sudo apt-get install -f + sudo dpkg -i polemarch_0.0.X-1_amd64.deb || sudo apt-get install -f 3. Run services with commands diff --git a/doc/restapi.rst b/doc/restapi.rst index 3784ade5..06061450 100644 --- a/doc/restapi.rst +++ b/doc/restapi.rst @@ -1683,7 +1683,9 @@ History records "stop_time":"2017-07-02T13:48:11.922777Z", "raw_inventory":"inventory", "raw_args": "ansible-playbook main.yml -i /tmp/tmpvMIwMg -v", - "raw_stdout":"text" + "raw_stdout":"text", + "initiator": 1, + "initiator_type": "users" } :>json number id: id of history record. @@ -1704,6 +1706,8 @@ History records :>json string raw_stdout: what Ansible wrote to stdout and stderr during execution. The size is limited to 10M characters. Full output in :http:get:`/api/v1/history/{id}/raw/`. + :>json number initiator: initiator id. + :>json string initiator_type: initiator type like in api url. :>json string url: url to this specific history record. .. |history_details_ref| replace:: **Response JSON Object:** response json fields @@ -1940,6 +1944,97 @@ History records :statuscode 424: facts still not ready because module is currently running or only scheduled for run. +Ansible +------- + +.. http:get:: /api/v1/ansible/ + + Get list of available methods in that category. All methods under + `/ansible/` designed to provide information about ansible installation which + Polemarch is currently using. + + Example request: + + .. sourcecode:: http + + GET /api/v1/ansible/ HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + Results: + + .. sourcecode:: js + + { + "cli-reference": "http://localhost:8000/api/v1/ansible/cli_reference/", + "modules": "http://localhost:8000/api/v1/ansible/modules/" + } + +.. http:get:: /api/v1/ansible/cli_reference/ + + Get list of available ansible command line tools arguments with their type + and hint. + + :query filter: filter by tool, for which you want get help (either `ansible` + or `ansible-playbook`). + + Example request: + + .. sourcecode:: http + + GET /api/v1/ansible/cli_reference/?filter=ansible HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + Results: + + .. sourcecode:: js + + { + "ansible": { + "extra-vars": { + "type": "text", + "help": "set additional variables as key=value or YAML/JSON" + }, + "help": { + "type": "boolean", + "help": "show this help message and exit" + }, + // there is much more arguments to type it here + // ... + } + } + +.. http:get:: /api/v1/ansible/modules/ + + Get list of installed ansible modules. + + :query filter: filter to search by module name. It is Python regular + expression. + + Example request: + + .. sourcecode:: http + + GET /api/v1/ansible/modules/?filter=\.git HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + Results: + + .. sourcecode:: js + + [ + "extras.source_control.git_config", + "extras.source_control.github_release", + "extras.source_control.github_hooks", + "extras.source_control.gitlab_user", + "extras.source_control.github_key", + "extras.source_control.gitlab_group", + "extras.source_control.gitlab_project", + "core.source_control.git" + ] + .. _variables: Variables diff --git a/polemarch/__init__.py b/polemarch/__init__.py index 5aba90a6..3fe48478 100644 --- a/polemarch/__init__.py +++ b/polemarch/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.0.5" +__version__ = "0.0.6" def _main(settings="polemarch.main.settings"): # pylint: disable=unused-variable diff --git a/polemarch/api/base.py b/polemarch/api/base.py index 41aacfd8..7bae5416 100644 --- a/polemarch/api/base.py +++ b/polemarch/api/base.py @@ -4,6 +4,7 @@ from django.db.models import Q from django.db.models.query import QuerySet from rest_framework import viewsets, views as rest_views +from rest_framework.reverse import reverse from rest_framework.response import Response as RestResponse from rest_framework.decorators import detail_route, list_route @@ -48,12 +49,14 @@ def _base_get_queryset(self): queryset = queryset.all() return queryset + def get_user_aval_projects(self): + return self.request.user.related_objects.values_list('projects', + flat=True) + def _get_extra_queryset(self): - aval_projs = self.request.user.related_objects.values_list('projects', - flat=True) return self.queryset.filter( Q(related_objects__user=self.request.user) | - Q(related_objects__projects__in=aval_projs) + Q(related_objects__projects__in=self.get_user_aval_projects()) ).distinct() def get_queryset(self): @@ -109,7 +112,7 @@ def get_paginated_route_response(self, queryset, serializer_class=None, def permissions(self, request, pk=None): # pylint: disable=unused-argument serializer = self.get_serializer(self.get_object()) - return serializer.permissions(request) + return serializer.permissions(request).resp @list_route(methods=["post"]) def filter(self, request): @@ -139,3 +142,33 @@ class HistoryModelViewSet(GenericViewSet, class ModelViewSetSet(GenericViewSet, viewsets.ModelViewSet): pass + + +class NonModelsViewSet(GenericViewSet): + base_name = None + + def get_queryset(self): + return QuerySet() + + +class ListNonModelViewSet(NonModelsViewSet, + viewsets.mixins.ListModelMixin): + + @property + def methods(self): + this_class_dict = ListNonModelViewSet.__dict__ + obj_class_dict = self.__class__.__dict__ + new_methods = list() + for name, attr in obj_class_dict.items(): + detail = getattr(attr, 'detail', True) + if name not in this_class_dict and not detail: + new_methods.append(name.replace('_', "-")) + return new_methods + + def list(self, request, *args, **kwargs): + routes = { + method: reverse("{}-{}".format(self.base_name, method), + request=request) + for method in self.methods + } + return Response(routes, 200).resp diff --git a/polemarch/api/routers.py b/polemarch/api/routers.py index 2776338d..015c12e6 100644 --- a/polemarch/api/routers.py +++ b/polemarch/api/routers.py @@ -16,6 +16,9 @@ def __init__(self, *args, **kwargs): super(_AbstractRouter, self).__init__(*args, **kwargs) def get_default_base_name(self, viewset): + base_name = getattr(viewset, 'base_name', None) + if base_name is not None: + return base_name queryset = getattr(viewset, 'queryset', None) model = getattr(viewset, 'model', None) if queryset is None: diff --git a/polemarch/api/urls.py b/polemarch/api/urls.py index c98dfd32..1598a76e 100644 --- a/polemarch/api/urls.py +++ b/polemarch/api/urls.py @@ -19,6 +19,7 @@ routerv1.register(r'periodic-tasks', v1.PeriodicTaskViewSet) routerv1.register(r'templates', v1.TemplateViewSet) routerv1.register(r'history', v1.HistoryViewSet) +routerv1.register(r'ansible', v1.AnsibleViewSet) routerv1.register_view(r'token', v1.TokenView) routerv1.register_view(r'_bulk', v1.BulkViewSet) diff --git a/polemarch/api/v1/filters.py b/polemarch/api/v1/filters.py index 09788bce..304e3baa 100644 --- a/polemarch/api/v1/filters.py +++ b/polemarch/api/v1/filters.py @@ -137,7 +137,9 @@ class Meta: 'status', 'inventory', 'start_time', - 'stop_time') + 'stop_time', + 'initiator', + 'initiator_type') class PeriodicTaskFilter(_BaseFilter): diff --git a/polemarch/api/v1/serializers.py b/polemarch/api/v1/serializers.py index 06bfb1b8..7d447f28 100644 --- a/polemarch/api/v1/serializers.py +++ b/polemarch/api/v1/serializers.py @@ -9,9 +9,10 @@ from rest_framework import serializers from rest_framework import exceptions -from rest_framework.response import Response +# from rest_framework.response import Response from ...main import models +from ..base import Response # Serializers field for usability @@ -155,6 +156,8 @@ class Meta: "inventory", "start_time", "stop_time", + "initiator", + "initiator_type", "url") @@ -172,6 +175,8 @@ class Meta: "raw_inventory", "raw_args", "raw_stdout", + "initiator", + "initiator_type", "url") def get_facts(self, request): @@ -212,10 +217,10 @@ def _get_objects(self, model, objs_id): ) return list(qs.filter(id__in=objs_id)) - def get_operation(self, request, attr): + def get_operation(self, method, data, attr): tp = getattr(self.instance, attr) - obj_list = self._get_objects(tp.model, request.data) - return self._operate(request, attr, obj_list) + obj_list = self._get_objects(tp.model, data) + return self._operate(method, data, attr, obj_list) def _response(self, total, found, code=200): data = dict(total=len(total)) @@ -223,6 +228,7 @@ def _response(self, total, found, code=200): data["not_found"] = data["total"] - data["operated"] return Response(data, status=code) + @transaction.atomic def _do_with_vars(self, method_name, *args, **kwargs): method = getattr(super(_WithVariablesSerializer, self), method_name) instance = method(*args, **kwargs) @@ -234,8 +240,8 @@ def _do_with_vars(self, method_name, *args, **kwargs): return instance @transaction.atomic() - def _operate(self, request, attr, obj_list): - action = self.operations[request.method] + def _operate(self, method, data, attr, obj_list): + action = self.operations[method] tp = getattr(self.instance, attr) if action == "all": if attr == "related_objects": @@ -248,7 +254,7 @@ def _operate(self, request, attr, obj_list): getattr(tp, "clear")() action = "add" getattr(tp, action)(*obj_list) - return self._response(request.data, obj_list) + return self._response(data, obj_list) def create(self, validated_data): return self._do_with_vars("create", validated_data=validated_data) @@ -261,7 +267,8 @@ def update(self, instance, validated_data): def permissions(self, request): pms = models.TypesPermissions.objects.filter(user__id__in=request.data) - return self._operate(request, "related_objects", pms) + return self._operate(request.method, request.data, + "related_objects", pms) class HostSerializer(_WithVariablesSerializer): @@ -353,11 +360,11 @@ class Meta: class _InventoryOperations(_WithVariablesSerializer): - def hosts_operations(self, request): - return self.get_operation(request, attr="hosts") + def hosts_operations(self, method, data): + return self.get_operation(method, data, attr="hosts") - def groups_operations(self, request): - return self.get_operation(request, attr="groups") + def groups_operations(self, method, data): + return self.get_operation(method, data, attr="groups") ################################### @@ -392,15 +399,15 @@ class Meta: class ValidationException(exceptions.ValidationError): status_code = 409 - def hosts_operations(self, request): + def hosts_operations(self, method, data): if self.instance.children: raise self.ValidationException("Group is children.") - return super(OneGroupSerializer, self).hosts_operations(request) + return super(OneGroupSerializer, self).hosts_operations(method, data) - def groups_operations(self, request): + def groups_operations(self, method, data): if not self.instance.children: raise self.ValidationException("Group is not children.") - return super(OneGroupSerializer, self).groups_operations(request) + return super(OneGroupSerializer, self).groups_operations(method, data) class InventorySerializer(_WithVariablesSerializer): @@ -467,8 +474,8 @@ class Meta: 'vars', 'url',) - def inventories_operations(self, request): - return self.get_operation(request, attr="inventories") + def inventories_operations(self, method, data): + return self.get_operation(method, data, attr="inventories") @transaction.atomic() def sync(self): @@ -481,7 +488,9 @@ def _execution(self, kind, request): inventory_id = int(data.pop("inventory")) target = str(data.pop(kind)) action = getattr(self.instance, "execute_ansible_{}".format(kind)) - history_id = action(target, inventory_id, **data) + history_id = action( + target, inventory_id, initiator=request.user.id, **data + ) rdata = dict(detail="Started at inventory {}.".format(inventory_id), history_id=history_id) return Response(rdata, 201) diff --git a/polemarch/api/v1/views.py b/polemarch/api/v1/views.py index 899ce01e..ff35bfeb 100644 --- a/polemarch/api/v1/views.py +++ b/polemarch/api/v1/views.py @@ -1,11 +1,12 @@ # pylint: disable=unused-argument,protected-access,too-many-ancestors from django.db import transaction +from django.db.models import Q from django.http import HttpResponse from rest_framework import exceptions as excepts, views as rest_views from rest_framework.authtoken import views as token_views from rest_framework.decorators import detail_route, list_route -from ...main.utils import CmdExecutor, KVExchanger +from ...main import utils from .. import base from ..permissions import SuperUserPermission, StaffPermission from . import filters @@ -64,15 +65,18 @@ class HostViewSet(base.ModelViewSetSet): class _GroupedViewSet(object): # pylint: disable=no-member + def _get_result(self, request, operation): + return operation(request.method, request.data).resp + @detail_route(methods=["post", "put", "delete", "get"]) def hosts(self, request, *args, **kwargs): serializer = self.get_serializer(self.get_object()) - return serializer.hosts_operations(request) + return self._get_result(request, serializer.hosts_operations) @detail_route(methods=["post", "put", "delete", "get"]) def groups(self, request, *args, **kwargs): serializer = self.get_serializer(self.get_object()) - return serializer.groups_operations(request) + return self._get_result(request, serializer.groups_operations) class GroupViewSet(base.ModelViewSetSet, _GroupedViewSet): @@ -102,19 +106,21 @@ def supported_repos(self, request): @detail_route(methods=["post", "put", "delete", "get"]) def inventories(self, request, *args, **kwargs): serializer = self.get_serializer(self.get_object()) - return serializer.inventories_operations(request) + return self._get_result(request, serializer.inventories_operations) @detail_route(methods=["post"]) def sync(self, request, *args, **kwargs): - return self.get_serializer(self.get_object()).sync() + return self.get_serializer(self.get_object()).sync().resp @detail_route(methods=["post"], url_path="execute-playbook") def execute_playbook(self, request, *args, **kwargs): - return self.get_serializer(self.get_object()).execute_playbook(request) + serializer = self.get_serializer(self.get_object()) + return serializer.execute_playbook(request).resp @detail_route(methods=["post"], url_path="execute-module") def execute_module(self, request, *args, **kwargs): - return self.get_serializer(self.get_object()).execute_module(request) + serializer = self.get_serializer(self.get_object()) + return serializer.execute_module(request).resp class TaskViewSet(base.ReadOnlyModelViewSet): @@ -137,6 +143,12 @@ class HistoryViewSet(base.HistoryModelViewSet): serializer_class_one = serializers.OneHistorySerializer filter_class = filters.HistoryFilter + def _get_extra_queryset(self): + return self.queryset.filter( + Q(initiator=self.request.user.id, initiator_type="users") | + Q(project__in=self.get_user_aval_projects()) + ).distinct() + @detail_route(methods=["get"]) def raw(self, request, *args, **kwargs): obj = self.get_object() @@ -153,7 +165,8 @@ def lines(self, request, *args, **kwargs): @detail_route(methods=["post"]) def cancel(self, request, *args, **kwargs): obj = self.get_object() - KVExchanger(CmdExecutor.CANCEL_PREFIX + str(obj.id)).send(True, 10) + exch = utils.KVExchanger(utils.CmdExecutor.CANCEL_PREFIX + str(obj.id)) + exch.send(True, 10) return base.Response("Task canceled: {}".format(obj.id), 200).resp @detail_route(methods=["get"]) @@ -179,7 +192,8 @@ class BulkViewSet(rest_views.APIView): _op_types = { "add": "perform_create", "set": "perform_update", - "del": "perform_delete" + "del": "perform_delete", + "mod": "perform_modify" } _allowed_types = [ 'host', 'group', 'inventory', 'project', 'periodictask', 'template' @@ -219,13 +233,21 @@ def perform_delete(self, item, pk): instance.delete() return base.Response("Ok", 200).resp_dict + def perform_modify(self, item, pk, data, method, data_type): + serializer = self.get_serializer(self.get_object(item, pk), item=item) + operation = getattr(serializer, "{}_operations".format(data_type)) + return operation(method, data).resp_dict + @transaction.atomic def post(self, request, *args, **kwargs): operations = request.data results = [] for operation in operations: - perf_method = getattr(self, self._op_types[operation.pop("type")]) - results.append(perf_method(**operation)) + op_type = operation.pop("type") + perf_method = getattr(self, self._op_types[op_type]) + result = perf_method(**operation) + result['type'] = op_type + results.append(result) return base.Response(results, 200).resp def get(self, request): @@ -234,3 +256,21 @@ def get(self, request): "operations_types": self._op_types.keys(), } return base.Response(response, 200).resp + + +class AnsibleViewSet(base.ListNonModelViewSet): + base_name = "ansible" + + @list_route(methods=["get"]) + def cli_reference(self, request): + reference = utils.AnsibleArgumentsReference() + return base.Response(reference.as_gui_dict( + request.query_params.get("filter", "") + ), 200).resp + + @list_route(methods=["get"]) + def modules(self, request): + _mods = utils.AnsibleModules().get( + request.query_params.get("filter", "") + ) + return base.Response(_mods, 200).resp diff --git a/polemarch/main/context_processors.py b/polemarch/main/context_processors.py index 55c570bc..083790db 100644 --- a/polemarch/main/context_processors.py +++ b/polemarch/main/context_processors.py @@ -3,11 +3,17 @@ def settings_constants(request): - host_url = request.build_absolute_uri('/') + # pylint: disable=unused-argument data = {"login_url": getattr(settings, 'LOGIN_URL', '/login/'), "logout_url": getattr(settings, 'LOGOUT_URL', '/logout/'), "docs_url": getattr(settings, 'DOC_URL', '/docs/'), - "debug": getattr(settings, 'DEBUG', False), - "host_url": host_url, - "polemarch_version": polemarch_version} + "debug": getattr(settings, 'DEBUG', False)} return data + + +def project_args(request): + host_url = request.build_absolute_uri('/') + return { + "host_url": host_url, + "polemarch_version": polemarch_version + } diff --git a/polemarch/main/management/base.py b/polemarch/main/management/base.py index f93b2639..23434356 100644 --- a/polemarch/main/management/base.py +++ b/polemarch/main/management/base.py @@ -7,6 +7,7 @@ CommandError as CommandErrorBase) from django.conf import settings import django +import celery from ... import __version__ from ..utils import exception_with_traceback @@ -42,9 +43,8 @@ def handle(self, *args, **options): self.LOG_LEVEL = LOG_LEVEL.upper() def get_version(self): - return u'IHService {c}, Django {d.__version__}'.format( - c=__version__, d=django, - ) + vstr = u'Polemarch {c}, Django {d.__version__}, Celery {r.__version__}' + return vstr.format(c=__version__, d=django, r=celery) def _print(self, info=""): self.stdout.write(str(info)) # pragma: no cover diff --git a/polemarch/main/migrations/0015_auto_20170814_1731.py b/polemarch/main/migrations/0015_auto_20170814_1731.py new file mode 100644 index 00000000..1096e664 --- /dev/null +++ b/polemarch/main/migrations/0015_auto_20170814_1731.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.2 on 2017-08-14 07:31 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0014_auto_20170727_0618'), + ] + + operations = [ + migrations.AddField( + model_name='history', + name='initiator', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='history', + name='initiator_type', + field=models.CharField(default='users', max_length=50), + ), + migrations.AlterIndexTogether( + name='history', + index_together=set([('id', 'project', 'mode', 'status', 'inventory', 'start_time', 'stop_time', 'initiator', 'initiator_type')]), + ), + ] diff --git a/polemarch/main/migrations/0016_typespermissions_template.py b/polemarch/main/migrations/0016_typespermissions_template.py new file mode 100644 index 00000000..01da5a92 --- /dev/null +++ b/polemarch/main/migrations/0016_typespermissions_template.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-08-25 00:16 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0015_auto_20170814_1731'), + ] + + operations = [ + migrations.AddField( + model_name='typespermissions', + name='template', + field=models.ManyToManyField(blank=True, null=True, related_name='related_objects', related_query_name='related_objects', to='main.Template'), + ), + ] diff --git a/polemarch/main/models/__init__.py b/polemarch/main/models/__init__.py index 4b9d68ab..df80b3fa 100644 --- a/polemarch/main/models/__init__.py +++ b/polemarch/main/models/__init__.py @@ -14,14 +14,23 @@ from .projects import Project from .users import TypesPermissions from .tasks import Task, PeriodicTask, History, HistoryLines, Template -from ..validators import validate_hostname +from ..validators import validate_hostname, RegexValidator from ..exceptions import UnknownTypeException -from ..utils import raise_context +from ..utils import raise_context, AnsibleArgumentsReference ##################################### # SIGNALS ##################################### +@receiver(signals.pre_save, sender=Group) +def validate_group_name(instance, **kwargs): + validate_name = RegexValidator( + regex=r'^[a-zA-Z0-9\-\._]*$', + message='Name must be Alphanumeric' + ) + validate_name(instance.name) + + @receiver(signals.m2m_changed, sender=Group.parents.through) def check_circular_deps(instance, action, pk_set, *args, **kw): if action in ["pre_add", "post_add"]: @@ -54,11 +63,16 @@ def validate_crontab(instance, **kwargs): @receiver(signals.pre_save, sender=Host) def validate_hosts(instance, **kwargs): - if instance.type == "HOST" and \ - instance.variables.filter(key="ansible_host").count(): - validate_hostname(instance.name) - elif instance.variables.filter(key="ansible_host").count(): + if instance.variables.filter(key="ansible_host").count(): validate_hostname(instance.variables.get("ansible_host")) + elif instance.type == "HOST": + validate_hostname(instance.name) + elif instance.type == "RANGE": + validate_name = RegexValidator( + regex=r'^[a-zA-Z0-9\-\._\[\]\:]*$', + message='Name must be Alphanumeric' + ) + validate_name(instance.name) @receiver(signals.pre_save, sender=Host) @@ -79,6 +93,13 @@ def validate_template(instance, **kwargs): ) if errors: raise ValidationError(errors) + command = "playbook" + ansible_args = dict(instance.data['vars']) + if instance.kind == "Module": + command = "module" + if instance.kind == "PeriodicTask" and instance.data["kind"] == "MODULE": + command = "module" + AnsibleArgumentsReference().validate_args(command, ansible_args) @receiver(signals.pre_delete, sender=Project) diff --git a/polemarch/main/models/projects.py b/polemarch/main/models/projects.py index 2646d621..5f2fbc1e 100644 --- a/polemarch/main/models/projects.py +++ b/polemarch/main/models/projects.py @@ -6,6 +6,7 @@ from django.conf import settings from django.utils import timezone +from .. import utils from . import hosts as hosts_models from .vars import AbstractModel, AbstractVarsQuerySet, BManager, models from ..exceptions import PMException @@ -67,7 +68,11 @@ def _prepare_kw(self, kind, mod_name, inventory_id, **extra): inventory=inventory, project=self, kind=kind, - raw_stdout="") + raw_stdout="", + initiator=extra.pop("initiator", 0)) + command = kind.lower() + ansible_args = dict(extra) + utils.AnsibleArgumentsReference().validate_args(command, ansible_args) history = History.objects.create(status="DELAY", **history_kwargs) kwargs = dict(target=mod_name, inventory=inventory, history=history) kwargs.update(extra) @@ -75,6 +80,7 @@ def _prepare_kw(self, kind, mod_name, inventory_id, **extra): def _execute(self, kind, task_class, *args, **extra): sync = extra.pop("sync", False) + kwargs = self._prepare_kw(kind, *args, **extra) history = kwargs['history'] if sync: diff --git a/polemarch/main/models/tasks.py b/polemarch/main/models/tasks.py index 6109f303..5cf82c44 100644 --- a/polemarch/main/models/tasks.py +++ b/polemarch/main/models/tasks.py @@ -13,7 +13,9 @@ from django.db import transaction from django.db.models import Q from django.utils import timezone +from django.contrib.auth.models import User +from ..utils import AnsibleArgumentsReference from . import Inventory from ..exceptions import DataNotReady, NotApplicable from .base import BModel, BManager, BQuerySet, models @@ -81,6 +83,17 @@ def get_vars(self): qs = self.variables.order_by("key") return OrderedDict(qs.values_list('key', 'value')) + @transaction.atomic() + def set_vars(self, variables): + command = "playbook" + ansible_args = {} + for key, value in variables.items(): + ansible_args[key] = value + if self.kind == "MODULE": + command = "module" + AnsibleArgumentsReference().validate_args(command, ansible_args) + return super(PeriodicTask, self).set_vars(variables) + def get_schedule(self): if self.type == "CRONTAB": return crontab(**self.crontab_kwargs) @@ -150,22 +163,25 @@ def create(self, **kwargs): class History(BModel): - objects = HistoryQuerySet.as_manager() - project = models.ForeignKey(Project, - on_delete=models.CASCADE, - related_query_name="history", - null=True) - inventory = models.ForeignKey(Inventory, - on_delete=models.CASCADE, - related_query_name="history", - blank=True, null=True, default=None) - mode = models.CharField(max_length=256) - kind = models.CharField(max_length=50, default="PLAYBOOK") - start_time = models.DateTimeField(default=timezone.now) - stop_time = models.DateTimeField(blank=True, null=True) - raw_args = models.TextField(default="") - raw_inventory = models.TextField(default="") - status = models.CharField(max_length=50) + objects = HistoryQuerySet.as_manager() + project = models.ForeignKey(Project, + on_delete=models.CASCADE, + related_query_name="history", + null=True) + inventory = models.ForeignKey(Inventory, + on_delete=models.CASCADE, + related_query_name="history", + blank=True, null=True, default=None) + mode = models.CharField(max_length=256) + kind = models.CharField(max_length=50, default="PLAYBOOK") + start_time = models.DateTimeField(default=timezone.now) + stop_time = models.DateTimeField(blank=True, null=True) + raw_args = models.TextField(default="") + raw_inventory = models.TextField(default="") + status = models.CharField(max_length=50) + initiator = models.IntegerField(default=0) + # Initiator type should be always as in urls for api + initiator_type = models.CharField(max_length=50, default="users") class NoFactsAvailableException(NotApplicable): def __init__(self): @@ -177,9 +193,13 @@ class Meta: ordering = ["-id"] index_together = [ ["id", "project", "mode", "status", "inventory", - "start_time", "stop_time"] + "start_time", "stop_time", "initiator", "initiator_type"] ] + @property + def initiator_object(self): + return User.objects.get(id=self.initiator) + @property def facts(self): if self.status not in ['OK', 'ERROR', 'OFFLINE']: diff --git a/polemarch/main/models/users.py b/polemarch/main/models/users.py index 9a0de5fa..70231900 100644 --- a/polemarch/main/models/users.py +++ b/polemarch/main/models/users.py @@ -7,7 +7,7 @@ from .base import models, BModel from .projects import Project -from .tasks import Task, PeriodicTask, History +from .tasks import Task, PeriodicTask, History, Template from . import hosts as hosts_models logger = logging.getLogger("polemarch") @@ -38,6 +38,9 @@ class TypesPermissions(BModel): history = models.ManyToManyField(History, related_query_name=def_rel_name, blank=True, null=True) + template = models.ManyToManyField(Template, + related_query_name=def_rel_name, + blank=True, null=True) class Meta: default_related_name = "related_objects" diff --git a/polemarch/main/models/utils.py b/polemarch/main/models/utils.py index f0f3a822..958e7102 100644 --- a/polemarch/main/models/utils.py +++ b/polemarch/main/models/utils.py @@ -9,6 +9,7 @@ KVExchanger, CalledProcessError) +PolemarchInventory = namedtuple("PolemarchInventory", "raw keys") AnsibleExtra = namedtuple('AnsibleExtraArgs', [ 'args', 'files', @@ -45,74 +46,96 @@ def write_output(self, line): line_number=self.counter, line=line) + def execute(self, cmd, cwd): + self.history.raw_args = " ".join(cmd) + return super(Executor, self).execute(cmd, cwd) + class AnsibleCommand(object): command_type = None + status_codes = { + 4: "OFFLINE", + -9: "INTERRUPTED", + "other": "ERROR" + } + + class Inventory(PolemarchInventory): + @property + def file(self): + self.__file = getattr(self, "__file", tmp_file(self.raw)) + return self.__file + + def close(self): + for key_file in self.keys: + key_file.close() + self.__file.close() + def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs def __parse_extra_args(self, **extra): extra_args, files = list(), list() + extra.pop("verbose", None) for key, value in extra.items(): - if key in ["extra_vars", "extra-vars"]: - key = "extra-vars" - elif key == "verbose": - continue - elif key in ["key_file", "key-file"]: + if key == "key-file": if "BEGIN RSA PRIVATE KEY" in value: - kfile = tmp_file() - kfile.write(value) + kfile = tmp_file(value) files.append(kfile) value = kfile.name else: value = "{}/{}".format(self.workdir, value) - key = "key-file" extra_args.append("--{}".format(key)) extra_args += [str(value)] if value else [] return AnsibleExtra(extra_args, files) def get_workdir(self): - return "/tmp" + return self.history.project.path @property def workdir(self): return self.get_workdir() + @property + def path_to_ansible(self): + return dirname(sys.executable) + "/" + self.command_type + + def prepare(self, target, inventory, history): + self.target, self.history = target, history + self.inventory_object = self.Inventory(*inventory.get_inventory()) + self.history.raw_inventory = self.inventory_object.raw + self.history.status = "RUN" + self.history.save() + self.executor = Executor(self.history) + + def get_args(self, target, extra_args): + return [self.path_to_ansible, target, + '-i', self.inventory_object.file.name, '-v'] + extra_args + + def error_handler(self, exception): + default_code = self.status_codes["other"] + if isinstance(exception, CalledProcessError): + self.history.raw_stdout = str(exception.output) + self.history.status = self.status_codes.get(exception.returncode, + default_code) + else: + self.history.raw_stdout = self.history.raw_stdout + str(exception) + self.history.status = default_code + def execute(self, target, inventory, history, **extra_args): - self.project = history.project - history.raw_inventory, key_files = inventory.get_inventory() - history.status = "RUN" - history.save() - path_to_ansible = dirname(sys.executable) + "/" + self.command_type - inventory_file = tmp_file() - inventory_file.write(history.raw_inventory) - status = "OK" + self.prepare(target, inventory, history) + self.history.status = "OK" try: extra = self.__parse_extra_args(**extra_args) - args = [path_to_ansible, target, '-i', - inventory_file.name, '-v'] + extra.args - history.raw_args = " ".join(args) - history.raw_stdout = Executor(history).execute(args, self.workdir) - except CalledProcessError as exception: - history.raw_stdout = str(exception.output) - if exception.returncode == 4: - status = "OFFLINE" - elif exception.returncode == -9: - status = "INTERRUPTED" - else: - status = "ERROR" - except Exception as exception: # pragma: no cover - history.raw_stdout = history.raw_stdout + str(exception) - status = "ERROR" + args = self.get_args(self.target, extra.args) + self.history.raw_stdout = self.executor.execute(args, self.workdir) + except Exception as exception: + self.error_handler(exception) finally: - inventory_file.close() - for key_file in key_files: - key_file.close() - history.stop_time = timezone.now() - history.status = status - history.save() + self.inventory_object.close() + self.history.stop_time = timezone.now() + self.history.save() def run(self): return self.execute(*self.args, **self.kwargs) @@ -121,9 +144,6 @@ def run(self): class AnsiblePlaybook(AnsibleCommand): command_type = "ansible-playbook" - def get_workdir(self): - return self.project.path - class AnsibleModule(AnsibleCommand): command_type = "ansible" @@ -134,8 +154,5 @@ def __init__(self, target, *pargs, **kwargs): kwargs.pop('args', None) super(AnsibleModule, self).__init__(*pargs, **kwargs) - def get_workdir(self): - return self.project.path - def execute(self, group, *args, **extra_args): return super(AnsibleModule, self).execute(group, *args, **extra_args) diff --git a/polemarch/main/settings.ini b/polemarch/main/settings.ini index 398fdb58..d29f3dda 100644 --- a/polemarch/main/settings.ini +++ b/polemarch/main/settings.ini @@ -22,6 +22,11 @@ # engine = django.db.backends.sqlite3 # name = {HOME}/db.sqlite3 +[database.options] +# Database options settings +############################################################## +# timeout = 10 + [cache] # Cache settings. # Read more: https://docs.djangoproject.com/en/1.10/ref/settings/#caches @@ -36,7 +41,7 @@ # !!! STRONGLY RECOMMENDED TO USE MEMCACHED OR REDIS BACKENDS !!! ############################################################## # backend = django.core.cache.backends.filebased.FileBasedCache -# location = {TMP}/ihservice_django_cache_locks{PY} +# location = {TMP}/polemarch_django_cache_locks{PY} [rpc] # Celery broker settings @@ -68,11 +73,3 @@ # How many results return by API. ############################################################## # rest_page_limit = 1000 - -[worker] -# Celery worker settings -############################################################## - -# Directory to store any cooperative data for workers -############################################################## -# exchange_dir = /tmp \ No newline at end of file diff --git a/polemarch/main/settings.py b/polemarch/main/settings.py index 2dfa116c..e258c418 100644 --- a/polemarch/main/settings.py +++ b/polemarch/main/settings.py @@ -17,6 +17,8 @@ from . import __file__ as file +APACHE = False if ("runserver" in sys.argv) else True + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(file))) PY_VER = sys.version_info[0] @@ -33,7 +35,15 @@ # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! +# To set key create file named `secret` near setting.ini file +# or set in POLEMARCH_SECRET_FILE env. +SECRET_FILE = os.getenv("POLEMARCH_SETTINGS_FILE", "/etc/polemarch/secret") SECRET_KEY = '*sg17)9wa_e+4$n%7n7r_(kqwlsc^^xdoc3&px$hs)sbz(-ml1' +try: + with open(SECRET_FILE, "r") as secret_file: + SECRET_KEY = secret_file.read() +except IOError: + pass # SECURITY WARNING: don't run with debug turned on in production! DEBUG = config.getboolean("main", "debug", fallback=False) @@ -105,6 +115,7 @@ 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'polemarch.main.context_processors.settings_constants', + 'polemarch.main.context_processors.project_args', ], }, }, @@ -124,6 +135,21 @@ import pymysql pymysql.install_as_MySQLdb() +try: + __DB_OPTIONS = { } + for k, v in config.items('database.options'): + if k in ["CONN_MAX_AGE", "timeout"]: + __DB_OPTIONS[k] = float(v) + continue + __DB_OPTIONS[k] = v.format(**__kwargs) + if not __DB_OPTIONS: raise NoSectionError('database.options') +except NoSectionError: + __DB_OPTIONS = { + "timeout": 10 + } + +__DB_SETTINGS["OPTIONS"] = __DB_OPTIONS + DATABASES = { 'default': __DB_SETTINGS } @@ -164,7 +190,6 @@ ), 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', - # 'rest_framework.renderers.AdminRenderer', 'rest_framework.renderers.BrowsableAPIRenderer', ), "DEFAULT_PERMISSION_CLASSES": ( @@ -207,7 +232,8 @@ 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ) -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +if APACHE: + STATIC_ROOT = os.path.join(BASE_DIR, 'static') # Documentation files # http://django-docs.readthedocs.io/en/latest/#docs-access-optional @@ -328,7 +354,6 @@ } } -APACHE = False if ("webserver" in sys.argv) or ("runserver" in sys.argv) else True if "test" in sys.argv: CELERY_TASK_ALWAYS_EAGER = True diff --git a/polemarch/main/templates/base.html b/polemarch/main/templates/base.html index 85e1a12b..e4ce8410 100644 --- a/polemarch/main/templates/base.html +++ b/polemarch/main/templates/base.html @@ -88,6 +88,7 @@ + @@ -103,7 +104,8 @@ - + + @@ -181,6 +183,26 @@