From 9b7359879f48a276f2236b3c9d6ac38706b5dd83 Mon Sep 17 00:00:00 2001 From: Marc Date: Tue, 26 Apr 2016 01:08:13 +0200 Subject: [PATCH] Add Django support. * Move from django-kaneda project. * Add integration tests. * Improve Django documentation --- CHANGELOG | 3 +- django_kaneda/__init__.py | 40 ++++++++++++++++++ django_kaneda/settings.py | 35 ++++++++++++++++ docs/django.rst | 44 +++++++++++++++----- docs/index.rst | 2 +- docs/queues.rst | 2 +- docs/settings.rst | 2 +- docs/usage.rst | 4 +- tests/__init__.py | 1 - tests/integration/django/__init__.py | 0 tests/integration/django/conftest.py | 30 ++++++++++++++ tests/integration/django/test_django.py | 29 +++++++++++++ tests/unit/conftest.py | 54 ++----------------------- tests/unit/test_utils.py | 37 ++++++++--------- 14 files changed, 196 insertions(+), 87 deletions(-) create mode 100644 django_kaneda/__init__.py create mode 100644 django_kaneda/settings.py create mode 100644 tests/integration/django/__init__.py create mode 100644 tests/integration/django/conftest.py create mode 100644 tests/integration/django/test_django.py diff --git a/CHANGELOG b/CHANGELOG index a2e11df..02be782 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,10 +7,11 @@ Changelog 0.3 (2016-03-28) ~~~~~~~~~~~~~~~~ -* Add LoggerBackend. Useful for debugging +* Add LoggerBackend. * Add exeception treatment on backend report methods. * Add connection timeout for storage backends. * Storage backends now support passing client or connection url as parameters instead all parameters. +* Add support to use django-kaneda on debug mode. 0.2 (2016-02-17) ~~~~~~~~~~~~~~~~ diff --git a/django_kaneda/__init__.py b/django_kaneda/__init__.py new file mode 100644 index 0000000..8f98b4b --- /dev/null +++ b/django_kaneda/__init__.py @@ -0,0 +1,40 @@ +import logging + +from django.utils.functional import LazyObject + + +class LazyMetrics(LazyObject): + + def _setup(self): + from kaneda import Metrics + from kaneda.utils import import_class, get_object_from_settings + from kaneda.exceptions import UnexistingKanedaClass, SettingsError + from . import settings + + if settings.DEBUG: + backend_class = import_class('kaneda.backends.LoggerBackend') + if settings.LOGGER: + backend = backend_class(logger=logging.getLogger(settings.LOGGER)) + elif settings.LOGGER_FILENAME: + backend = backend_class(filename=settings.LOGGER_FILENAME) + else: + backend = backend_class() + _metrics = Metrics(backend=backend) + else: + if not settings.BACKEND and not settings.QUEUE: + raise SettingsError('You need to set KANEDA_BACKEND or KANEDA_QUEUE on settings.py to django_kaneda') + if settings.BACKEND: + try: + backend = get_object_from_settings(settings.BACKEND, settings) + _metrics = Metrics(backend=backend) + except UnexistingKanedaClass: + raise UnexistingKanedaClass('The selected KANEDA_BACKEND class does not exists.') + if settings.QUEUE: + try: + queue = get_object_from_settings(settings.QUEUE, settings) + _metrics = Metrics(queue=queue) + except UnexistingKanedaClass: + raise UnexistingKanedaClass('The selected KANEDA_QUEUE class does not exists.') + self._wrapped = _metrics + +metrics = LazyMetrics() diff --git a/django_kaneda/settings.py b/django_kaneda/settings.py new file mode 100644 index 0000000..6349894 --- /dev/null +++ b/django_kaneda/settings.py @@ -0,0 +1,35 @@ +from django.conf import settings + +BACKEND = getattr(settings, 'KANEDA_BACKEND', None) +QUEUE = getattr(settings, 'KANEDA_QUEUE', None) + +# Elasticsearch backend settings +ELASTIC_INDEX_NAME = getattr(settings, 'KANEDA_ELASTIC_INDEX_NAME', 'kaneda') +ELASTIC_APP_NAME = getattr(settings, 'KANEDA_ELASTIC_APP_NAME', 'default') +ELASTIC_CONNECTION_URL = getattr(settings, 'KANEDA_ELASTIC_CONNECTION_URL', None) +ELASTIC_HOST = getattr(settings, 'KANEDA_ELASTIC_HOST', None) +ELASTIC_PORT = getattr(settings, 'KANEDA_ELASTIC_PORT', None) +ELASTIC_USER = getattr(settings, 'KANEDA_ELASTIC_USER', None) +ELASTIC_PASSWORD = getattr(settings, 'KANEDA_ELASTIC_PASSWORD', None) +ELASTIC_TIMEOUT = getattr(settings, 'KANEDA_ELASTIC_TIMEOUT', 0.3) + +# MongoDB backend settings +MONGO_DB_NAME = getattr(settings, 'KANEDA_MONGO_DB_NAME', 'kaneda') +MONGO_COLLECTION_NAME = getattr(settings, 'KANEDA_MONGO_COLLECTION_NAME', 'default') +MONGO_CONNECTION_URL = getattr(settings, 'KANEDA_MONGO_CONNECTION_URL', None) +MONGO_HOST = getattr(settings, 'KANEDA_MONGO_HOST', None) +MONGO_PORT = getattr(settings, 'KANEDA_MONGO_PORT', None) +MONGO_TIMEOUT = getattr(settings, 'KANEDA_MONGO_TIMEOUT', 300) + +# Debug backend mode settings +DEBUG = getattr(settings, 'KANEDA_DEBUG', False) +LOGGER = getattr(settings, 'KANEDA_LOGGER', None) +LOGGER_FILENAME = getattr(settings, 'KANEDA_LOGGER_FILENAME', None) + +# Celery queue settings +CELERY_BROKER = getattr(settings, 'KANEDA_CELERY_BROKER', '') +CELERY_QUEUE_NAME = getattr(settings, 'KANEDA_CELERY_QUEUE_NAME', '') + +# RQ queue settings +RQ_REDIS_URL = getattr(settings, 'KANEDA_RQ_REDIS_URL', 'kaneda') +RQ_QUEUE_NAME = getattr(settings, 'KANEDA_RQ_QUEUE_NAME', None) diff --git a/docs/django.rst b/docs/django.rst index f911aee..8f87ba5 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -5,14 +5,9 @@ Django Setup Kaneda can be use with Django as a mechanism to reporting metrics and events. -1. Install `kaneda` and `django-kaneda` packages:: +1. Add :code:`django_kaneda` to :code:`INSTALLED_APPS` in :file:`settings.py`. - pip install git+https://gitlab.apsl.net/apsl/kaneda.git - pip install git+https://gitlab.apsl.net/apsl/django-kaneda.git - -2. Add :code:`django_kaneda` to :code:`INSTALLED_APPS` in :file:`settings.py`. - -3. Set :code:`KANEDA_BACKEND` and the properly configuration of your selected backend in :file:`settings.py`. If you want to use Elasticsearch our configuration will be something like this:: +2. Set :code:`KANEDA_BACKEND` and the properly configuration of your selected backend in :file:`settings.py`. If you want to use Elasticsearch our configuration will be something like this:: KANEDA_BACKEND = 'kaneda.backends.ElasticsearchBackend' KANEDA_ELASTIC_INDEX_NAME = 'kaneda' @@ -22,6 +17,11 @@ Kaneda can be use with Django as a mechanism to reporting metrics and events. KANEDA_ELASTIC_USER = 'user' KANEDA_ELASTIC_PASSWORD = 'pass' +Alternatively you can set :code:`KANEDA_QUEUE` to specify a :ref:`queue ` configuration to use Kaneda in :ref:`async mode `:: + + KANEDA_BACKEND = 'kaneda.queues.CeleryQueue' + KANEDA_CELERY_BROKER = 'redis://localhost:6379/0' + With this, you can use Kaneda in everyplace of your Django project:: from django_kaneda import metrics @@ -82,10 +82,13 @@ KANEDA_ELASTIC_INDEX_NAME (='kaneda') KANEDA_ELASTIC_APP_NAME (='default') Name of the app/project where metrics are used. -KANEDA_ELASTIC_HOST (='localhost') +KANEDA_ELASTIC_CONNECTION_URL (=None) + Elasticsearch connection url (https://user:secret@localhost:9200). + +KANEDA_ELASTIC_HOST (=None) Server host. -KANEDA_ELASTIC_PORT (=9200) +KANEDA_ELASTIC_PORT (=None) Server port. KANEDA_ELASTIC_USER (=None) @@ -105,15 +108,34 @@ KANEDA_MONGO_DB_NAME (='kaneda') KANEDA_MONGO_COLLECTION_NAME (='default') Name of the MongoDB collection used to store metric data. -KANEDA_MONGO_HOST (='localhost') +KANEDA_MONGO_CONNECTION_URL (=None) + Mongo connection url (mongodb://localhost:27017/). + +KANEDA_MONGO_HOST (=None) Server host. -KANEDA_MONGO_PORT (=27017) +KANEDA_MONGO_PORT (=None) Server port. KANEDA_MONGO_TIMEOUT (=300) MongoDB connection timeout (milliseconds). +Celery +------ +KANEDA_CELERY_BROKER (='') + Broker connection url. + +KANEDA_CELERY_QUEUE_NAME (='') + Name of the Celery queue. + +RQ +-- +KANEDA_RQ_REDIS_URL (='') + Redis connection url. + +KANEDA_RQ_QUEUE_NAME (='kaneda') + Name of the RQ queue. + Debug ----- KANEDA_DEBUG (=True) diff --git a/docs/index.rst b/docs/index.rst index 243298d..c4bcc3a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,10 +16,10 @@ Example:: :hidden: usage - django metrics backends queues settings + django changelog diff --git a/docs/queues.rst b/docs/queues.rst index e59403d..b222d57 100644 --- a/docs/queues.rst +++ b/docs/queues.rst @@ -3,7 +3,7 @@ Queues ====== -Kaneda provides builtin queues to store metrics and events to perform :ref:`async`. If you want to use your +Kaneda provides builtin queues to store metrics and events to perform :ref:`asynchronous reporting `. If you want to use your custom asynchronous queue system you need to subclass :code:`BaseQueue` and implement your custom :code:`report` method which is the responsible to pass metrics data to a job queue. diff --git a/docs/settings.rst b/docs/settings.rst index c17ca2e..fa798f4 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -1,7 +1,7 @@ Settings ======== -Kaneda can be used with a settings file as a similar way to use with :ref:`Django `. Simply define a +Kaneda can be used with a settings file as the same way to use with :ref:`Django `. Simply define a :file:`kanedasettings.py` file with the backend or queue settings. Alternatively you can define the environment variable `DEFAULT_SETTINGS_ENVAR` pointing to the desired settings filename. diff --git a/docs/usage.rst b/docs/usage.rst index f673ca5..88d3a9c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -36,7 +36,7 @@ Asynchronous reporting Depending the selection of the backend the process of reporting metrics could be "slow" if the response time of your application is critical (e.g: a website). Furthermore if your application doesn't need the see the reported metrics in real time you probably have to consider to using asynchronous reporting. With this system you are allowed to send a -metric report in background without affecting your performance. +metric report in background without adding too much overhead. To use this system you need to install a queue system and use one of the builtin Kaneda :ref:`queues` classes. To setup Kaneda in async mode follow these steps. @@ -79,5 +79,5 @@ As in the backend example it can be used passing a queue client:: q = Queue(queue_name, connection=Redis()) queue = RQQueue(queue=q) -Notice that you are able to specify a Redis connection url (or a broker url if you use :ref:`Celery`). Notice this allows you +Also you are able to specify a Redis connection url (or a broker url if you use :ref:`Celery`). Notice this allows you to run the worker on a different server. \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 7c68785..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/tests/integration/django/__init__.py b/tests/integration/django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/django/conftest.py b/tests/integration/django/conftest.py new file mode 100644 index 0000000..7088947 --- /dev/null +++ b/tests/integration/django/conftest.py @@ -0,0 +1,30 @@ +import pytest + + +@pytest.fixture +def django_settings_backend(elastic_settings): + elastic_settings.DEBUG = False + elastic_settings.QUEUE = None + return elastic_settings + + +@pytest.fixture +def django_settings_debug(): + class Settings: + DEBUG = True + LOGGER = None + LOGGER_FILENAME = None + + return Settings + + +@pytest.fixture +def django_settings_queue(celery_settings): + celery_settings.DEBUG = False + celery_settings.BACKEND = None + return celery_settings + + +def pytest_configure(): + from django.conf import settings + settings.configure() diff --git a/tests/integration/django/test_django.py b/tests/integration/django/test_django.py new file mode 100644 index 0000000..1cb6e3d --- /dev/null +++ b/tests/integration/django/test_django.py @@ -0,0 +1,29 @@ +from kaneda.backends import LoggerBackend, ElasticsearchBackend +from kaneda.queues import CeleryQueue +from django_kaneda import settings + + +class TestDjango(object): + + def test_django_kaneda_with_backend(self, mocker, django_settings_backend): + mocker.patch('django_kaneda.settings', django_settings_backend) + from django_kaneda import LazyMetrics + metrics = LazyMetrics() + assert isinstance(metrics.backend, ElasticsearchBackend) + result = metrics.gauge('test_gauge', 42) + assert result + + def test_django_kaneda_with_debug(self, mocker, django_settings_debug): + mocker.patch('django_kaneda.settings', django_settings_debug) + from django_kaneda import LazyMetrics + metrics = LazyMetrics() + metrics.gauge('test_gauge', 42) + assert isinstance(metrics.backend, LoggerBackend) + + def test_django_kaneda_with_queue(self, mocker, django_settings_queue): + mocker.patch('django_kaneda.settings', django_settings_queue) + from django_kaneda import LazyMetrics + metrics = LazyMetrics() + assert isinstance(metrics.queue, CeleryQueue) + result = metrics.gauge('test_gauge', 42) + assert result diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2c00e59..ab705f1 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,60 +12,12 @@ def report(self, name, metric, value, tags, id_=None): self.reported_data[name] = payload -@pytest.fixture(scope='module') +@pytest.fixture def dummy_backend(): return DummyBackend() -@pytest.fixture(scope='module') -def elasticsearch_backend_settings(): - class Settings: - BACKEND = 'kaneda.backends.ElasticsearchBackend' - ELASTIC_INDEX_NAME = 'test' - ELASTIC_APP_NAME = 'test' - ELASTIC_HOST = 'localhost' - ELASTIC_PORT = 9200 - ELASTIC_USER = 'test' - ELASTIC_PASSWORD = 'test' - ELASTIC_TIMEOUT = 0.3 - - return Settings - - -@pytest.fixture(scope='module') -def mongo_backend_settings(): - class Settings: - BACKEND = 'kaneda.backends.MongoBackend' - MONGO_DB_NAME = 'test' - MONGO_COLLECTION_NAME = 'test' - MONGO_HOST = 'localhost' - MONGO_PORT = 27017 - MONGO_TIMEOUT = 300 - - return Settings - - -@pytest.fixture(scope='module') -def rq_queue_settings(): - class Settings: - QUEUE = 'kaneda.queues.RQQueue' - RQ_REDIS_URL = 'redis://localhost:6379/1' - RQ_QUEUE_NAME = '' - - return Settings - - -@pytest.fixture(scope='module') -def celery_queue_settings(): - class Settings: - QUEUE = 'kaneda.queues.CeleryQueue' - CELERY_BROKER = 'redis://localhost:6379/1' - CELERY_QUEUE_NAME = '' - - return Settings - - -@pytest.fixture(scope='module') +@pytest.fixture def empty_settings(): class Settings: pass @@ -73,7 +25,7 @@ class Settings: return Settings -@pytest.fixture(scope='module') +@pytest.fixture def unexisting_backend_settings(): class Settings: BACKEND = 'kaneda.backends.UnexsitingBackend' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6a70aea..274e408 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5,8 +5,7 @@ from kaneda.utils import import_class, get_object_from_settings, get_kaneda_objects, get_backend from kaneda.backends import ElasticsearchBackend, MongoBackend, LoggerBackend -from .conftest import elasticsearch_backend_settings, mongo_backend_settings, rq_queue_settings, celery_queue_settings, \ - empty_settings +from .conftest import empty_settings class TestUtils(object): @@ -19,13 +18,6 @@ class TestUtils(object): def test_import_backend_class(self, backend_path_module, backend_class): assert import_class(backend_path_module) is backend_class - @pytest.mark.parametrize('backend_settings, backend_class', [ - (elasticsearch_backend_settings(), ElasticsearchBackend), - (mongo_backend_settings(), MongoBackend), - ]) - def test_get_backend_from_settings(self, backend_settings, backend_class): - assert isinstance(get_object_from_settings(backend_settings.BACKEND, backend_settings), backend_class) - @pytest.mark.parametrize('queue_path_module, queue_class', [ ('kaneda.queues.RQQueue', RQQueue), ('kaneda.queues.CeleryQueue', CeleryQueue) @@ -33,24 +25,33 @@ def test_get_backend_from_settings(self, backend_settings, backend_class): def test_import_queue_class(self, queue_path_module, queue_class): assert import_class(queue_path_module) is queue_class - @pytest.mark.parametrize('queue_settings, queue_class', [ - (rq_queue_settings(), RQQueue), - (celery_queue_settings(), CeleryQueue), + @pytest.mark.parametrize('backend_name, backend_class', [ + ('elastic', ElasticsearchBackend), + ('mongo', MongoBackend), + ]) + def test_get_backend_from_settings(self, kaneda_settings, backend_name, backend_class): + backend_settings = getattr(kaneda_settings, backend_name) + assert isinstance(get_object_from_settings(backend_settings.BACKEND, backend_settings), backend_class) + + @pytest.mark.parametrize('queue_name, queue_class', [ + ('rq', RQQueue), + ('celery', CeleryQueue), ]) - def test_get_queue_from_settings(self, queue_settings, queue_class): + def test_get_queue_from_settings(self, kaneda_settings, queue_name, queue_class): + queue_settings = getattr(kaneda_settings, queue_name) assert isinstance(get_object_from_settings(queue_settings.QUEUE, queue_settings), queue_class) def test_get_object_from_settings_with_error(self, unexisting_backend_settings): with pytest.raises(UnexistingKanedaClass): get_object_from_settings(unexisting_backend_settings.BACKEND, unexisting_backend_settings) - @pytest.mark.parametrize('settings, has_backend, has_queue', [ - (elasticsearch_backend_settings(), True, False), - (rq_queue_settings(), False, True), + @pytest.mark.parametrize('config_object, has_backend, has_queue', [ + ('elastic', True, False), + ('rq', False, True), ]) - def test_get_kaneda_objects(self, mocker, settings, has_backend, has_queue): + def test_get_kaneda_objects(self, mocker, kaneda_settings, config_object, has_backend, has_queue): mock_get_settings = mocker.patch('kaneda.utils.get_settings') - mock_get_settings.return_value = settings + mock_get_settings.return_value = getattr(kaneda_settings, config_object) backend, queue = get_kaneda_objects() assert bool(backend) == has_backend assert bool(queue) == has_queue