From efc40586de920f90ceae62b285be72d0e4ae033f Mon Sep 17 00:00:00 2001 From: Sam Pearson Date: Wed, 13 Apr 2022 16:16:55 +0100 Subject: [PATCH] Redis: Initial support for Redis Sentinel This adds basic support for using Redis Sentinel to mediate connections to the primary Redis server used by the API functionality. Setting `REDIS_SENTINEL_HOSTS` to a dict of "'host': port" key/values will override any settings for `REDIS_DB_HOST` and `REDIS_DB_PORT` with values provided by Sentinel. Note that for the purposes of running tests, this will circumvent the patching of `api_keys.utils.redis.StrictRedis` by mockredis as calls to `redis_connection()` will use `sentinel.master_for` rather than `redis.StrictRedis`, so you'll need functioning Redis and Sentinel services in this case. Set `REDIS_SENTINEL_HOSTS` to `null` to fall-back to the existing mocked connection. Fixes #132 --- api_keys/utils.py | 28 ++++++++++++++++++++++------ conf/general.yml-example | 14 ++++++++++++++ mapit_mysociety_org/settings.py | 2 ++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/api_keys/utils.py b/api_keys/utils.py index 94b9d314..e0c4e5da 100644 --- a/api_keys/utils.py +++ b/api_keys/utils.py @@ -1,4 +1,5 @@ import redis +from redis.sentinel import Sentinel from django.conf import settings @@ -9,12 +10,27 @@ def redis_connection(): global _connection if _connection is None: - _connection = redis.StrictRedis( - host=settings.REDIS_DB_HOST, - port=settings.REDIS_DB_PORT, - db=settings.REDIS_DB_NUMBER, - password=settings.REDIS_DB_PASSWORD - ) + if settings.REDIS_SENTINEL_HOSTS is not None: + # If we have listed any sentinels, use those to manage the connection. + # REDIS_SENTINEL_HOSTS will be a dict, but this needs an array of tuples [('host', port)] + sentinel = Sentinel( + [(host, port) for host, port in settings.REDIS_SENTINEL_HOSTS.items()], + socket_timeout=0.1 + ) + _connection = sentinel.master_for( + settings.REDIS_SENTINEL_SET, + socket_timeout=0.1, + db=settings.REDIS_DB_NUMBER, + password=settings.REDIS_DB_PASSWORD + ) + else: + # Otherwise fall back to a regular connection + _connection = redis.StrictRedis( + host=settings.REDIS_DB_HOST, + port=settings.REDIS_DB_PORT, + db=settings.REDIS_DB_NUMBER, + password=settings.REDIS_DB_PASSWORD + ) return _connection diff --git a/conf/general.yml-example b/conf/general.yml-example index 2957524e..dbf1928e 100644 --- a/conf/general.yml-example +++ b/conf/general.yml-example @@ -14,10 +14,24 @@ MAPIT_DB_PORT: '5432' MAPIT_DB_RO_HOST: 'replica' MAPIT_DB_RO_PORT: '5433' +# Connection details for Redis. +# Note that REDIS_DB_HOST and REDIS_DB_PORT will be ignored +# if REDIS_SENTINEL_HOSTS is set as the connection to the +# Redis primary will be mediated by Sentinel. +# REDIS_DB_NUMBER and REDIS_DB_PASSWORD will be used in either +# case when making the connection to the primary. REDIS_DB_HOST: 'localhost' REDIS_DB_PORT: 6379 REDIS_DB_NUMBER: 0 REDIS_DB_PASSWORD: null +# REDIS_SENTINEL_HOSTS should be a dict of "'host': port" pairs. +# If you don't want to use Sentinel, set it to null. +REDIS_SENTINEL_HOSTS: + 'localhost': 26379 +# Note that this will not be used unless REDIS_SENTINEL_HOSTS is set. +# It refers to the set of Redis hosts Sentinel should return connection +# details for. +REDIS_SENTINEL_SET: data # Country is currently one of GB, NO, or KE. Optional; country specific things won't happen if not set. COUNTRY: 'GB' diff --git a/mapit_mysociety_org/settings.py b/mapit_mysociety_org/settings.py index 8239d48d..602ca556 100644 --- a/mapit_mysociety_org/settings.py +++ b/mapit_mysociety_org/settings.py @@ -117,6 +117,8 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): REDIS_DB_PORT = config.get('REDIS_DB_PORT') REDIS_DB_NUMBER = config.get('REDIS_DB_NUMBER') REDIS_DB_PASSWORD = config.get('REDIS_DB_PASSWORD') +REDIS_SENTINEL_HOSTS = config.get('REDIS_SENTINEL_HOSTS', None) +REDIS_SENTINEL_SET = config.get('REDIS_SENTINEL_SET') EMAIL_BACKEND = "mailer.backend.DbBackend" # Configurable email port, to make it easier to develop email sending