From c508354ea3afc6676e30cee012d0fcc79790ac76 Mon Sep 17 00:00:00 2001 From: Attila Kovacs Date: Sun, 5 Jan 2025 14:49:24 +0100 Subject: [PATCH] README: add cluster docs --- README.md | 122 ++++++++++++++++++++++++++++++++++++++++++- src/redisx-cluster.c | 30 ++++++++--- 2 files changed, 145 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4235f59..6402a4e 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Last Updated: 31 December 2024 - [Publish / subscribe (PUB/SUB) support](#publish-subscribe-support) - [Atomic execution blocks and LUA scripts](#atomic-transaction-blocks-and-lua-scripts) - [Advanced queries and pipelining](#advanced-queries) + - [Redis clusters](#cluster-support) - [Error handling](#error-handling) - [Debug support](#debug-support) - [Future plans](#future-plans) @@ -150,7 +151,7 @@ And at every step, you should check for and [handle errors](#error-handling) as | pipelined (batch) processing | __yes__ | dedicated (high-bandwidth) client / user-defined callback | | PUB/SUB support | __yes__ | dedicated client / user callbacks / subscription management | | Sentinel support | __yes__ | _help me test it_ | - | cluster support | no | _coming soon..._ | + | cluster support | __yes__ | _help me test it_ | | TLS support | no | _coming soon..._ | @@ -205,6 +206,10 @@ prior to invoking `make`. The following build variables can be configured: - `LDFLAGS`: Extra linker flags (default is _not set_). Note, `-lm -lxchange` will be added automatically. + - `WITH_OPENMP`: If set to 1 (default), we will compile and link with OpenMP (i.e., `-fopenmp` is added to both + `CFLAGS` and `LDFLAGS` automatically). Since OpenMP is not available on all platforms / compilers, you may want + to explicitly set `WITH_OPENMP=0` prior to calling `make` to disable. + - `CHECKEXTRA`: Extra options to pass to `cppcheck` for the `make check` target - `XCHANGE`: If the [Smithsonian/xchange](https://github.com/Smithsonian/xchange) library is not installed on your @@ -1341,7 +1346,122 @@ practices to help deal with pipeline responses are summarized here: __RedisX__ optimizes the pipeline client for high throughput (bandwidth), whereas the interactive and subscription clients are optimized for low-latency, at the socket level. + +----------------------------------------------------------------------------- + + +## Redis clusters + + - [Cluster basics](#cluster-basics) + - [Explicit connection management](#cluster-explicit-connect) + - [Automatic reconfiguration](#cluster-reconfiguration) + +__RedisX__ provides support for clusters also. In cluster configuration the database is distributed over a collection +of servers, each node of which serves only a subset of the Redis keys. + + 1. Configure a known cluster node as usual. + 2. Initialize a cluster using the known, configured node. (All shards in the cluster will inherit the configuration.) + 3. For every request, you must obtain the appropriate cluster node for the given key to process. (PUB/SUB may be + processed on any cluster node.) + 4. Destroy the cluster resources when done using it. + + + +### Cluster basics + +Specifically, start by configuring a known node of the cluster as usual for a single Redis server, setting +authentication, socket configuration, callbacks etc.: + +```c + // Initialize a known node of the cluster for obtaining the current cluster configuration + Redis *node = redisxInit(...); + ... +``` + +Next, you can use the known node to obtain the cluster configuration and thus initialize a cluster configuration: + +```c + // Try obtain the cluster configuration from the known node. + RedisCluster *cluster = redisxClusterInit(node); + if(cluster == NULL) { + // Oops, that did not work. Perhaps try another node... + ... + } + // Discard the configuring node if no longer needed... + redisxDestroy(node); +``` + +The above will query the cluster configuration from the node (the node need not be explicitly connected prior to the +initialization, and will be returned in the same connection state as before). The cluster will inherit the configuration +of the node, such as pipelining, socket configuration authentication, protocol, and callbacks, from the configuring node. +If the initialization fails on a particular node, you might try other known nodes until one of then succeeds. (You +might use `redisxSetHostName()` and `redisxSetPort()` on the original node to update the address of the configuring node, +while leaving other configuration settings intact.) + +Once the cluster is configured, you may discard the configuring node instance, unless you need it specifically for other +reasons. + +You can start using the cluster right away. You can obtain a connected `Redis` instance for a given key using +`redisxClusterGetShard()`, e.g.: + +```c + const char *key = "my-key"; // The Redis keyword of interest, can use Redis hashes + + // Get the connected Redis server instance that serves the given key + Redis *shard = redisxClusterGetShard(cluster, key); + if(shard == NULL) { + // Oops, there seems to be no server for the given key currently. Perhaps try again later... + ... + } + + // Run your query on using the given Redis key / keys. + int status = X_SUCCESS; + RESP *reply = redisxRequest(shard, "GET", key, NULL, NULL, &status); +``` + +As a matter a best practice you should never assume that a given keyword is persistently served by the same shard. +I.e., you should obtain the current shard for the key each time you want to use the cluster with a particular Redis +key. + +Finally, when you are done using the cluster, simply discard it: + +```c + // Disconnect all shards and free up all resources by the cluster + redisxClusterDestroy(cluster); +``` + + +### Explicit connection management + +The `redisxClusterGetShard()` will automatically connect the associated shard, if not already connected. Thus, you do +not need to explicitly connect to the cluster. However, in some cases you might want to connect all shards before +running queries to eliminate the connection overhead during use. If so, you can call `redisxClusterConnect()` to +explicitly connect to all shards, before uisng the cluster. Similarly, you can also explicitly disconnect from all +connected shards using `redisxClusterDisconnect()`, e.g. to close unnecessary sockets. You may continue to use the +cluster after calling `redisxClusterDisconnect()`, as successive calls to `redisxClusterGetShard()` will automatically +reconnect the shards as needed. + + +### Automatic reconfiguration + +Redis cluster can be reconfigured on the fly, and the __RedisX__ library provides support for automatically detecting +cluster reconfigurations. The process is automatic, but it is not entirely transparent to users, who may need to +re-submit failed queries after reconfiguration, since the library does not know how many pending queries may have been +sent, that are/were affected by the reconfiguration. In general, the reconfiguration is handled as follows: + + 1. __RedisX__ will automatically capture the cluster reconfiguration when a request returns an error message with + the error `MOVED`. Upon receiving the telltale error, the library will immediately launch a thread to reconfigure + the cluster in the background. (The cluster will be blocked by a mutex while the reconfiguration takes place, such + that successive `redisxClusterGetShard()` call will block until the new configuration is in place.) + + 2. Requests sent to the now invalid shards of the old configuration will return an error (message type `RESP_ERROR`) + to the caller. For interactive queries this would be the just for the last request sent, but for pipelined queries + it may be some subset of the submitted and pending queries, which are affected. The caller is responsible handling + these errors, and re-submitting the failed queries as necessary, by once again calling `redisxClusterGetShard()` on + the affected keywords. + + ----------------------------------------------------------------------------- diff --git a/src/redisx-cluster.c b/src/redisx-cluster.c index 2f55de5..e2f56e7 100644 --- a/src/redisx-cluster.c +++ b/src/redisx-cluster.c @@ -136,8 +136,16 @@ static void rDiscardShardsAsync(RedisCluster *cluster) { p->n_shards = 0; } - -static RedisShard *rClusterDiscoverAsync(Redis *redis, int *n_shards, RedisCluster *cluster) { +/** + * Returns the current cluster configuration obtained from the specified node + * + * @param cluster Pointer to cluster to set as the parent to the discovered shards + * @param redis The node to use for discovery. It need not be in a connected state. + * @param[out] n_shards Pointer to integer in which to return the number of shards discovered + * or else an error code <0. + * @return Array containing the discovered shards or NULL if there was an error. + */ +static RedisShard *rClusterDiscoverAsync(const RedisCluster *cluster, Redis *redis, int *n_shards) { static const char *fn = "rClusterDiscoverAsync"; RESP *reply; @@ -220,7 +228,7 @@ void *ClusterRefreshThread(void *pCluster) { for(m = 0; m < s->n_servers; m++) { int n_shards = 0; - RedisShard *shard = rClusterDiscoverAsync(s->redis[m], &n_shards, cluster); + RedisShard *shard = rClusterDiscoverAsync(cluster, s->redis[m], &n_shards); if(n_shards >= 0) { rDiscardShardsAsync(cluster); @@ -257,15 +265,21 @@ int rClusterRefresh(RedisCluster *cluster) { p = (ClusterPrivate *) cluster->priv; if(!p) return x_error(X_NO_INIT, ENXIO, fn, "cluster is not initialized"); + // Return immediately if the cluster is being reconfigured at present. + // This is important so we may process all pending MOVED responses while + // the reconfiguration takes place. if(p->reconfiguring) return X_SUCCESS; pthread_mutex_lock(&p->mutex); + // After obtaining the exclusive lock, check again that no other thread has + // begun reconfiguration. if(p->reconfiguring) { pthread_mutex_unlock(&p->mutex); return X_SUCCESS; } + // We are now officially in charge of reconfiguring the cluster... p->reconfiguring = TRUE; errno = 0; @@ -283,10 +297,14 @@ int rClusterRefresh(RedisCluster *cluster) { * Returns the Redis server in a cluster which is to be used for queries relating to the * specified Redis keyword. In Redis cluster configurations, the database is distributed in * a way that each cluster node serves only a subset of the Redis keys. Thus, this function - * allows to identify the node that serves a given key. + * allows to identify the node that serves a given key. The function supports Redish hashes + * according to the specification. * * @param cluster Pointer to a Redis cluster configuration - * @param key The Redis keyword of interest + * @param key The Redis keyword of interest. It may use hashes (i.e., if the keyword + * contains a segment enclosed in {} brackets, then the hash will be + * calculated on the bracketed segment only. E.g. `{user:1000}.name` and + * `{user:1000}.address` will both return the same hash for `user:1000` only. * @return A connected Redis server (cluster shard), which can be used for * queries on the given keyword, or NULL if either input pointer is NULL * (errno = EINVAL), or the cluster has not been initialized (errno = ENXIO), @@ -395,7 +413,7 @@ RedisCluster *redisxClusterInit(Redis *node) { pthread_mutex_init(&p->mutex, NULL); - p->shard = rClusterDiscoverAsync(node, &p->n_shards, cluster); + p->shard = rClusterDiscoverAsync(cluster, node, &p->n_shards); if(p->n_shards <= 0) { redisxClusterDestroy(cluster); return x_trace_null(fn, NULL);