diff --git a/docs/modules/ngx_http_upstream_iwrr_module.md b/docs/modules/ngx_http_upstream_iwrr_module.md new file mode 100644 index 0000000000..7f20a31e3b --- /dev/null +++ b/docs/modules/ngx_http_upstream_iwrr_module.md @@ -0,0 +1,61 @@ + +## Name + +ngx_http_upstream_iwrr_module. + + +## Introduction + +The `IWRR` module is an efficient load balancing algorithm with `O(1)` time complexity, but `IWRR` is no need to incremental initialization. + +Compared with Nginx's official `SWRR` algorithm and `VNSWRR`, `IWRR` abandons smoothness on the premise of ensuring the correctness of the weighted load balancing algorithm, ensuring that no matter how the total weight of the cluster changes, `IWRR` space The complexity is always `O(n)`. + +## Example + +``` +http { + + upstream backend { + iwrr; # enable IWRR load balancing algorithm. + 127.0.0.1 port=81; + 127.0.0.1 port=82 weight=2; + 127.0.0.1 port=83; + 127.0.0.1 port=84 backup; + 127.0.0.1 port=85 down; + } + + server { + server_name localhost; + + location / { + proxy_pass http://backend; + } + } +} + +``` + +## Installation + +Build Tengine with this module from source: + +``` + +./configure --add-module=./modules/ngx_http_upstream_iwrr_module/ +make +make install + +``` + + +## Directive + +iwrr +======= +``` +Syntax: iwrr [max_init=number] +Default: none +Context: upstream +``` + +Enable `iwrr` load balancing algorithm. diff --git a/docs/modules/ngx_http_upstream_iwrr_module_cn.md b/docs/modules/ngx_http_upstream_iwrr_module_cn.md new file mode 100644 index 0000000000..71c08c5cbe --- /dev/null +++ b/docs/modules/ngx_http_upstream_iwrr_module_cn.md @@ -0,0 +1,62 @@ + +## 名称 + +ngx_http_upstream_iwrr_module. + + +## 介绍 + +`IWRR`模块是一个高效的负载均衡算法，与`VNSWRR`相同，它具有`O(1)`的时间复杂度，但是`IWRR`不需要执行渐进式初始化操作。 + +同Nginx官方的加权轮询负载均衡算法及`VNSWRR`相比，`IWRR`在保证加权负载均衡算法正确性的前提下，牺牲了平滑的特点，保证无论集群总权重如何变化，`IWRR`空间复杂度总是`O(n)`的。 + +## 配置列子 + +``` +http { + + upstream backend { + iwrr; # enable IWRR load balancing algorithm. + 127.0.0.1 port=81; + 127.0.0.1 port=82 weight=2; + 127.0.0.1 port=83; + 127.0.0.1 port=84 backup; + 127.0.0.1 port=85 down; + } + + server { + server_name localhost; + + location / { + proxy_pass http://backend; + } + } +} + +``` + +## 安装方法 + +在Tengine中，通过源码安装此模块： + + +``` + +./configure --add-module=./modules/ngx_http_upstream_iwrr_module +make +make install + +``` + + +## 指令描述 + +iwrr +======= +``` +Syntax: iwrr +Default: none +Context: upstream +``` + +在upstream里面启用 `iwrr` 加权轮询算法。 diff --git a/modules/ngx_http_upstream_iwrr_module/config b/modules/ngx_http_upstream_iwrr_module/config new file mode 100644 index 0000000000..095521dd3e --- /dev/null +++ b/modules/ngx_http_upstream_iwrr_module/config @@ -0,0 +1,14 @@ +ngx_addon_name=ngx_http_upstream_iwrr_module +HTTP_UPSTREAM_IWRR_SRCS="$ngx_addon_dir/ngx_http_upstream_iwrr_module.c" + +if test -n "$ngx_module_link"; then + ngx_module_type=HTTP + ngx_module_name=$ngx_addon_name + ngx_module_deps= + ngx_module_srcs="$HTTP_UPSTREAM_IWRR_SRCS" + + . auto/module +else + HTTP_MODULES="$HTTP_MODULES ngx_http_upstream_iwrr_module" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $HTTP_UPSTREAM_IWRR_SRCS" +fi \ No newline at end of file diff --git a/modules/ngx_http_upstream_iwrr_module/ngx_http_upstream_iwrr_module.c b/modules/ngx_http_upstream_iwrr_module/ngx_http_upstream_iwrr_module.c new file mode 100644 index 0000000000..13caf6ff23 --- /dev/null +++ b/modules/ngx_http_upstream_iwrr_module/ngx_http_upstream_iwrr_module.c @@ -0,0 +1,464 @@ + +/* + * Copyright (C) 2010-2023 Alibaba Group Holding Limited + */ + + +#include +#include +#include + +#if (NGX_HTTP_UPSTREAM_CHECK) +#include "ngx_http_upstream_check_module.h" +#endif + +typedef struct { + ngx_queue_t queue; + ngx_uint_t index; + ngx_uint_t weight; + ngx_uint_t remainder; + ngx_http_upstream_rr_peer_t *peer; +} ngx_http_upstream_iwrr_queue_t; + +typedef struct ngx_http_upstream_iwrr_srv_conf_s ngx_http_upstream_iwrr_srv_conf_t; + +struct ngx_http_upstream_iwrr_srv_conf_s { + ngx_uint_t init_number; + ngx_http_upstream_iwrr_queue_t *active; + ngx_http_upstream_iwrr_queue_t *expired; + ngx_http_upstream_iwrr_srv_conf_t *next; +}; + +typedef struct { + /* the round robin data must be first */ + ngx_http_upstream_rr_peer_data_t rrp; + + ngx_http_upstream_iwrr_srv_conf_t *uiscf; +} ngx_http_upstream_iwrr_peer_data_t; + +static char *ngx_http_upstream_iwrr(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static void *ngx_http_upstream_iwrr_create_srv_conf(ngx_conf_t *cf); +static ngx_int_t ngx_http_upstream_init_iwrr(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_init_iwrr_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_get_iwrr_peer(ngx_peer_connection_t *pc, + void *data); +static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_iwrr( + ngx_http_upstream_iwrr_peer_data_t *oip); +static ngx_http_upstream_iwrr_queue_t *ngx_http_upstream_iwrr_queue_next( + ngx_http_upstream_iwrr_srv_conf_t *uiscf); + +static inline ngx_uint_t ngx_http_upstream_iwrr_gcd(ngx_uint_t a, ngx_uint_t b); + +static ngx_command_t ngx_http_upstream_iwrr_commands[] = { + + { ngx_string("iwrr"), + NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS, + ngx_http_upstream_iwrr, + 0, + 0, + NULL }, + + ngx_null_command +}; + +static ngx_http_module_t ngx_http_upstream_iwrr_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + ngx_http_upstream_iwrr_create_srv_conf, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + +ngx_module_t ngx_http_upstream_iwrr_module = { + NGX_MODULE_V1, + &ngx_http_upstream_iwrr_module_ctx, /* module context */ + ngx_http_upstream_iwrr_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + +static char * +ngx_http_upstream_iwrr(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + if (uscf->peer.init_upstream) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "load balancing method redefined"); + } + + uscf->peer.init_upstream = ngx_http_upstream_init_iwrr; + + uscf->flags = NGX_HTTP_UPSTREAM_CREATE + |NGX_HTTP_UPSTREAM_WEIGHT + |NGX_HTTP_UPSTREAM_BACKUP + |NGX_HTTP_UPSTREAM_MAX_FAILS + |NGX_HTTP_UPSTREAM_FAIL_TIMEOUT +#if defined(nginx_version) && nginx_version >= 1011005 + |NGX_HTTP_UPSTREAM_MAX_CONNS +#endif + |NGX_HTTP_UPSTREAM_DOWN; + + return NGX_CONF_OK; +} + +static void * +ngx_http_upstream_iwrr_create_srv_conf(ngx_conf_t *cf) +{ + ngx_http_upstream_iwrr_srv_conf_t *uiscf; + + uiscf = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_iwrr_srv_conf_t)); + if (uiscf == NULL) { + return NULL; + } + + /* + * set by ngx_pcalloc(): + * + * uiscf->active = NULL; + * uiscf->expired = NULL; + * uiscf->next = NULL; + */ + + uiscf->init_number = NGX_CONF_UNSET_UINT; + + return uiscf; +} + +static ngx_int_t +ngx_http_upstream_init_iwrr(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_rr_peer_t *peer; + ngx_http_upstream_iwrr_srv_conf_t *uiscf; + ngx_http_upstream_iwrr_queue_t *item; + ngx_uint_t i, g; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "init iwrr"); + + if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + us->peer.init = ngx_http_upstream_init_iwrr_peer; + + uiscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_iwrr_module); + + peers = (ngx_http_upstream_rr_peers_t *) us->peer.data; + + for (peers = (ngx_http_upstream_rr_peers_t *) us->peer.data; + peers; + peers = peers->next) + { + + g = 0; + + for (peer = peers->peer; + peer; + peer = peer->next) + { + g = ngx_http_upstream_iwrr_gcd(g, peer->weight); + } + + uiscf->active = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_iwrr_queue_t)); + if (uiscf->active == NULL) { + return NGX_ERROR; + } + ngx_queue_init(&uiscf->active->queue); + + uiscf->expired = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_iwrr_queue_t)); + if (uiscf->active == NULL) { + return NGX_ERROR; + } + ngx_queue_init(&uiscf->expired->queue); + + for (peer = peers->peer, i = 0; + peer; + peer = peer->next, i++) + { + + item = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_iwrr_queue_t)); + if (item == NULL) { + return NGX_ERROR; + } + + item->index = i; + item->weight = peer->weight / g; + item->remainder = item->weight; + item->peer = peer; + ngx_queue_insert_tail(&uiscf->active->queue, &item->queue); + } + + if (peers->next) { + uiscf->next = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_iwrr_srv_conf_t)); + if (uiscf->next == NULL) { + return NGX_ERROR; + } + + uiscf->next->init_number = NGX_CONF_UNSET_UINT; + + uiscf = uiscf->next; + } + } + + return NGX_OK; +} + +static ngx_int_t +ngx_http_upstream_init_iwrr_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_http_upstream_iwrr_srv_conf_t *uiscf; + ngx_http_upstream_iwrr_peer_data_t *oip; + + uiscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_iwrr_module); + if (uiscf == NULL) { + return NGX_ERROR; + } + + oip = ngx_palloc(r->pool, sizeof(ngx_http_upstream_iwrr_peer_data_t)); + if (oip == NULL) { + return NGX_ERROR; + } + + oip->uiscf = uiscf; + r->upstream->peer.data = &oip->rrp; + + if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) { + return NGX_ERROR; + } + + r->upstream->peer.get = ngx_http_upstream_get_iwrr_peer; + + return NGX_OK; +} + +static ngx_int_t +ngx_http_upstream_get_iwrr_peer(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_iwrr_peer_data_t *oip = data; + + ngx_int_t rc; + ngx_uint_t i, n; + ngx_http_upstream_rr_peer_t *peer; + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_rr_peer_data_t *rrp; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get iwrr peer, try: %ui", pc->tries); + + pc->cached = 0; + pc->connection = NULL; + + rrp = &oip->rrp; + + peers = rrp->peers; + ngx_http_upstream_rr_peers_wlock(peers); + + if (peers->single) { + peer = peers->peer; + + if (peer->down) { + goto failed; + } + + if (peer->max_conns && peer->conns >= peer->max_conns) { + goto failed; + } + +#if (NGX_HTTP_UPSTREAM_CHECK) + if (ngx_http_upstream_check_peer_down(peer->check_index)) { + goto failed; + } +#endif + rrp->current = peer; + + } else { + + /* there are several peers */ + + peer = ngx_http_upstream_get_iwrr(oip); + + if (peer == NULL) { + goto failed; + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get iwrr peer, current: %p %i", + peer, peer->current_weight); + } + + pc->sockaddr = peer->sockaddr; + pc->socklen = peer->socklen; + pc->name = &peer->name; +#if (T_NGX_HTTP_DYNAMIC_RESOLVE) + pc->host = &peer->host; +#endif + + peer->conns++; + + ngx_http_upstream_rr_peers_unlock(peers); + + return NGX_OK; + + +failed: + + if (peers->next) { + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers"); + + rrp->peers = peers->next; + + oip->uiscf = oip->uiscf ? oip->uiscf->next : oip->uiscf; + + n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1)) + / (8 * sizeof(uintptr_t)); + + for (i = 0; i < n; i++) { + rrp->tried[i] = 0; + } + + ngx_http_upstream_rr_peers_unlock(peers); + + rc = ngx_http_upstream_get_iwrr_peer(pc, oip); + + if (rc != NGX_BUSY) { + return rc; + } + + ngx_http_upstream_rr_peers_wlock(peers); + } + + ngx_http_upstream_rr_peers_unlock(peers); + + pc->name = peers->name; + + return NGX_BUSY; +} + +static ngx_http_upstream_rr_peer_t * +ngx_http_upstream_get_iwrr(ngx_http_upstream_iwrr_peer_data_t *oip) +{ + time_t now; + uintptr_t m; + ngx_uint_t i, j, n; + ngx_http_upstream_rr_peer_t *peer; + ngx_http_upstream_rr_peers_t *peers; + ngx_http_upstream_rr_peer_data_t *rrp; + ngx_http_upstream_iwrr_srv_conf_t *uiscf; + ngx_http_upstream_iwrr_queue_t *item; + + now = ngx_time(); + + rrp = &oip->rrp; + peers = rrp->peers; + uiscf = oip->uiscf; + +#if (T_NGX_HTTP_UPSTREAM_RANDOM) + if (uiscf->init_number == NGX_CONF_UNSET_UINT) { + uiscf->init_number = ngx_random() % peers->number; + + for (i = 0; i < uiscf->init_number; i++) { + ngx_http_upstream_iwrr_queue_next(uiscf); + } + } +#endif + + for (j = 0; j < peers->number; j++) { + item = ngx_http_upstream_iwrr_queue_next(uiscf); + + i = item->index; + peer = item->peer; + + n = i / (8 * sizeof(uintptr_t)); + m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); + + if (rrp->tried[n] & m) { + continue; + } + + if (peer->down) { + continue; + } + +#if (NGX_HTTP_UPSTREAM_CHECK) + if (ngx_http_upstream_check_peer_down(peer->check_index)) { + continue; + } +#endif + + if (peer->max_fails + && peer->fails >= peer->max_fails + && now - peer->checked <= peer->fail_timeout) + { + continue; + } + + if (peer->max_conns && peer->conns >= peer->max_conns) { + continue; + } + + rrp->current = peer; + + return peer; + } + + return NULL; +} + +static ngx_http_upstream_iwrr_queue_t * +ngx_http_upstream_iwrr_queue_next(ngx_http_upstream_iwrr_srv_conf_t *uiscf) +{ + ngx_http_upstream_iwrr_queue_t *temp, *item; + + if (ngx_queue_empty(&uiscf->active->queue)) { + temp = uiscf->active; + uiscf->active = uiscf->expired; + uiscf->expired = temp; + } + + item = (ngx_http_upstream_iwrr_queue_t *) ngx_queue_head(&uiscf->active->queue); + ngx_queue_remove(&item->queue); + + item->remainder--; + if (item->remainder) { + ngx_queue_insert_tail(&uiscf->active->queue, &item->queue); + } else { + item->remainder = item->weight; + ngx_queue_insert_tail(&uiscf->expired->queue, &item->queue); + } + + return item; +} + +static inline ngx_uint_t ngx_http_upstream_iwrr_gcd(ngx_uint_t a, ngx_uint_t b) +{ + ngx_uint_t r; + while (b) { + r = a % b; + a = b; + b = r; + } + return a; +} \ No newline at end of file diff --git a/tests/nginx-tests/tengine-tests/iwrr.t b/tests/nginx-tests/tengine-tests/iwrr.t new file mode 100644 index 0000000000..fb2f28f037 --- /dev/null +++ b/tests/nginx-tests/tengine-tests/iwrr.t @@ -0,0 +1,233 @@ +#!/usr/bin/perl + +# Copyright (C) 2010-2023 Alibaba Group Holding Limited + +# Tests for upstream iwrr balancer module. + +############################################################################### + +use warnings; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx qw/ :DEFAULT :gzip/ ; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http proxy upstream_zone iwrr/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; +worker_processes 1; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + upstream u { + iwrr; + server 127.0.0.1:8081; + server 127.0.0.1:8082; + server 127.0.0.1:8083 down; + } + + upstream w { + iwrr; + server 127.0.0.1:8081; + server 127.0.0.1:8082 weight=2; + } + + upstream zone { + iwrr; + server 127.0.0.1:8081; + } + + upstream b { + iwrr; + server 127.0.0.1:8081 down; + server 127.0.0.1:8082 backup; + } + + upstream d { + iwrr; + server 127.0.0.1:8081; + server 127.0.0.1:8082; + server 127.0.0.1:8083 weight=2; + server 127.0.0.1:8084 down; + } + + upstream g { + iwrr; + server 127.0.0.1:8081 weight=2; + server 127.0.0.1:8082 weight=4; + server 127.0.0.1:8083 weight=8; + } + + upstream h { + iwrr; + server 127.0.0.1:8081; + server 127.0.0.1:8082; + server 127.0.0.1:8083; + } + + upstream i { + iwrr; + server 127.0.0.1:8081 down; + server 127.0.0.1:8082 backup; + server 127.0.0.1:8083 backup; + } + + server { + listen 127.0.0.1:8081; + listen 127.0.0.1:8082; + listen 127.0.0.1:8083; + server_name localhost; + + location / { + return 200 $server_port; + } + } + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location / { + proxy_pass http://u; + } + + location /w { + proxy_pass http://w; + } + + location /zone { + proxy_pass http://zone; + } + + location /b { + proxy_pass http://b; + } + + location /d { + proxy_pass http://d; + } + + location /g { + proxy_pass http://g; + } + + location /h { + proxy_pass http://h; + } + + location /i { + proxy_pass http://i; + } + } +} + +EOF + +$t->try_run('no upstream iwrr')->plan(18); + +############################################################################### +my $r; +my %list = (); + +$list{'8083'} = 0; +$list{http_get_body('/')} = 1; +$list{http_get_body('/')} = 1; +$list{http_get_body('/')} = 1; + +is($list{'8081'}, 1, 'iwrr 8081'); +is($list{'8082'}, 1, 'iwrr 8082'); +is($list{'8083'}, 0, 'peer down'); + +%list = (); +$list{http_get_body('/w')} += 1; +$list{http_get_body('/w')} += 1; +$list{http_get_body('/w')} += 1; + +is($list{'8081'}, 1, 'weight 1'); +is($list{'8082'}, 2, 'weight 2'); + +%list = (); +$list{http_get_body('/zone')} += 1; + +is($list{'8081'}, 1, 'iwrr zone'); + +%list = (); +$list{http_get_body('/b')} += 1; + +is($list{'8082'}, 1, 'iwrr backup'); + +%list = (); +$list{http_get_body('/d')} += 1; +$list{http_get_body('/d')} += 1; +$list{http_get_body('/d')} += 1; +$list{http_get_body('/d')} += 1; + +is($list{'8081'}, 1, 'weight 1'); +is($list{'8082'}, 1, 'weight 1'); +is($list{'8083'}, 2, 'weight 2'); + +%list = (); +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; +$list{http_get_body('/g')} += 1; + +is($list{'8081'}, 2, 'weight 2'); +is($list{'8082'}, 4, 'weight 4'); +is($list{'8083'}, 8, 'weight 8'); + +%list = (); +$list{http_get_body('/h')} += 1; +$list{http_get_body('/h')} += 1; +$list{http_get_body('/h')} += 1; + +is($list{'8081'}, 1, 'weight 1'); +is($list{'8082'}, 1, 'weight 1'); +is($list{'8083'}, 1, 'weight 1'); + +%list = (); +$list{http_get_body('/i')} += 1; +$list{http_get_body('/i')} += 1; + +is($list{'8082'}, 1, 'weight 1'); +is($list{'8083'}, 1, 'weight 1'); + + +############################################################################### + +sub http_get_body { + my ($uri) = @_; + + return undef if !defined $uri; + + http_get($uri) =~ /(.*?)\x0d\x0a?\x0d\x0a?(.*)/ms; + + return $2; +} + +############################################################################### \ No newline at end of file