From 234e5e03fc75ea248f31e692ed2a95fc6745652c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Mon, 20 May 2024 16:45:50 +0200 Subject: [PATCH 1/3] Add support for Redis logical DBs in async mode --- lib/3scale/backend/storage_async/client.rb | 29 ++++++++++++++++------ lib/async/redis/protocol/extended_resp2.rb | 28 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 lib/async/redis/protocol/extended_resp2.rb diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 29ecf0cc..62c38af2 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,6 +1,7 @@ require 'async/io' require 'async/redis/client' require 'async/redis/sentinels' +require 'async/redis/protocol/extended_resp2' module ThreeScale module Backend @@ -71,20 +72,34 @@ def initialize_client(opts) end def init_host_client(opts) - uri = URI(opts[:url] || '') - host = uri.host || DEFAULT_HOST - port = uri.port || DEFAULT_PORT - - endpoint = Async::IO::Endpoint.tcp(host, port) - Async::Redis::Client.new(endpoint, limit: opts[:max_connections]) + endpoint = make_redis_endpoint(opts) + protocol = make_redis_protocol(opts) + Async::Redis::Client.new(endpoint, protocol: protocol, limit: opts[:max_connections]) end def init_sentinels_client(opts) uri = URI(opts[:url] || '') name = uri.host role = opts[:role] || :master + protocol = make_redis_protocol(opts) + + Async::Redis::SentinelsClient.new(name, opts[:sentinels], role, protocol) + end + + # RESP2 with support for logical DBs + def make_redis_protocol(opts) + uri = URI(opts[:url] || "") + db = uri.path[1..-1] + + Async::Redis::Protocol::ExtendedRESP2.new(db: db) + end + + def make_redis_endpoint(opts) + uri = URI(opts[:url] || "") + host = uri.host || DEFAULT_HOST + port = uri.port || DEFAULT_PORT - Async::Redis::SentinelsClient.new(name, opts[:sentinels], role) + Async::IO::Endpoint.tcp(host, port) end end end diff --git a/lib/async/redis/protocol/extended_resp2.rb b/lib/async/redis/protocol/extended_resp2.rb new file mode 100644 index 00000000..72ffbca5 --- /dev/null +++ b/lib/async/redis/protocol/extended_resp2.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'async/redis/protocol/resp2' + +module Async + module Redis + module Protocol + + # Custom Redis Protocol supporting Redis logical DBs + class ExtendedRESP2 + def initialize(db: nil) + @db = db + end + + def client(stream) + client = Async::Redis::Protocol::RESP2.client(stream) + + if @db + client.write_request(["SELECT", @db]) + client.read_response + end + + client + end + end + end + end +end From 41a9925738ecd5eff28879ef56861461edd64ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Tue, 21 May 2024 14:42:15 +0200 Subject: [PATCH 2/3] Move the protocol to our namespace --- .../backend/redis/protocol/extended_resp2.rb | 30 +++++++++++++++++++ lib/3scale/backend/stats/cleaner.rb | 2 +- lib/3scale/backend/storage_async/client.rb | 4 +-- lib/3scale/backend/storage_sync.rb | 2 +- lib/async/redis/protocol/extended_resp2.rb | 28 ----------------- test/unit/worker_test.rb | 2 +- 6 files changed, 35 insertions(+), 33 deletions(-) create mode 100644 lib/3scale/backend/redis/protocol/extended_resp2.rb delete mode 100644 lib/async/redis/protocol/extended_resp2.rb diff --git a/lib/3scale/backend/redis/protocol/extended_resp2.rb b/lib/3scale/backend/redis/protocol/extended_resp2.rb new file mode 100644 index 00000000..6498ba1d --- /dev/null +++ b/lib/3scale/backend/redis/protocol/extended_resp2.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'async/redis/protocol/resp2' + +module ThreeScale + module Backend + module Redis + module Protocol + + # Custom Redis Protocol supporting Redis logical DBs + class ExtendedRESP2 + def initialize(db: nil) + @db = db + end + + def client(stream) + client = Async::Redis::Protocol::RESP2.client(stream) + + if @db + client.write_request(["SELECT", @db]) + client.read_response + end + + client + end + end + end + end + end +end diff --git a/lib/3scale/backend/stats/cleaner.rb b/lib/3scale/backend/stats/cleaner.rb index 63b31d9e..37fa9308 100644 --- a/lib/3scale/backend/stats/cleaner.rb +++ b/lib/3scale/backend/stats/cleaner.rb @@ -45,7 +45,7 @@ class Cleaner STATS_KEY_PREFIX = 'stats/'.freeze private_constant :STATS_KEY_PREFIX - REDIS_CONN_ERRORS = [Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze + REDIS_CONN_ERRORS = [::Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze private_constant :REDIS_CONN_ERRORS MAX_RETRIES_REDIS_ERRORS = 3 diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 62c38af2..3f67b242 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,7 +1,7 @@ require 'async/io' require 'async/redis/client' require 'async/redis/sentinels' -require 'async/redis/protocol/extended_resp2' +require '3scale/backend/redis/protocol/extended_resp2' module ThreeScale module Backend @@ -91,7 +91,7 @@ def make_redis_protocol(opts) uri = URI(opts[:url] || "") db = uri.path[1..-1] - Async::Redis::Protocol::ExtendedRESP2.new(db: db) + ThreeScale::Backend::Redis::Protocol::ExtendedRESP2.new(db: db) end def make_redis_endpoint(opts) diff --git a/lib/3scale/backend/storage_sync.rb b/lib/3scale/backend/storage_sync.rb index 40135f0b..30ba0744 100644 --- a/lib/3scale/backend/storage_sync.rb +++ b/lib/3scale/backend/storage_sync.rb @@ -20,7 +20,7 @@ def instance(reset = false) end def new(options) - Redis.new options + ::Redis.new options end private diff --git a/lib/async/redis/protocol/extended_resp2.rb b/lib/async/redis/protocol/extended_resp2.rb deleted file mode 100644 index 72ffbca5..00000000 --- a/lib/async/redis/protocol/extended_resp2.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require 'async/redis/protocol/resp2' - -module Async - module Redis - module Protocol - - # Custom Redis Protocol supporting Redis logical DBs - class ExtendedRESP2 - def initialize(db: nil) - @db = db - end - - def client(stream) - client = Async::Redis::Protocol::RESP2.client(stream) - - if @db - client.write_request(["SELECT", @db]) - client.read_response - end - - client - end - end - end - end -end diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index 5969b3b3..1da10021 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -35,7 +35,7 @@ def test_format_of_a_job def test_no_jobs_in_the_queue # Stub blpop to avoid waiting until timeout. - Redis.any_instance.stubs(:blpop).returns(nil) + ::Redis.any_instance.stubs(:blpop).returns(nil) StorageAsync::Client.any_instance.stubs(:blpop).returns(nil) Worker.work(:one_off => true) From 7a2da06b635818bfda7d0401b7357d322e177655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20Lled=C3=B3?= Date: Wed, 22 May 2024 12:30:46 +0200 Subject: [PATCH 3/3] Move `ExtendedRESP2` to `AsyncRedis` namespace --- .../backend/{redis => async_redis}/protocol/extended_resp2.rb | 2 +- lib/3scale/backend/stats/cleaner.rb | 2 +- lib/3scale/backend/storage_async/client.rb | 4 ++-- lib/3scale/backend/storage_sync.rb | 2 +- test/unit/worker_test.rb | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename lib/3scale/backend/{redis => async_redis}/protocol/extended_resp2.rb (96%) diff --git a/lib/3scale/backend/redis/protocol/extended_resp2.rb b/lib/3scale/backend/async_redis/protocol/extended_resp2.rb similarity index 96% rename from lib/3scale/backend/redis/protocol/extended_resp2.rb rename to lib/3scale/backend/async_redis/protocol/extended_resp2.rb index 6498ba1d..2e8a8031 100644 --- a/lib/3scale/backend/redis/protocol/extended_resp2.rb +++ b/lib/3scale/backend/async_redis/protocol/extended_resp2.rb @@ -4,7 +4,7 @@ module ThreeScale module Backend - module Redis + module AsyncRedis module Protocol # Custom Redis Protocol supporting Redis logical DBs diff --git a/lib/3scale/backend/stats/cleaner.rb b/lib/3scale/backend/stats/cleaner.rb index 37fa9308..63b31d9e 100644 --- a/lib/3scale/backend/stats/cleaner.rb +++ b/lib/3scale/backend/stats/cleaner.rb @@ -45,7 +45,7 @@ class Cleaner STATS_KEY_PREFIX = 'stats/'.freeze private_constant :STATS_KEY_PREFIX - REDIS_CONN_ERRORS = [::Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze + REDIS_CONN_ERRORS = [Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze private_constant :REDIS_CONN_ERRORS MAX_RETRIES_REDIS_ERRORS = 3 diff --git a/lib/3scale/backend/storage_async/client.rb b/lib/3scale/backend/storage_async/client.rb index 3f67b242..c989e72a 100644 --- a/lib/3scale/backend/storage_async/client.rb +++ b/lib/3scale/backend/storage_async/client.rb @@ -1,7 +1,7 @@ require 'async/io' require 'async/redis/client' require 'async/redis/sentinels' -require '3scale/backend/redis/protocol/extended_resp2' +require '3scale/backend/async_redis/protocol/extended_resp2' module ThreeScale module Backend @@ -91,7 +91,7 @@ def make_redis_protocol(opts) uri = URI(opts[:url] || "") db = uri.path[1..-1] - ThreeScale::Backend::Redis::Protocol::ExtendedRESP2.new(db: db) + ThreeScale::Backend::AsyncRedis::Protocol::ExtendedRESP2.new(db: db) end def make_redis_endpoint(opts) diff --git a/lib/3scale/backend/storage_sync.rb b/lib/3scale/backend/storage_sync.rb index 30ba0744..40135f0b 100644 --- a/lib/3scale/backend/storage_sync.rb +++ b/lib/3scale/backend/storage_sync.rb @@ -20,7 +20,7 @@ def instance(reset = false) end def new(options) - ::Redis.new options + Redis.new options end private diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index 1da10021..5969b3b3 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -35,7 +35,7 @@ def test_format_of_a_job def test_no_jobs_in_the_queue # Stub blpop to avoid waiting until timeout. - ::Redis.any_instance.stubs(:blpop).returns(nil) + Redis.any_instance.stubs(:blpop).returns(nil) StorageAsync::Client.any_instance.stubs(:blpop).returns(nil) Worker.work(:one_off => true)