From 89a495ceae741b9b002e27012aca6c8861d0abf3 Mon Sep 17 00:00:00 2001 From: Nikos Vasileiou Date: Mon, 4 Dec 2023 13:29:31 +0200 Subject: [PATCH] Add Redis in rate limiter --- config/defaults.yml | 17 ++++++++++++++--- package-lock.json | 18 ++++++++++++++++++ package.json | 1 + src/helpers/ioredis.js | 11 +++++++++++ src/helpers/ratelimit.js | 12 +++++++++++- src/index.js | 1 + src/queue/index.js | 1 + src/routes/content.js | 1 + src/routes/languages.js | 2 ++ src/routes/status.js | 13 ------------- src/server.js | 4 +--- src/services/cache/strategies/redis/index.js | 4 ++-- .../registry/strategies/redis/index.js | 4 ++-- src/telemetry.js | 9 +++++++-- 14 files changed, 72 insertions(+), 26 deletions(-) delete mode 100644 src/routes/status.js diff --git a/config/defaults.yml b/config/defaults.yml index 116788b..3d3676b 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -203,6 +203,9 @@ queue: limits: # Rate limits for push content (max requests per seconds) # https://github.com/nfriedly/express-rate-limit + pull: + window_sec: 2 + max_req: 1000 push: window_sec: 60 max_req: 20 @@ -212,19 +215,27 @@ limits: jobs: window_sec: 5 max_req: 120 + # Prefix namespace for keys in Redis. Modify if you want to separate + # data, between multiple environments (e.g. beta, staging) that are using + # the same Redis instance, e.g. + # TX__LIMITS__PREFIX="beta:ratelimit:" + prefix: 'ratelimit:' + telemetry: # If usage telemetry should be enabled enabled: true - # Telemetry service host host: https://telemetry.svc.transifex.net - # Request timeout to Telemetry service req_timeout_sec: 5 - # Max concurrent requests per process max_concurrent_req: 10 + # Prefix namespace for keys in Redis. Modify if you want to separate + # data, between multiple environments (e.g. beta, staging) that are using + # the same Redis instance, e.g. + # TX__TELEMETRY__PREFIX="beta:telemetry:" + prefix: 'telemetry:' # # Manually pass AWS config (local testing) # aws: diff --git a/package-lock.json b/package-lock.json index 8467f2d..ce754ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "newrelic": "^11.6.0", "node-cache": "^5.1.2", "prom-client": "^15.0.0", + "rate-limit-redis": "^4.2.0", "rate-limiter-flexible": "^3.0.4", "uuid": "^8.3.2", "winston": "^3.11.0", @@ -8455,6 +8456,17 @@ "node": ">= 0.6" } }, + "node_modules/rate-limit-redis": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-4.2.0.tgz", + "integrity": "sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA==", + "engines": { + "node": ">= 16" + }, + "peerDependencies": { + "express-rate-limit": ">= 6" + } + }, "node_modules/rate-limiter-flexible": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-3.0.4.tgz", @@ -16672,6 +16684,12 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limit-redis": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-4.2.0.tgz", + "integrity": "sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA==", + "requires": {} + }, "rate-limiter-flexible": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-3.0.4.tgz", diff --git a/package.json b/package.json index 367ee05..5c619bb 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "newrelic": "^11.6.0", "node-cache": "^5.1.2", "prom-client": "^15.0.0", + "rate-limit-redis": "^4.2.0", "rate-limiter-flexible": "^3.0.4", "uuid": "^8.3.2", "winston": "^3.11.0", diff --git a/src/helpers/ioredis.js b/src/helpers/ioredis.js index 4e43129..6f84f2f 100644 --- a/src/helpers/ioredis.js +++ b/src/helpers/ioredis.js @@ -1,6 +1,8 @@ const Redis = require('ioredis'); const config = require('../config'); +let singletonClient; + function createClient() { const redisUrl = config.get('redis:host'); if (redisUrl) { @@ -21,6 +23,15 @@ function createClient() { }); } +function getClient() { + if (singletonClient) { + return singletonClient; + } + singletonClient = createClient(); + return singletonClient; +} + module.exports = { createClient, + getClient, }; diff --git a/src/helpers/ratelimit.js b/src/helpers/ratelimit.js index bcbbd1a..8e63139 100644 --- a/src/helpers/ratelimit.js +++ b/src/helpers/ratelimit.js @@ -1,5 +1,9 @@ -const rateLimit = require('express-rate-limit'); +const { rateLimit } = require('express-rate-limit'); +const { default: RedisStore } = require('rate-limit-redis'); const config = require('../config'); +const { getClient } = require('./ioredis'); + +const redisClient = getClient(); /** * Create a scope based rate limiter based on project_token. @@ -14,6 +18,7 @@ const config = require('../config'); * @return {*} */ function createRateLimiter(scope) { + const redisKeyPrefix = config.get('limits:prefix'); const limitPushWindowMsec = config.get(`limits:${scope}:window_sec`) * 1000; const limitPushMaxReq = config.get(`limits:${scope}:max_req`) * 1; return rateLimit({ @@ -24,6 +29,11 @@ function createRateLimiter(scope) { status: 429, message: 'Too many requests, please try again later.', }, + // Redis store configuration + store: new RedisStore({ + sendCommand: (...args) => redisClient.call(...args), + prefix: `${redisKeyPrefix}${scope}:`, + }), }); } diff --git a/src/index.js b/src/index.js index b5c1284..83a80a9 100644 --- a/src/index.js +++ b/src/index.js @@ -35,6 +35,7 @@ async function launch() { const keepAliveTimeoutSec = config.get('settings:keep_alive_timeout_sec'); if (isNumber(keepAliveTimeoutSec) && keepAliveTimeoutSec >= 0) { attachedApplication.keepAliveTimeout = keepAliveTimeoutSec * 1000; + attachedApplication.headersTimeout = (keepAliveTimeoutSec + 1) * 1000; } // graceful shutdown of server diff --git a/src/queue/index.js b/src/queue/index.js index 1da4e99..1f21852 100644 --- a/src/queue/index.js +++ b/src/queue/index.js @@ -5,6 +5,7 @@ const logger = require('../logger'); const config = require('../config'); const worker = require('./worker'); +// Always create a new Redis client const queue = new Queue(config.get('queue:name'), { createClient: () => createClient(), }); diff --git a/src/routes/content.js b/src/routes/content.js index 488bccc..1e2509a 100644 --- a/src/routes/content.js +++ b/src/routes/content.js @@ -67,6 +67,7 @@ async function getContent(req, res) { router.get( '/:lang_code', validateHeader('public'), + createRateLimiter('pull'), getContent, ); diff --git a/src/routes/languages.js b/src/routes/languages.js index f53a68b..a1e4b62 100644 --- a/src/routes/languages.js +++ b/src/routes/languages.js @@ -1,12 +1,14 @@ const express = require('express'); const { validateHeader } = require('../middlewares/headers'); const utils = require('../helpers/utils'); +const { createRateLimiter } = require('../helpers/ratelimit'); const router = express.Router(); router.get( '/', validateHeader('public'), + createRateLimiter('pull'), async (req, res) => { const filter = req.query.filter || {}; const key = `${req.token.project_token}:languages`; diff --git a/src/routes/status.js b/src/routes/status.js deleted file mode 100644 index fe9ae44..0000000 --- a/src/routes/status.js +++ /dev/null @@ -1,13 +0,0 @@ -const express = require('express'); - -const router = express.Router(); - -router.get('/:lang_code', async (req, res) => { - res.send(''); -}); - -router.get('/', async (req, res) => { - res.send(''); -}); - -module.exports = router; diff --git a/src/server.js b/src/server.js index eeac50c..bdde789 100644 --- a/src/server.js +++ b/src/server.js @@ -10,7 +10,6 @@ const logger = require('./logger'); const metrics = require('./middlewares/metrics'); const languagesRouter = require('./routes/languages'); const contentRouter = require('./routes/content'); -const statusRouter = require('./routes/status'); const invalidateRouter = require('./routes/invalidate'); const purgeRouter = require('./routes/purge'); const jobsRouter = require('./routes/jobs'); @@ -67,12 +66,11 @@ module.exports = () => { app.use('/languages', languagesRouter); app.use('/content', contentRouter); - app.use('/status', statusRouter); app.use('/invalidate', invalidateRouter); app.use('/purge', purgeRouter); app.use('/jobs', jobsRouter); - app.get('/', (req, res) => res.send('ok')); + app.get('/', (req, res) => res.send(`Transifex CDS - v${version}`)); // The error handler must be before any other error middleware sentry.expressError(app); diff --git a/src/services/cache/strategies/redis/index.js b/src/services/cache/strategies/redis/index.js index a0d95fc..0e99f40 100644 --- a/src/services/cache/strategies/redis/index.js +++ b/src/services/cache/strategies/redis/index.js @@ -1,10 +1,10 @@ const config = require('../../../../config'); const logger = require('../../../../logger'); -const { createClient } = require('../../../../helpers/ioredis'); +const { getClient } = require('../../../../helpers/ioredis'); const prefix = config.get('cache:redis:prefix') || ''; const expireSec = config.get('cache:redis:expire_min') * 60; -const client = createClient(); +const client = getClient(); /** * Convert a user key to Redis key with prefix included diff --git a/src/services/registry/strategies/redis/index.js b/src/services/registry/strategies/redis/index.js index 2f742a1..5c0924d 100644 --- a/src/services/registry/strategies/redis/index.js +++ b/src/services/registry/strategies/redis/index.js @@ -1,9 +1,9 @@ const _ = require('lodash'); const config = require('../../../../config'); -const { createClient } = require('../../../../helpers/ioredis'); +const { getClient } = require('../../../../helpers/ioredis'); const prefix = config.get('registry:prefix') || ''; -const client = createClient(); +const client = getClient(); /** * Convert a user key to Redis key with prefix included diff --git a/src/telemetry.js b/src/telemetry.js index 799f32c..a3ce989 100644 --- a/src/telemetry.js +++ b/src/telemetry.js @@ -1,16 +1,21 @@ -const { RateLimiterMemory } = require('rate-limiter-flexible'); +const { RateLimiterRedis } = require('rate-limiter-flexible'); const config = require('./config'); const logger = require('./logger'); const axios = require('./helpers/axios'); +const { getClient } = require('./helpers/ioredis'); +const redisClient = getClient(); const hasTelemetry = config.get('telemetry:enabled'); const telemetryHost = config.get('telemetry:host'); const reqTelemetryTimeoutMsec = config.get('telemetry:req_timeout_sec') * 1000; const maxConcurrentReq = config.get('telemetry:max_concurrent_req'); +const redisKeyPrefix = config.get('telemetry:prefix'); -const rateLimiter = new RateLimiterMemory({ +const rateLimiter = new RateLimiterRedis({ + storeClient: redisClient, points: 2, // points to consume duration: 1, // per seconds + keyPrefix: redisKeyPrefix, }); let concurrentReq = 0;