Add Redis in rate limiter

config/defaults.yml

@@ -203,6 +203,9 @@ queue:
 limits:
   # Rate limits for push content (max requests per seconds)
   # https://github.com/nfriedly/express-rate-limit
+  pull:
+    window_sec: 2
+    max_req: 1000
   push:
     window_sec: 60
     max_req: 20
@@ -212,19 +215,27 @@ limits:
   jobs:
     window_sec: 5
     max_req: 120
+  # Prefix namespace for keys in Redis. Modify if you want to separate
+  # data, between multiple environments (e.g. beta, staging) that are using
+  # the same Redis instance, e.g.
+  # TX__LIMITS__PREFIX="beta:ratelimit:"
+  prefix: 'ratelimit:'
+
 
 telemetry:
   # If usage telemetry should be enabled
   enabled: true
-
   # Telemetry service host
   host: https://telemetry.svc.transifex.net
-
   # Request timeout to Telemetry service
   req_timeout_sec: 5
-
   # Max concurrent requests per process
   max_concurrent_req: 10
+  # Prefix namespace for keys in Redis. Modify if you want to separate
+  # data, between multiple environments (e.g. beta, staging) that are using
+  # the same Redis instance, e.g.
+  # TX__TELEMETRY__PREFIX="beta:telemetry:"
+  prefix: 'telemetry:'
 
 # # Manually pass AWS config (local testing)
 # aws:

package.json

@@ -53,6 +53,7 @@
     "newrelic": "^11.6.0",
     "node-cache": "^5.1.2",
     "prom-client": "^15.0.0",
+    "rate-limit-redis": "^4.2.0",
     "rate-limiter-flexible": "^3.0.4",
     "uuid": "^8.3.2",
     "winston": "^3.11.0",

src/helpers/ioredis.js

@@ -1,6 +1,8 @@
 const Redis = require('ioredis');
 const config = require('../config');
 
+let singletonClient;
+
 function createClient() {
   const redisUrl = config.get('redis:host');
   if (redisUrl) {
@@ -21,6 +23,15 @@ function createClient() {
   });
 }
 
+function getClient() {
+  if (singletonClient) {
+    return singletonClient;
+  }
+  singletonClient = createClient();
+  return singletonClient;
+}
+
 module.exports = {
   createClient,
+  getClient,
 };

src/helpers/ratelimit.js

@@ -1,5 +1,9 @@
-const rateLimit = require('express-rate-limit');
+const { rateLimit } = require('express-rate-limit');
+const { default: RedisStore } = require('rate-limit-redis');
 const config = require('../config');
+const { getClient } = require('./ioredis');
+
+const redisClient = getClient();
 
 /**
  * Create a scope based rate limiter based on project_token.
@@ -14,6 +18,7 @@ const config = require('../config');
  * @return {*}
  */
 function createRateLimiter(scope) {
+  const redisKeyPrefix = config.get('limits:prefix');
   const limitPushWindowMsec = config.get(`limits:${scope}:window_sec`) * 1000;
   const limitPushMaxReq = config.get(`limits:${scope}:max_req`) * 1;
   return rateLimit({
@@ -24,6 +29,11 @@ function createRateLimiter(scope) {
       status: 429,
       message: 'Too many requests, please try again later.',
     },
+    // Redis store configuration
+    store: new RedisStore({
+      sendCommand: (...args) => redisClient.call(...args),
+      prefix: `${redisKeyPrefix}${scope}:`,
+    }),
   });
 }
 

src/index.js

@@ -35,6 +35,7 @@ async function launch() {
     const keepAliveTimeoutSec = config.get('settings:keep_alive_timeout_sec');
     if (isNumber(keepAliveTimeoutSec) && keepAliveTimeoutSec >= 0) {
       attachedApplication.keepAliveTimeout = keepAliveTimeoutSec * 1000;
+      attachedApplication.headersTimeout = (keepAliveTimeoutSec + 1) * 1000;
     }
 
     // graceful shutdown of server

src/queue/index.js

@@ -5,6 +5,7 @@ const logger = require('../logger');
 const config = require('../config');
 const worker = require('./worker');
 
+// Always create a new Redis client
 const queue = new Queue(config.get('queue:name'), {
   createClient: () => createClient(),
 });

src/routes/content.js

@@ -67,6 +67,7 @@ async function getContent(req, res) {
 router.get(
   '/:lang_code',
   validateHeader('public'),
+  createRateLimiter('pull'),
   getContent,
 );
 

src/routes/languages.js

@@ -1,12 +1,14 @@
 const express = require('express');
 const { validateHeader } = require('../middlewares/headers');
 const utils = require('../helpers/utils');
+const { createRateLimiter } = require('../helpers/ratelimit');
 
 const router = express.Router();
 
 router.get(
   '/',
   validateHeader('public'),
+  createRateLimiter('pull'),
   async (req, res) => {
     const filter = req.query.filter || {};
     const key = `${req.token.project_token}:languages`;

src/server.js

@@ -10,7 +10,6 @@ const logger = require('./logger');
 const metrics = require('./middlewares/metrics');
 const languagesRouter = require('./routes/languages');
 const contentRouter = require('./routes/content');
-const statusRouter = require('./routes/status');
 const invalidateRouter = require('./routes/invalidate');
 const purgeRouter = require('./routes/purge');
 const jobsRouter = require('./routes/jobs');
@@ -67,12 +66,11 @@ module.exports = () => {
 
   app.use('/languages', languagesRouter);
   app.use('/content', contentRouter);
-  app.use('/status', statusRouter);
   app.use('/invalidate', invalidateRouter);
   app.use('/purge', purgeRouter);
   app.use('/jobs', jobsRouter);
 
-  app.get('/', (req, res) => res.send('ok'));
+  app.get('/', (req, res) => res.send(`Transifex CDS - v${version}`));
 
   // The error handler must be before any other error middleware
   sentry.expressError(app);

src/services/cache/strategies/redis/index.js

@@ -1,10 +1,10 @@
 const config = require('../../../../config');
 const logger = require('../../../../logger');
-const { createClient } = require('../../../../helpers/ioredis');
+const { getClient } = require('../../../../helpers/ioredis');
 
 const prefix = config.get('cache:redis:prefix') || '';
 const expireSec = config.get('cache:redis:expire_min') * 60;
-const client = createClient();
+const client = getClient();
 
 /**
  * Convert a user key to Redis key with prefix included

src/services/registry/strategies/redis/index.js

@@ -1,9 +1,9 @@
 const _ = require('lodash');
 const config = require('../../../../config');
-const { createClient } = require('../../../../helpers/ioredis');
+const { getClient } = require('../../../../helpers/ioredis');
 
 const prefix = config.get('registry:prefix') || '';
-const client = createClient();
+const client = getClient();
 
 /**
  * Convert a user key to Redis key with prefix included

src/telemetry.js

@@ -1,16 +1,21 @@
-const { RateLimiterMemory } = require('rate-limiter-flexible');
+const { RateLimiterRedis } = require('rate-limiter-flexible');
 const config = require('./config');
 const logger = require('./logger');
 const axios = require('./helpers/axios');
+const { getClient } = require('./helpers/ioredis');
 
+const redisClient = getClient();
 const hasTelemetry = config.get('telemetry:enabled');
 const telemetryHost = config.get('telemetry:host');
 const reqTelemetryTimeoutMsec = config.get('telemetry:req_timeout_sec') * 1000;
 const maxConcurrentReq = config.get('telemetry:max_concurrent_req');
+const redisKeyPrefix = config.get('telemetry:prefix');
 
-const rateLimiter = new RateLimiterMemory({
+const rateLimiter = new RateLimiterRedis({
+  storeClient: redisClient,
   points: 2, // points to consume
   duration: 1, // per seconds
+  keyPrefix: redisKeyPrefix,
 });
 
 let concurrentReq = 0;