Merge pull request #264 from rstudio/joe/feature/more-timeouts

More control over http keepalive and sockjs timeouts

NEWS

@@ -1,3 +1,11 @@
+shiny-server 1.5.2
+--------------------------------------------------------------------------------
+
+* Add additional configuration directives `http_keepalive_timeout`,
+  `sockjs_heartbeat_delay`, and `sockjs_disconnect_delay` to allow working
+  with very slow connections and large SockJS payloads. (The default values
+  for these options are the same as in previous versions of Shiny Server.)
+
 shiny-server 1.5.1
 --------------------------------------------------------------------------------
 

config/shiny-server-rules.config

@@ -141,6 +141,27 @@ app_idle_timeout {
   maxcount 1;
 }
 
+http_keepalive_timeout {
+  desc "Defines how long a keepalive connection will sit between HTTP requests/responses before it is closed. Defaults to 45 seconds.";
+  param Float timeout "The number of seconds to keep a connection alive between requests/responses.";
+  at $;
+  maxcount 1;
+}
+
+sockjs_heartbeat_delay {
+  desc "How often the SockJS server should send heartbeat packets to the server. These are used to prevent proxies and load balancers from closing active SockJS connections. Defaults to 25 seconds.";
+  param Float delay "The number of seconds to wait between heartbeat packets.";
+  at $;
+  maxcount 1;
+}
+
+sockjs_disconnect_delay {
+  desc "How long the SockJS server should wait between HTTP requests before considering the client to be disconnected. Defaults to 5 seconds. If this value needs to be adjusted above 10 seconds, it's a good idea to disable websockets using the `disable_websockets` directive, as that transport protocol has an effective 10 second limit built in.";
+  param Float delay "The number of seconds to wait before giving up.";
+  at $;
+  maxcount 1;
+}
+
 simple_scheduler {
   desc "A basic scheduler which will spawn one single-threaded R worker for each application. If no scheduler is specified, this is the default scheduler.";
   param Integer [maxRequests] "The maximum number of requests to assign to this scheduler before it should start returning rejecting incoming traffic using a '503 - Service Unavailable' message. Once this threshold is hit, users attempting to initialize a new session will receive 503 errors." 100;

lib/main.js

@@ -178,6 +178,8 @@ app.use(connect_util.filterByRegex(
 ));
 app.use(shinyProxy.httpListener);
 
+var socketTimeout = 45 * 1000;
+
 // Now create a server and hook everything up.
 var server = new Server();
 server.on('connection', function(socket) {
@@ -186,7 +188,14 @@ server.on('connection', function(socket) {
   // SockJS sends a heartbeat every 25s so as long as we wait significantly
   // longer than that to timeout, we shouldn't need to worry about closing
   // active connections.
-  socket.setTimeout(45 * 1000);
+  //
+  // jcheng 11/17/2016: This doesn't work as well as you'd think. The timeout
+  // timer starts at e.g. the last invocation of write(), not waiting for
+  // that write to actually complete. In other words, there can be actual
+  // activity happening over the socket and yet the timeout can be hit. It's
+  // unclear whether the Node maintainers consider this a bug or not. See
+  // PR @rstudio/shiny-server#264 for all the gory details.
+  socket.setTimeout(socketTimeout);
 });
 server.on('request', _.bind(app.handle, app));
 server.on('error', function(err) {
@@ -226,9 +235,12 @@ var loadConfig_p = qutil.serialized(function() {
     transport.setSocketDir(configRouter.socketDir);
 
     // Create SockJS server
-    sockjsServer = proxy_sockjs.createServer(metarouter, schedulerRegistry);
+    sockjsServer = proxy_sockjs.createServer(metarouter, schedulerRegistry,
+      configRouter.sockjsHeartbeatDelay, configRouter.sockjsDisconnectDelay);
     sockjsHandler = sockjsServer.middleware();
 
+    socketTimeout = configRouter.httpKeepaliveTimeout;
+
     return createLogger_p(configRouter.accessLogSpec)
     .then(function(logfunc) {
       requestLogger = logfunc;

lib/proxy/sockjs.js

@@ -22,7 +22,16 @@ var RobustSockJS = require('./robust-sockjs');
 var errorcode = require("./errorcode");
 
 exports.createServer = createServer;
-function createServer(router, schedulerRegistry) {
+function createServer(router, schedulerRegistry, heartbeatDelay, disconnectDelay) {
+  if (!heartbeatDelay || heartbeatDelay < 0) {
+    logger.warn("Ignoring invalid SockJS heartbeat delay: " + heartbeatDelay);
+    heartbeatDelay = 25 * 1000;
+  }
+  if (!disconnectDelay || disconnectDelay < 0) {
+    logger.warn("Ignoring invalid SockJS disconnect delay: " + disconnectDelay);
+    disconnectDelay = 5 * 1000;
+  }
+
   // Create a single SockJS server that will serve all applications. We'll use
   // the connection.url to dispatch among the different worker processes'
   // websocket ports. Once a connection is established, we simply pipe IO
@@ -31,7 +40,9 @@ function createServer(router, schedulerRegistry) {
     // TODO: make URL configurable
     sockjs_url: '//d1fxtkz8shb9d2.cloudfront.net/sockjs-0.3.min.js',
     prefix: '.*/__sockjs__(/[no]=\\w+)?',
-		log: function() {}
+    log: function() {},
+    heartbeat_delay: heartbeatDelay,
+    disconnect_delay: disconnectDelay
   });
 
   var robust = new RobustSockJS();

lib/router/config-router.js

@@ -108,6 +108,21 @@ function ConfigRouter(conf, schedulerRegistry) {
   this.$allowAppOverride = conf.getValues('allow_app_override').enabled;
   this.$templateDir = conf.getValues('template_dir').dir;
 
+  this.httpKeepaliveTimeout = 45 * 1000;
+  if (conf.getOne('http_keepalive_timeout')) {
+    this.httpKeepaliveTimeout = conf.getValues('http_keepalive_timeout').timeout * 1000;
+  }
+
+  this.sockjsHeartbeatDelay = 25 * 1000;
+  if (conf.getOne('sockjs_heartbeat_delay')) {
+    this.sockjsHeartbeatDelay = conf.getValues('sockjs_heartbeat_delay').delay * 1000;
+  }
+
+  this.sockjsDisconnectDelay = 5 * 1000;
+  if (conf.getOne('sockjs_disconnect_delay')) {
+    this.sockjsDisconnectDelay = conf.getValues('sockjs_disconnect_delay').delay * 1000;
+  }
+
   var apps = conf.search("application", true);
   if (apps && apps.length > 0){
     logger.error("The `application` configuration has been deprecated. Please "+