From 82d4c7d03ccc1cbd97e7b7a75e514af6705be8ec Mon Sep 17 00:00:00 2001 From: Jacob Greenleaf Date: Thu, 10 Sep 2015 20:51:19 +0000 Subject: [PATCH 1/4] Add configuration parameter to disable longpoll (aka big red button) --- config.go | 1 + config.yml | 2 ++ incus/main.go | 6 +++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index 2c0fc12..7756a13 100644 --- a/config.go +++ b/config.go @@ -27,6 +27,7 @@ func NewConfig(configFilePath string) { ConfigOption("datadog_host", "127.0.0.1") } + ConfigOption("longpoll_enabled", true) ConfigOption("redis_enabled", false) if viper.GetBool("redis_enabled") { diff --git a/config.yml b/config.yml index 4f7e668..5d52a19 100644 --- a/config.yml +++ b/config.yml @@ -16,6 +16,8 @@ datadog_enabled: false # Datadog host if datadog enabled datadog_host: "127.0.0.1" +longpoll_enabled: true + # ----- Redis Support ----- # Bool; Redis must be enabled if running Incus in a cluster. diff --git a/incus/main.go b/incus/main.go index 2fbfd9d..6d91109 100644 --- a/incus/main.go +++ b/incus/main.go @@ -75,7 +75,11 @@ func main() { go server.LogConnectedClientsPeriodically(20 * time.Second) go server.ListenFromRedis() go server.ListenFromSockets() - go server.ListenFromLongpoll() + + if viper.GetBool("longpoll_enabled") { + go server.ListenFromLongpoll() + } + go server.ListenForHTTPPings() go server.SendHeartbeatsPeriodically(20 * time.Second) From f357b8a8fe9c7f2ae3aebe5fecce46b3548b0540 Mon Sep 17 00:00:00 2001 From: Jacob Greenleaf Date: Thu, 10 Sep 2015 21:48:12 +0000 Subject: [PATCH 2/4] fix: Use redis ttl as a kill switch rather than configuration value --- config.go | 2 +- config.yml | 3 ++- incus/main.go | 6 ++---- server.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/config.go b/config.go index 7756a13..eba16cd 100644 --- a/config.go +++ b/config.go @@ -27,7 +27,7 @@ func NewConfig(configFilePath string) { ConfigOption("datadog_host", "127.0.0.1") } - ConfigOption("longpoll_enabled", true) + ConfigOption("longpoll_killswitch", "longpoll_killswitch") ConfigOption("redis_enabled", false) if viper.GetBool("redis_enabled") { diff --git a/config.yml b/config.yml index 5d52a19..c56e378 100644 --- a/config.yml +++ b/config.yml @@ -16,7 +16,8 @@ datadog_enabled: false # Datadog host if datadog enabled datadog_host: "127.0.0.1" -longpoll_enabled: true +# Redis key to monitor. If it exists, longpolling is disabled, for performance reasons. +longpoll_killswitch: "longpoll_killswitch" # ----- Redis Support ----- diff --git a/incus/main.go b/incus/main.go index 6d91109..f42f3bc 100644 --- a/incus/main.go +++ b/incus/main.go @@ -75,10 +75,8 @@ func main() { go server.LogConnectedClientsPeriodically(20 * time.Second) go server.ListenFromRedis() go server.ListenFromSockets() - - if viper.GetBool("longpoll_enabled") { - go server.ListenFromLongpoll() - } + go server.ListenFromLongpoll() + go server.MonitorLongpollKillswitch() go server.ListenForHTTPPings() go server.SendHeartbeatsPeriodically(20 * time.Second) diff --git a/server.go b/server.go index f28f236..1070c72 100644 --- a/server.go +++ b/server.go @@ -4,11 +4,13 @@ import ( "crypto/md5" "encoding/json" "fmt" + "github.com/garyburd/redigo/redis" "io" "log" "net/http" "os" "os/signal" + "sync/atomic" "syscall" "time" @@ -31,6 +33,10 @@ const ( closeCodeUnexpectedError = 1011 ) +var ( + disableLongpoll atomic.Value +) + type GCMClient interface { Send(*gcm.Message, int) (*gcm.Response, error) } @@ -168,6 +174,13 @@ func (this *Server) ListenFromLongpoll() { w.Header().Set("Connection", "keep-alive") //w.Header().Set("Content-Encoding", "gzip") + longpollIsDisabled := disableLongpoll.Load() + if longpollIsDisabled != nil && longpollIsDisabled.(bool) == true { + sock.Close() + w.WriteHeader(503) + return + } + if DEBUG { log.Printf("Long poll connected via \n") } @@ -311,3 +324,27 @@ func (this *Server) GetAPNSClient(build string) apns.APNSClient { func (this *Server) GetGCMClient() GCMClient { return this.gcmProvider() } + +func (this *Server) MonitorLongpollKillswitch() { + if !viper.GetBool("redis_enabled") { + return + } + + killswitchKey := viper.Get("longpoll_killswitch") + + for { + result := this.Store.redis.redisPendingQueue.RunAsyncTimeout(5*time.Second, func(conn redis.Conn) (result interface{}, err error) { + return conn.Do("TTL", killswitchKey) + }) + + if result.Error == nil { + if result.Value.(int64) >= -1 { + disableLongpoll.Store(true) + } else { + disableLongpoll.Store(false) + } + } + + time.Sleep(5 * time.Second) + } +} From d11234265485ff46398e5fedea7ea60fd092702d Mon Sep 17 00:00:00 2001 From: Jacob Greenleaf Date: Thu, 10 Sep 2015 22:11:19 +0000 Subject: [PATCH 3/4] fix: Refactor killswitch to be internal to redis store --- redis_store.go | 35 +++++++++++++++++++++++++++++++++++ redis_store_test.go | 37 +++++++++++++++++++++++++++++++++++++ server.go | 17 ++++------------- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/redis_store.go b/redis_store.go index 8a03610..a46b09c 100644 --- a/redis_store.go +++ b/redis_store.go @@ -7,6 +7,7 @@ import ( "time" "github.com/garyburd/redigo/redis" + "github.com/spf13/viper" ) const ClientsKey = "SocketClients" @@ -229,6 +230,40 @@ func (this *RedisStore) QueryIsUserActive(user string, nowTimestamp int64) (bool } } +func (this *RedisStore) GetIsLongpollKillswitchActive() (bool, error) { + killswitchKey := viper.Get("longpoll_killswitch") + + result := this.redisPendingQueue.RunAsyncTimeout(5*time.Second, func(conn redis.Conn) (result interface{}, err error) { + return conn.Do("TTL", killswitchKey) + }) + + if result.Error == nil { + if result.Value.(int64) >= -1 { + return true, nil + } else { + return false, nil + } + } + + return false, timedOut +} + +func (this *RedisStore) ActivateLongpollKillswitch(seconds int64) error { + killswitchKey := viper.Get("longpoll_killswitch") + + return this.redisPendingQueue.RunAsyncTimeout(5*time.Second, func(conn redis.Conn) (result interface{}, err error) { + return conn.Do("SETEX", killswitchKey, seconds, "1") + }).Error +} + +func (this *RedisStore) DeactivateLongpollKillswitch() error { + killswitchKey := viper.Get("longpoll_killswitch") + + return this.redisPendingQueue.RunAsyncTimeout(5*time.Second, func(conn redis.Conn) (result interface{}, err error) { + return conn.Do("DEL", killswitchKey) + }).Error +} + func (this *RedisStore) Publish(channel string, message string) { publisher, err := this.GetConn() if err != nil { diff --git a/redis_store_test.go b/redis_store_test.go index 4ddd111..9d3eee8 100644 --- a/redis_store_test.go +++ b/redis_store_test.go @@ -254,3 +254,40 @@ func TestUserPresenceIsImmediatelyRemovedUponMarkingInactive(t *testing.T) { t.Fatalf("Expected 'bazbar' to be inactive after affirmatively marking as inactive") } } + +func TestKillswitch(t *testing.T) { + store := newTestRedisStore() + + store.DeactivateLongpollKillswitch() + + active, err := store.GetIsLongpollKillswitchActive() + + if err != nil { + t.Fatalf("Unexpected error: %s", err.Error()) + } + if active { + t.Fatalf("Expected precondition that killswitch is inactive") + } + + store.ActivateLongpollKillswitch(3) + + active, err = store.GetIsLongpollKillswitchActive() + + if err != nil { + t.Fatalf("Unexpected error: %s", err.Error()) + } + if !active { + t.Fatalf("Expected killswitch to be active") + } + + time.Sleep(4 * time.Second) + + active, err = store.GetIsLongpollKillswitchActive() + + if err != nil { + t.Fatalf("Unexpected error: %s", err.Error()) + } + if active { + t.Fatalf("Expected killswitch to be inactive") + } +} diff --git a/server.go b/server.go index 1070c72..d8273d2 100644 --- a/server.go +++ b/server.go @@ -4,7 +4,6 @@ import ( "crypto/md5" "encoding/json" "fmt" - "github.com/garyburd/redigo/redis" "io" "log" "net/http" @@ -330,19 +329,11 @@ func (this *Server) MonitorLongpollKillswitch() { return } - killswitchKey := viper.Get("longpoll_killswitch") - for { - result := this.Store.redis.redisPendingQueue.RunAsyncTimeout(5*time.Second, func(conn redis.Conn) (result interface{}, err error) { - return conn.Do("TTL", killswitchKey) - }) - - if result.Error == nil { - if result.Value.(int64) >= -1 { - disableLongpoll.Store(true) - } else { - disableLongpoll.Store(false) - } + longpollSwitchedOff, err := this.Store.redis.GetIsLongpollKillswitchActive() + + if err == nil { + disableLongpoll.Store(longpollSwitchedOff) } time.Sleep(5 * time.Second) From fc8bf1968038c4163bd77f91fe0040d52878dc45 Mon Sep 17 00:00:00 2001 From: Jacob Greenleaf Date: Thu, 10 Sep 2015 22:45:11 +0000 Subject: [PATCH 4/4] fix: Don't create socket channels for connections that are just gonna be closed --- server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index d8273d2..1e27b84 100644 --- a/server.go +++ b/server.go @@ -166,7 +166,6 @@ func (this *Server) ListenFromLongpoll() { } }() - sock := newSocket(nil, w, this, "") w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "private, no-store, no-cache, must-revalidate, post-check=0, pre-check=0") @@ -175,11 +174,13 @@ func (this *Server) ListenFromLongpoll() { longpollIsDisabled := disableLongpoll.Load() if longpollIsDisabled != nil && longpollIsDisabled.(bool) == true { - sock.Close() + w.Header().Set("Connection", "close") w.WriteHeader(503) return } + sock := newSocket(nil, w, this, "") + if DEBUG { log.Printf("Long poll connected via \n") }