From a416e358b4c776f71cb9991dabee6acc1b53b23b Mon Sep 17 00:00:00 2001
From: Karl Stoney <karl.stoney@autotrader.co.uk>
Date: Mon, 6 Jul 2020 17:48:34 +0100
Subject: [PATCH] Improved graceful shutdown handling to minimise errors during
 version upgrades

---
 .../kconmon/templates/agent/daemonset.yaml    |  1 +
 .../templates/controller/deployment.yaml      |  3 ++-
 lib/apps/agent/index.ts                       |  6 +++---
 lib/apps/controller/index.ts                  | 19 +++++++++++++------
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/helmfile/charts/kconmon/templates/agent/daemonset.yaml b/helmfile/charts/kconmon/templates/agent/daemonset.yaml
index 7cc251e..b31535a 100644
--- a/helmfile/charts/kconmon/templates/agent/daemonset.yaml
+++ b/helmfile/charts/kconmon/templates/agent/daemonset.yaml
@@ -21,6 +21,7 @@ spec:
         app: kconmon
         component: agent
     spec:
+      terminationGracePeriodSeconds: 15
       dnsConfig:
         options:
         - name: ndots
diff --git a/helmfile/charts/kconmon/templates/controller/deployment.yaml b/helmfile/charts/kconmon/templates/controller/deployment.yaml
index 897139e..f0ad3cf 100644
--- a/helmfile/charts/kconmon/templates/controller/deployment.yaml
+++ b/helmfile/charts/kconmon/templates/controller/deployment.yaml
@@ -25,6 +25,7 @@ spec:
         app: kconmon
         component: controller
     spec:
+      terminationGracePeriodSeconds: 15
       dnsConfig:
         options:
         - name: ndots
@@ -32,7 +33,7 @@ spec:
         searches:
         - {{ .Release.Namespace }}.svc.cluster.local
         - svc.cluster.local
-        - cluster.loca
+        - cluster.local
       serviceAccountName: kconmon
       containers:
       - name: agent
diff --git a/lib/apps/agent/index.ts b/lib/apps/agent/index.ts
index 0eb6e74..38d560b 100644
--- a/lib/apps/agent/index.ts
+++ b/lib/apps/agent/index.ts
@@ -57,12 +57,13 @@ const delay = (ms: number) => {
   }
   await webServer.start(handlerInit)
   await tester.start()
+  logger.info('agent started successfully')
 
   async function shutdown() {
     const shutdownPeriod = 7500
-    logger.info('stopping tester')
+    logger.info(`stopping agent, will exit in ${shutdownPeriod}ms`)
     await tester.stop()
-    logger.info(`shutting down web and udp server in ${shutdownPeriod}ms`)
+    await discovery.stop()
     setTimeout(async () => {
       await udpServer.stop()
       await webServer.stop()
@@ -74,7 +75,6 @@ const delay = (ms: number) => {
 
   process.on('SIGINT', shutdown)
   process.on('SIGTERM', shutdown)
-
   process.on('unhandledRejection', (error) => {
     console.error('Unhandled Rejection!')
     console.error(error)
diff --git a/lib/apps/controller/index.ts b/lib/apps/controller/index.ts
index e1d542e..c7fdc31 100644
--- a/lib/apps/controller/index.ts
+++ b/lib/apps/controller/index.ts
@@ -15,6 +15,7 @@ import IndexController from 'lib/apps/controller/controllers'
 import IndexRoutes from 'lib/apps/controller/routes'
 import config from 'lib/config'
 import Kubernetes from 'lib/kubernetes/client'
+import Logger from 'lib/logger'
 const kubernetes = new Kubernetes()
 const webServer = new WebServer(config)
 const discovery = new KubernetesDiscovery(config, kubernetes)
@@ -24,22 +25,28 @@ const handlerInit = (app: Application): Promise<void> => {
   new IndexRoutes().applyRoutes(app, indexController)
   return Promise.resolve()
 }
-
+const logger = new Logger('controller')
 ;(async () => {
   await discovery.start()
   await webServer.start(handlerInit)
+  logger.log('controller started successfully')
 })()
 
 async function shutdown() {
-  await webServer.stop()
-  setTimeout(() => {
-    process.exit(0)
-  }, 1000)
+  const shutdownPeriod = 7500
+  logger.info(`stopping controller, will exit in ${shutdownPeriod}ms`)
+  setTimeout(async () => {
+    await webServer.stop()
+    await discovery.stop()
+    setTimeout(() => {
+      logger.info('controller stopped')
+      process.exit(0)
+    }, 1000)
+  }, shutdownPeriod)
 }
 
 process.on('SIGINT', shutdown)
 process.on('SIGTERM', shutdown)
-
 process.on('unhandledRejection', (error) => {
   console.error('Unhandled Rejection!')
   console.error(error)