Improved graceful shutdown handling to minimise errors during version…

… upgrades

helmfile/charts/kconmon/templates/agent/daemonset.yaml

@@ -21,6 +21,7 @@ spec:
         app: kconmon
         component: agent
     spec:
+      terminationGracePeriodSeconds: 15
       dnsConfig:
         options:
         - name: ndots

helmfile/charts/kconmon/templates/controller/deployment.yaml

@@ -25,14 +25,15 @@ spec:
         app: kconmon
         component: controller
     spec:
+      terminationGracePeriodSeconds: 15
       dnsConfig:
         options:
         - name: ndots
           value: "1"
         searches:
         - {{ .Release.Namespace }}.svc.cluster.local
         - svc.cluster.local
-        - cluster.loca
+        - cluster.local
       serviceAccountName: kconmon
       containers:
       - name: agent

lib/apps/agent/index.ts

@@ -57,12 +57,13 @@ const delay = (ms: number) => {
   }
   await webServer.start(handlerInit)
   await tester.start()
+  logger.info('agent started successfully')
 
   async function shutdown() {
     const shutdownPeriod = 7500
-    logger.info('stopping tester')
+    logger.info(`stopping agent, will exit in ${shutdownPeriod}ms`)
     await tester.stop()
-    logger.info(`shutting down web and udp server in ${shutdownPeriod}ms`)
+    await discovery.stop()
     setTimeout(async () => {
       await udpServer.stop()
       await webServer.stop()
@@ -74,7 +75,6 @@ const delay = (ms: number) => {
 
   process.on('SIGINT', shutdown)
   process.on('SIGTERM', shutdown)
-
   process.on('unhandledRejection', (error) => {
     console.error('Unhandled Rejection!')
     console.error(error)

lib/apps/controller/index.ts

@@ -15,6 +15,7 @@ import IndexController from 'lib/apps/controller/controllers'
 import IndexRoutes from 'lib/apps/controller/routes'
 import config from 'lib/config'
 import Kubernetes from 'lib/kubernetes/client'
+import Logger from 'lib/logger'
 const kubernetes = new Kubernetes()
 const webServer = new WebServer(config)
 const discovery = new KubernetesDiscovery(config, kubernetes)
@@ -24,22 +25,28 @@ const handlerInit = (app: Application): Promise<void> => {
   new IndexRoutes().applyRoutes(app, indexController)
   return Promise.resolve()
 }
-
+const logger = new Logger('controller')
 ;(async () => {
   await discovery.start()
   await webServer.start(handlerInit)
+  logger.log('controller started successfully')
 })()
 
 async function shutdown() {
-  await webServer.stop()
-  setTimeout(() => {
-    process.exit(0)
-  }, 1000)
+  const shutdownPeriod = 7500
+  logger.info(`stopping controller, will exit in ${shutdownPeriod}ms`)
+  setTimeout(async () => {
+    await webServer.stop()
+    await discovery.stop()
+    setTimeout(() => {
+      logger.info('controller stopped')
+      process.exit(0)
+    }, 1000)
+  }, shutdownPeriod)
 }
 
 process.on('SIGINT', shutdown)
 process.on('SIGTERM', shutdown)
-
 process.on('unhandledRejection', (error) => {
   console.error('Unhandled Rejection!')
   console.error(error)