Fix: Send a SIGKILL to container when its really time to stop (#726)

When the goroutine in clearContainer() is unblocked by the select (grace
period has expired or worker receives a SIGTERM), then its really time
to stop the container. In this case, we're going to send a SIGKILL to it
instead of a SIGTERM.

This might fix the issue related to unmounting images.

pkg/worker/image.go

@@ -207,6 +207,7 @@ func (c *ImageClient) Cleanup() error {
 		return true // Continue iteration
 	})
 
+	log.Println("Cleaning up blobfs image cache:", c.imageCachePath)
 	if c.config.BlobCache.BlobFs.Enabled && c.cacheClient != nil {
 		err := c.cacheClient.Cleanup()
 		if err != nil {

pkg/worker/lifecycle.go

@@ -71,14 +71,9 @@ func (s *Worker) stopContainer(containerId string, kill bool) error {
 
 	err := s.runcHandle.Kill(context.Background(), containerId, signal, &runc.KillOpts{All: true})
 	if err != nil {
-		log.Printf("<%s> - unable to stop container: %v\n", containerId, err)
-
-		if strings.Contains(err.Error(), "container does not exist") {
-			s.containerNetworkManager.TearDown(containerId)
-			return nil
-		}
-
-		return err
+		log.Printf("<%s> - error stopping container: %v\n", containerId, err)
+		s.containerNetworkManager.TearDown(containerId)
+		return nil
 	}
 
 	log.Printf("<%s> - container stopped.\n", containerId)
@@ -137,7 +132,7 @@ func (s *Worker) clearContainer(containerId string, request *types.ContainerRequ
 		// If the container is still running, stop it. This happens when a sigterm is detected.
 		container, err := s.runcHandle.State(context.TODO(), containerId)
 		if err == nil && container.Status == types.RuncContainerStatusRunning {
-			if err := s.stopContainer(containerId, false); err != nil {
+			if err := s.stopContainer(containerId, true); err != nil {
 				log.Printf("<%s> - failed to stop container: %v\n", containerId, err)
 			}
 		}

pkg/worker/worker.go

@@ -361,7 +361,6 @@ func (s *Worker) processStopContainerEvents() {
 		default:
 			err := s.stopContainer(event.ContainerId, event.Kill)
 			if err != nil {
-				s.stopContainerChan <- event
 				time.Sleep(time.Second)
 			}
 		}