Terminate instance when not found.

jenkinsci · Nov 28, 2024 · e55da55 · e55da55
1 parent 5ba199e
commit e55da55
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 3 deletions.
diff --git a/src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineComputerLauncher.java b/src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineComputerLauncher.java
@@ -16,6 +16,7 @@
 
 package com.google.jenkins.plugins.computeengine;
 
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.services.compute.model.AccessConfig;
 import com.google.api.services.compute.model.Instance;
 import com.google.api.services.compute.model.NetworkInterface;
@@ -40,6 +41,7 @@
 import java.io.PrintStream;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
+import java.net.SocketTimeoutException;
 import java.util.Base64;
 import java.util.Optional;
 import java.util.logging.Level;
@@ -349,6 +351,10 @@ protected Connection connectToSsh(ComputeEngineComputer computer, TaskListener l
                             + ")");
                 }
                 Instance instance = computer.refreshInstance();
+                // the instance will be null when the node is terminated
+                if (instance == null) {
+                    return null;
+                }
 
                 String host = "";
 
@@ -416,11 +422,21 @@ protected Connection connectToSsh(ComputeEngineComputer computer, TaskListener l
                         SSH_TIMEOUT_MILLIS);
                 logInfo(computer, listener, "Connected via SSH.");
                 return conn;
-            } catch (IOException e) {
+            } catch (GoogleJsonResponseException e) {
+                if (e.getStatusCode() == 404) {
+                    log(LOGGER, Level.SEVERE, listener, String.format("Instance not found. Terminating instance."));
+                    terminateNode(computer, listener);
+                }
+            } catch (SocketTimeoutException e) {
                 // keep retrying until SSH comes up
-                logInfo(computer, listener, "Failed to connect via ssh: " + e.getMessage());
-                logInfo(computer, listener, "Waiting for SSH to come up. Sleeping 5.");
+                logInfo(computer, listener, String.format("Failed to connect via ssh: %s", e.getMessage()));
+                logInfo(
+                        computer,
+                        listener,
+                        String.format("Waiting for SSH to come up. Sleeping %n.", SSH_SLEEP_MILLIS / 1000));
                 Thread.sleep(SSH_SLEEP_MILLIS);
+            } catch (IOException e) {
+                logWarning(computer, listener, String.format("An error occured: %s", e.getMessage()));
             }
         }
     }

diff --git a/src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineLinuxLauncher.java b/src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineLinuxLauncher.java
@@ -85,6 +85,9 @@ private Optional<Connection> bootstrap(
                 logInfo(computer, listener, "Authenticating as " + node.getSshUser());
                 try {
                     bootstrapConn = connectToSsh(computer, listener);
+                    if (bootstrapConn == null) {
+                        break;
+                    }
                     isAuthenticated = bootstrapConn.authenticateWithPublicKey(
                             node.getSshUser(),
                             Secret.toString(keyCred.getPrivateKey()).toCharArray(),

diff --git a/src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineWindowsLauncher.java b/src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineWindowsLauncher.java
@@ -92,6 +92,9 @@ private Optional<Connection> bootstrap(ComputeEngineComputer computer, TaskListe
                 logInfo(computer, listener, "Authenticating as " + node.getSshUser());
                 try {
                     bootstrapConn = connectToSsh(computer, listener);
+                    if (bootstrapConn == null) {
+                        break;
+                    }
                     isAuthenticated = authenticateSSH(node.getSshUser(), windowsConfig, bootstrapConn, listener);
                 } catch (IOException e) {
                     logException(computer, listener, "Exception trying to authenticate", e);