Introduce a ProvisionerStrategy to provision a node without delay (je…

…nkinsci#183)

* Introduce a ProvisionerStrategy to provision a node without delay
* Add ITs for the NoDelayProvisionerStrategy
* fix dependencies required when using the jenkins-pipeline library as the LTS has been bumped

docs/Home.md

@@ -110,3 +110,22 @@ Instance configurations have many options that were not listed above. A few of t
   access from metadata. For more info, review the service account documentation.
 
 
+# No delay provisioning
+
+By default Jenkins estimates load to avoid over-provisioning of cloud nodes.
+This plugin will use its own provisioning strategy by default, with this strategy, a new node is created on GCP as soon as NodeProvisioner detects need for more agents.
+In worst case scenarios, this will results in some extra nodes provisioned on GCP, which will be shortly terminated.
+
+## How to configure
+
+### Using a system property
+
+If you want to turn off this Strategy globally then you can set a SystemProperty `com.google.jenkins.plugins.computeengine.disableNoDelayProvisioning=true`
+
+### Using the the UI
+
+Follow the steps below to configure it:
+
+ 1. Go to Manage Jenkins, then Configure System
+ 2. At the bottom of the page there will be a Cloud Section
+ 3. Select the cloud project and look for the `No delay provisioning` checkbox, and click on to enable it. 

pom.xml

@@ -98,6 +98,27 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <!-- Require upper bound dependencies -->
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>jcl-over-slf4j</artifactId>
+        <version>1.7.26</version>
+      </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>log4j-over-slf4j</artifactId>
+        <version>1.7.26</version>
+      </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-api</artifactId>
+        <version>1.7.26</version>
+      </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-jdk14</artifactId>
+        <version>1.7.26</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   <dependencies>

src/main/java/com/google/jenkins/plugins/computeengine/ComputeEngineCloud.java

@@ -91,6 +91,7 @@ public class ComputeEngineCloud extends AbstractCloudImpl {
   private List<InstanceConfiguration> configurations;
 
   private transient volatile ComputeClient client;
+  private boolean noDelayProvisioning;
 
   @DataBoundConstructor
   public ComputeEngineCloud(
@@ -143,6 +144,15 @@ public String getDisplayName() {
     return getCloudName();
   }
 
+  public boolean isNoDelayProvisioning() {
+    return noDelayProvisioning;
+  }
+
+  @DataBoundSetter
+  public void setNoDelayProvisioning(boolean noDelayProvisioning) {
+    this.noDelayProvisioning = noDelayProvisioning;
+  }
+
   protected Object readResolve() {
     if (configurations != null) {
       for (InstanceConfiguration configuration : configurations) {

src/main/java/com/google/jenkins/plugins/computeengine/NoDelayProvisionerStrategy.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 Elastic, and a number of other of contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.jenkins.plugins.computeengine;
+
+import hudson.Extension;
+import hudson.model.Label;
+import hudson.model.LoadStatistics;
+import hudson.slaves.Cloud;
+import hudson.slaves.CloudProvisioningListener;
+import hudson.slaves.NodeProvisioner;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import jenkins.model.Jenkins;
+
+/**
+ * Implementation of {@link NodeProvisioner.Strategy} which will provision a new node immediately as
+ * a task enter the queue.
+ */
+@Extension(ordinal = 100)
+public class NoDelayProvisionerStrategy extends NodeProvisioner.Strategy {
+  private static final Logger LOGGER = Logger.getLogger(NoDelayProvisionerStrategy.class.getName());
+
+  private static final boolean DISABLE_NODELAY_PROVISING =
+      Boolean.valueOf(
+          System.getProperty(
+              "com.google.jenkins.plugins.computeengine.disableNoDelayProvisioning"));
+
+  /** {@inheritDoc} */
+  @Override
+  public NodeProvisioner.StrategyDecision apply(NodeProvisioner.StrategyState strategyState) {
+    if (DISABLE_NODELAY_PROVISING) {
+      LOGGER.log(Level.FINE, "Provisioning not complete, NoDelayProvisionerStrategy is disabled");
+      return NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES;
+    }
+    final Label label = strategyState.getLabel();
+
+    LoadStatistics.LoadStatisticsSnapshot snapshot = strategyState.getSnapshot();
+    int availableCapacity =
+        snapshot.getAvailableExecutors() // live executors
+            + snapshot.getConnectingExecutors() // executors present but not yet connected
+            + strategyState
+                .getPlannedCapacitySnapshot() // capacity added by previous strategies from previous
+            // rounds
+            + strategyState
+                .getAdditionalPlannedCapacity(); // capacity added by previous strategies _this
+    // round_
+    int currentDemand = snapshot.getQueueLength();
+    LOGGER.log(
+        Level.FINE,
+        "Available capacity={0}, currentDemand={1}",
+        new Object[] {availableCapacity, currentDemand});
+    if (availableCapacity < currentDemand) {
+      List<Cloud> jenkinsClouds = new ArrayList<>(Jenkins.get().clouds);
+      Collections.shuffle(jenkinsClouds);
+      for (Cloud cloud : jenkinsClouds) {
+        int workloadToProvision = currentDemand - availableCapacity;
+        if (!(cloud instanceof ComputeEngineCloud)) continue;
+        if (!cloud.canProvision(label)) continue;
+        ComputeEngineCloud gcp = (ComputeEngineCloud) cloud;
+        if (!gcp.isNoDelayProvisioning()) continue;
+        for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
+          if (cl.canProvision(cloud, strategyState.getLabel(), workloadToProvision) != null) {
+            continue;
+          }
+        }
+        Collection<NodeProvisioner.PlannedNode> plannedNodes =
+            cloud.provision(label, workloadToProvision);
+        LOGGER.log(Level.FINE, "Planned {0} new nodes", plannedNodes.size());
+        fireOnStarted(cloud, strategyState.getLabel(), plannedNodes);
+        strategyState.recordPendingLaunches(plannedNodes);
+        availableCapacity += plannedNodes.size();
+        LOGGER.log(
+            Level.FINE,
+            "After provisioning, available capacity={0}, currentDemand={1}",
+            new Object[] {availableCapacity, currentDemand});
+        break;
+      }
+    }
+    if (availableCapacity >= currentDemand) {
+      LOGGER.log(Level.FINE, "Provisioning completed");
+      return NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED;
+    } else {
+      LOGGER.log(Level.FINE, "Provisioning not complete, consulting remaining strategies");
+      return NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES;
+    }
+  }
+
+  private static void fireOnStarted(
+      final Cloud cloud,
+      final Label label,
+      final Collection<NodeProvisioner.PlannedNode> plannedNodes) {
+    for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
+      try {
+        cl.onStarted(cloud, label, plannedNodes);
+      } catch (Error e) {
+        throw e;
+      } catch (Throwable e) {
+        LOGGER.log(
+            Level.SEVERE,
+            "Unexpected uncaught exception encountered while "
+                + "processing onStarted() listener call in "
+                + cl
+                + " for label "
+                + label.toString(),
+            e);
+      }
+    }
+  }
+}

src/main/resources/com/google/jenkins/plugins/computeengine/ComputeEngineCloud/config.jelly

@@ -29,6 +29,9 @@
     <f:entry field="credentialsId" title="${%Service Account Credentials}">
         <c:select checkMethod="post" />
     </f:entry>
+    <f:entry title="${%No delay provisioning}" field="noDelayProvisioning">
+      <f:checkbox/>
+    </f:entry>
     <f:entry title="${%Instance Configurations}"
              description="${%List of instance configurations that can be launched as Jenkins agents}">
         <f:repeatable field="configurations">

src/main/resources/com/google/jenkins/plugins/computeengine/ComputeEngineCloud/help-noDelayProvisioning.html

@@ -0,0 +1,16 @@
+<!--
+ Copyright 2020 Elastic
+
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
+ compliance with the License. You may obtain a copy of the License at
+
+        https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied. See the License for the specific language governing permissions and limitations under the
+ License.
+-->
+By default Jenkins estimates load to avoid over-provisioning of cloud nodes.
+With this option enabled, a new node is created on GCP as soon as NodeProvisioner detects need for more agents.
+In worst case scenarios, this will results in some extra nodes provisioned on GCP, which will be shortly terminated.

src/test/java/com/google/jenkins/plugins/computeengine/integration/ConfigAsCodeTestIT.java

@@ -39,6 +39,8 @@ public class ConfigAsCodeTestIT {
 
   private static ComputeClient client;
   private static Map<String, String> label = getLabel(ConfigAsCodeTestIT.class);
+  private static String DISABLE_NO_DELAY_SYSTEM_PROPERTY =
+      "com.google.jenkins.plugins.computeengine.disableNoDelayProvisioning";
 
   @BeforeClass
   public static void init() throws Exception {
@@ -55,7 +57,20 @@ public static void teardown() throws IOException {
   }
 
   @Test
-  public void testWorkerCreated() throws Exception {
+  public void testWorkerCreatedWithoutNoDelayProvision() throws Exception {
+    // Disable NoDelayProvision with the environment variable flag
+    System.setProperty(DISABLE_NO_DELAY_SYSTEM_PROPERTY, "true");
+    testWorkerCreated();
+  }
+
+  @Test
+  public void testWorkerCreatedWithNoDelayProvision() throws Exception {
+    // Enable NoDelayProvision with the environment variable flag
+    System.setProperty(DISABLE_NO_DELAY_SYSTEM_PROPERTY, "false");
+    testWorkerCreated();
+  }
+
+  private void testWorkerCreated() throws Exception {
     ComputeEngineCloud cloud =
         (ComputeEngineCloud) jenkinsRule.jenkins.clouds.getByName("gce-integration");
 

src/test/resources/com/google/jenkins/plugins/computeengine/configuration-as-code.yml

@@ -5,6 +5,7 @@ jenkins:
         projectId: gce-jenkins
         instanceCapStr: 53
         credentialsId: gce-jenkins
+        noDelayProvisioning: false
         configurations:
           - namePrefix:         jenkins-agent-image
             description:        Jenkins agent

src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-it.yml

@@ -5,6 +5,7 @@ jenkins:
         projectId: ${env.GOOGLE_PROJECT_ID}
         instanceCapStr: 10
         credentialsId: ${env.GOOGLE_PROJECT_ID}
+        noDelayProvisioning: true
         configurations:
           - namePrefix:         integration
             description:        integration