add reboot queue backoff reset command

docs/ckecli.md

@@ -47,6 +47,7 @@ $ ckecli [--config FILE] <subcommand> args...
   - [`ckecli reboot-queue list`](#ckecli-reboot-queue-list)
   - [`ckecli reboot-queue cancel INDEX`](#ckecli-reboot-queue-cancel-index)
   - [`ckecli reboot-queue cancel-all`](#ckecli-reboot-queue-cancel-all)
+  - [`ckecli reboot-queue reset-backoff`](#ckecli-reboot-queue-reset-backoff)
 - [`ckecli sabakan`](#ckecli-sabakan)
   - [`ckecli sabakan enable|disable`](#ckecli-sabakan-enabledisable)
   - [`ckecli sabakan is-enabled`](#ckecli-sabakan-is-enabled)
@@ -299,6 +300,11 @@ Cancel the specified reboot queue entry.
 
 Cancel all the reboot queue entries.
 
+### `ckecli reboot-queue reset-backoff`
+
+Reset `drain_backoff_count` and `drain_backoff_expire` of the entries in reboot queue.
+Resetting these values makes CKE try to reboot nodes again immediately.
+
 ## `ckecli sabakan`
 
 Control [sabakan integration feature](sabakan-integration.md).

mtest/reboot_test.go

@@ -406,6 +406,15 @@ func testRebootOperations() {
 		Expect(re).Should(HaveLen(1))
 		Expect(re[0].Status).Should(Equal(cke.RebootStatusQueued))
 		Expect(re[0].DrainBackOffExpire).ShouldNot(Equal(time.Time{}))
+		Expect(re[0].DrainBackOffCount).Should(Not(BeZero()))
+
+		By("Checking reset-backoff command resets drain backoff")
+		ckecliSafe("reboot-queue", "reset-backoff")
+		re, err = getRebootEntries()
+		Expect(err).ShouldNot(HaveOccurred())
+		Expect(re).Should(HaveLen(1))
+		Expect(re[0].DrainBackOffExpire).Should(Equal(time.Time{}))
+		Expect(re[0].DrainBackOffCount).Should(BeZero())
 
 		By("Waiting for reboot completion")
 		waitRebootCompletion(cluster)

pkg/ckecli/cmd/reboot_queue_reset_backoff.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/cybozu-go/cke"
+	"github.com/cybozu-go/well"
+	"github.com/spf13/cobra"
+)
+
+var rebootQueueResetBackoffCmd = &cobra.Command{
+	Use:   "reset-backoff",
+	Short: "Reset drain backoff of the entries in reboot queue",
+	Long:  `Reset drain_backoff_count and drain_backoff_expire of the entries in reboot queue`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		well.Go(func(ctx context.Context) error {
+			entries, err := storage.GetRebootsEntries(ctx)
+			if err != nil {
+				return err
+			}
+			for _, entry := range entries {
+				entry.DrainBackOffCount = 0
+				entry.DrainBackOffExpire = time.Time{}
+				err := storage.UpdateRebootsEntry(ctx, entry)
+				if err == cke.ErrNotFound {
+					// The entry has just finished
+					continue
+				}
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+		well.Stop()
+		return well.Wait()
+	},
+}
+
+func init() {
+	rebootQueueCmd.AddCommand(rebootQueueResetBackoffCmd)
+}