add timeout to helm operations

nuvo · Oct 18, 2018 · a3b09cf · a3b09cf
1 parent db59932
commit a3b09cf
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 14 deletions.
diff --git a/pkg/orca/chart.go b/pkg/orca/chart.go
@@ -22,6 +22,7 @@ type chartCmd struct {
 	helmTLSStore string
 	repo         string
 	inject       bool
+	timeout      int
 
 	out io.Writer
 }
@@ -50,7 +51,7 @@ func NewDeployChartCmd(out io.Writer) *cobra.Command {
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			utils.DeployChartFromRepository(c.releaseName, c.name, c.version, c.kubeContext, c.namespace, c.repo, c.helmTLSStore, c.tls, c.packedValues, c.set, true, c.inject)
+			utils.DeployChartFromRepository(c.releaseName, c.name, c.version, c.kubeContext, c.namespace, c.repo, c.helmTLSStore, c.tls, c.packedValues, c.set, true, c.inject, c.timeout)
 		},
 	}
 
@@ -67,6 +68,7 @@ func NewDeployChartCmd(out io.Writer) *cobra.Command {
 	f.BoolVar(&c.tls, "tls", utils.GetBoolEnvVar("ORCA_TLS", false), "enable TLS for request. Overrides $ORCA_TLS")
 	f.StringVar(&c.helmTLSStore, "helm-tls-store", os.Getenv("HELM_TLS_STORE"), "path to TLS certs and keys. Overrides $HELM_TLS_STORE")
 	f.BoolVar(&c.inject, "inject", utils.GetBoolEnvVar("ORCA_INJECT", false), "enable injection during helm upgrade. Overrides $ORCA_INJECT (requires helm inject plugin: https://github.com/maorfr/helm-inject)")
+	f.IntVar(&c.timeout, "timeout", utils.GetIntEnvVar("ORCA_TIMEOUT", 300), "time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks). Overrides $ORCA_TIMEOUT")
 
 	return cmd
 }

diff --git a/pkg/orca/env.go b/pkg/orca/env.go
@@ -37,6 +37,7 @@ type envCmd struct {
 	force                         bool
 	deployOnlyOverrideIfEnvExists bool
 	parallel                      int
+	timeout                       int
 
 	out io.Writer
 }
@@ -141,12 +142,12 @@ func NewDeployEnvCmd(out io.Writer) *cobra.Command {
 			installedReleases := utils.GetInstalledReleases(e.kubeContext, e.name, includeFailed)
 			releasesToInstall := utils.GetReleasesDelta(desiredReleases, installedReleases)
 
-			utils.DeployChartsFromRepository(releasesToInstall, e.kubeContext, e.name, e.repo, e.helmTLSStore, e.tls, e.packedValues, e.set, e.inject, e.parallel)
+			utils.DeployChartsFromRepository(releasesToInstall, e.kubeContext, e.name, e.repo, e.helmTLSStore, e.tls, e.packedValues, e.set, e.inject, e.parallel, e.timeout)
 
 			if !e.deployOnlyOverrideIfEnvExists {
 				installedReleases = utils.GetInstalledReleases(e.kubeContext, e.name, includeFailed)
 				releasesToDelete := utils.GetReleasesDelta(installedReleases, desiredReleases)
-				utils.DeleteReleases(releasesToDelete, e.kubeContext, e.helmTLSStore, e.tls, e.parallel)
+				utils.DeleteReleases(releasesToDelete, e.kubeContext, e.helmTLSStore, e.tls, e.parallel, e.timeout)
 			}
 			unlockEnvironment(e.name, e.kubeContext, print)
 		},
@@ -166,6 +167,7 @@ func NewDeployEnvCmd(out io.Writer) *cobra.Command {
 	f.BoolVar(&e.inject, "inject", utils.GetBoolEnvVar("ORCA_INJECT", false), "enable injection during helm upgrade. Overrides $ORCA_INJECT (requires helm inject plugin: https://github.com/maorfr/helm-inject)")
 	f.BoolVarP(&e.deployOnlyOverrideIfEnvExists, "deploy-only-override-if-env-exists", "x", false, "if environment exists - deploy only override(s) (support for features spanning multiple services). Overrides $ORCA_DEPLOY_ONLY_OVERRIDE_IF_ENV_EXISTS")
 	f.IntVarP(&e.parallel, "parallel", "p", utils.GetIntEnvVar("ORCA_PARALLEL", 1), "number of releases to act on in parallel. set this flag to 0 for full parallelism. Overrides $ORCA_PARALLEL")
+	f.IntVar(&e.timeout, "timeout", utils.GetIntEnvVar("ORCA_TIMEOUT", 300), "time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks). Overrides $ORCA_TIMEOUT")
 
 	f.BoolVar(&e.createNS, "create-ns", utils.GetBoolEnvVar("ORCA_CREATE_NS", false), "should create new namespace. Overrides $ORCA_CREATE_NS")
 	f.MarkDeprecated("create-ns", "namespace will be created if it does not exist")
@@ -200,7 +202,7 @@ func NewDeleteEnvCmd(out io.Writer) *cobra.Command {
 
 			includeFailed := true
 			releases := utils.GetInstalledReleases(e.kubeContext, e.name, includeFailed)
-			utils.DeleteReleases(releases, e.kubeContext, e.helmTLSStore, e.tls, e.parallel)
+			utils.DeleteReleases(releases, e.kubeContext, e.helmTLSStore, e.tls, e.parallel, e.timeout)
 
 			if nsExists {
 				utils.DeleteNamespace(e.name, e.kubeContext, print)
@@ -217,6 +219,7 @@ func NewDeleteEnvCmd(out io.Writer) *cobra.Command {
 	f.StringVar(&e.helmTLSStore, "helm-tls-store", os.Getenv("HELM_TLS_STORE"), "path to TLS certs and keys. Overrides $HELM_TLS_STORE")
 	f.BoolVar(&e.force, "force", utils.GetBoolEnvVar("ORCA_FORCE", false), "force environment deletion. Overrides $ORCA_FORCE")
 	f.IntVarP(&e.parallel, "parallel", "p", utils.GetIntEnvVar("ORCA_PARALLEL", 1), "number of releases to act on in parallel. set this flag to 0 for full parallelism. Overrides $ORCA_PARALLEL")
+	f.IntVar(&e.timeout, "timeout", utils.GetIntEnvVar("ORCA_TIMEOUT", 300), "time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks). Overrides $ORCA_TIMEOUT")
 
 	return cmd
 }

diff --git a/pkg/utils/helm.go b/pkg/utils/helm.go
@@ -11,7 +11,7 @@ import (
 )
 
 // DeployChartsFromRepository deploys a list of Helm charts from a repository in parallel
-func DeployChartsFromRepository(releasesToInstall []ReleaseSpec, kubeContext, namespace, repo, helmTLSStore string, tls bool, packedValues, set []string, inject bool, parallel int) {
+func DeployChartsFromRepository(releasesToInstall []ReleaseSpec, kubeContext, namespace, repo, helmTLSStore string, tls bool, packedValues, set []string, inject bool, parallel, timeout int) {
 
 	totalReleases := len(releasesToInstall)
 	if parallel == 0 {
@@ -48,7 +48,7 @@ func DeployChartsFromRepository(releasesToInstall []ReleaseSpec, kubeContext, na
 
 				// deploy chart
 				log.Println("deploying chart", c.ChartName, "version", c.ChartVersion)
-				DeployChartFromRepository(c.ReleaseName, c.ChartName, c.ChartVersion, kubeContext, namespace, repo, helmTLSStore, tls, packedValues, set, false, inject)
+				DeployChartFromRepository(c.ReleaseName, c.ChartName, c.ChartVersion, kubeContext, namespace, repo, helmTLSStore, tls, packedValues, set, false, inject, timeout)
 				log.Println("deployed chart", c.ChartName, "version", c.ChartVersion)
 
 				// Deployment is done, remove chart from dependencies
@@ -63,7 +63,7 @@ func DeployChartsFromRepository(releasesToInstall []ReleaseSpec, kubeContext, na
 }
 
 // DeployChartFromRepository deploys a Helm chart from a chart repository
-func DeployChartFromRepository(releaseName, name, version, kubeContext, namespace, repo, helmTLSStore string, tls bool, packedValues, set []string, isIsolated, inject bool) {
+func DeployChartFromRepository(releaseName, name, version, kubeContext, namespace, repo, helmTLSStore string, tls bool, packedValues, set []string, isIsolated, inject bool, timeout int) {
 	tempDir := MkRandomDir()
 
 	if releaseName == "" {
@@ -79,7 +79,7 @@ func DeployChartFromRepository(releaseName, name, version, kubeContext, namespac
 	valuesChain := CreateValuesChain(name, tempDir, packedValues)
 	setChain := CreateSetChain(name, set)
 
-	UpgradeRelease(name, releaseName, kubeContext, namespace, valuesChain, setChain, tls, helmTLSStore, tempDir, isIsolated, inject)
+	UpgradeRelease(name, releaseName, kubeContext, namespace, valuesChain, setChain, tls, helmTLSStore, tempDir, isIsolated, inject, timeout)
 
 	os.RemoveAll(tempDir)
 }
@@ -203,14 +203,14 @@ func CreateSetChain(name string, inputSet []string) string {
 }
 
 // UpgradeRelease performs helm upgrade -i
-func UpgradeRelease(name, releaseName, kubeContext, namespace, values, set string, tls bool, helmTLSStore, dir string, print, inject bool) {
+func UpgradeRelease(name, releaseName, kubeContext, namespace, values, set string, tls bool, helmTLSStore, dir string, print, inject bool, timeout int) {
 	var injectStr string
 	kubeContextFlag := "kube-context"
 	if inject {
 		injectStr = "inject "
 		kubeContextFlag = "kubecontext"
 	}
-	cmd := fmt.Sprintf("helm %supgrade%s -i %s --%s %s --namespace %s%s%s %s/%s", injectStr, getTLS(tls, kubeContext, helmTLSStore), releaseName, kubeContextFlag, kubeContext, namespace, values, set, dir, name)
+	cmd := fmt.Sprintf("helm %supgrade%s -i %s --%s %s --namespace %s%s%s %s/%s --timeout %d", injectStr, getTLS(tls, kubeContext, helmTLSStore), releaseName, kubeContextFlag, kubeContext, namespace, values, set, dir, name, timeout)
 	output := Exec(cmd)
 	if print {
 		fmt.Println(cmd)
@@ -219,7 +219,8 @@ func UpgradeRelease(name, releaseName, kubeContext, namespace, values, set strin
 }
 
 // DeleteReleases deletes a list of releases in parallel
-func DeleteReleases(releasesToDelete []ReleaseSpec, kubeContext, helmTLSStore string, tls bool, parallel int) {
+func DeleteReleases(releasesToDelete []ReleaseSpec, kubeContext, helmTLSStore string, tls bool, parallel, timeout int) {
+	print := false
 	totalReleases := len(releasesToDelete)
 	if parallel == 0 {
 		parallel = totalReleases
@@ -232,16 +233,16 @@ func DeleteReleases(releasesToDelete []ReleaseSpec, kubeContext, helmTLSStore st
 		go func(c ReleaseSpec) {
 			defer bwg.Done()
 			log.Println("deleting", c.ReleaseName)
-			DeleteRelease(c.ReleaseName, kubeContext, tls, helmTLSStore, false)
+			DeleteRelease(c.ReleaseName, kubeContext, tls, helmTLSStore, timeout, print)
 			log.Println("deleted", c.ReleaseName)
 		}(c)
 	}
 	bwg.Wait()
 }
 
 // DeleteRelease deletes a release from Kubernetes
-func DeleteRelease(releaseName, kubeContext string, tls bool, helmTLSStore string, print bool) {
-	cmd := fmt.Sprintf("helm delete%s --purge %s --kube-context %s", getTLS(tls, kubeContext, helmTLSStore), releaseName, kubeContext)
+func DeleteRelease(releaseName, kubeContext string, tls bool, helmTLSStore string, timeout int, print bool) {
+	cmd := fmt.Sprintf("helm delete%s --purge %s --kube-context %s --timeout %d", getTLS(tls, kubeContext, helmTLSStore), releaseName, kubeContext, timeout)
 	output := Exec(cmd)
 	if print {
 		fmt.Println(cmd)