Add debug flag for CLI (#140)

* add debug flag for CLI

.github/workflows/release.yaml

@@ -70,12 +70,12 @@ jobs:
         go-version: '1.17.1'
     - name: Build the KIT CLI
       run: |
-        GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o ./bin/kitcli ./cmd/
-        zip ./bin/kitcli_${{ env.RELEASE_VERSION }}_${{ matrix.os }}_${{ matrix.arch }}.zip -j ./bin/kitcli
+        GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o ./bin/kitctl ./cmd/
+        zip ./bin/kitctl_${{ env.RELEASE_VERSION }}_${{ matrix.os }}_${{ matrix.arch }}.zip -j ./bin/kitctl
       working-directory: substrate
     - name: Attach the binary to release
       uses: softprops/action-gh-release@v1
       with:
-        files: "./substrate/bin/kitcli_${{ env.RELEASE_VERSION }}_${{ matrix.os }}_${{ matrix.arch }}.zip"
+        files: "./substrate/bin/kitctl_${{ env.RELEASE_VERSION }}_${{ matrix.os }}_${{ matrix.arch }}.zip"
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

kitcli.rb → kitctl.rb

@@ -1,16 +1,16 @@
 # Update the version and SHA256 for the CLI when new version is released
 require 'formula'
-class Kitcli < Formula
+class Kitctl < Formula
   homepage 'https://github.com/awslabs/kubernetes-iteration-toolkit/substrate'
   version '$VERSION'
   if OS.mac? && Hardware::CPU.is_64_bit?
-    url 'https://github.com/awslabs/kubernetes-iteration-toolkit/releases/download/$VERSION/kitcli_$VERSION_darwin_amd64.zip'
+    url 'https://github.com/awslabs/kubernetes-iteration-toolkit/releases/download/$VERSION/kitctl_$VERSION_darwin_amd64.zip'
     sha256 '$SHA256'
   else
     echo "Hardware not supported"
     exit 1
   end
   def install
-    bin.install 'kitcli'
+    bin.install 'kitctl'
   end
 end

substrate/README.md

@@ -1,39 +1,106 @@
-# KITCLI
+# kitctl
 
-## Installing KITCLI
+## Installing kitctl
 
 ```bash
 brew tap awslabs/kit https://github.com/awslabs/kubernetes-iteration-toolkit.git
-brew install kitcli
+brew install kitctl
 ```
 
 ## Usage
+kitctl helps provision an AWS infrastructure environment to deploy Kubernetes clusters using kit-operator. It runs a single node kubernetes cluster in a VPC installed with all the required controllers like Karpeneter, KIT-operator, ELB controller and EBS CSI Driver to manage the lifecycle of clusters. kitctl also creates the necessary IAM permissions required for these controllers.
+To get started make sure you have admin access to AWS.
 
-```sh
-kitcli apply
-kitcli delete
+### Bootstrap an environment in AWS to run Kubernetes clusters using kit-operator
+
+```bash
+kitctl bootstrap
 ```
+> Set KUBECONFIG to access the environment with the kubeconfig location provided from this command
 
-```shell
-cat <<EOF | kitcli apply -f -
-apiVersion: kit.sh/v1alpha1
-kind: Substrate
+### Configure Karpenter to provision nodes for the Kubernetes cluster control plane
+
+```bash
+GUEST_CLUSTER_NAME=foo # Desired Cluster name
+SUBSTRATE_CLUSTER_NAME=test-substrate
+```
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
 metadata:
-  name: ${USER}
+  name: master-nodes
 spec:
-  instanceType: c5.2xlarge
----
-apiVersion: kit.sh/v1alpha1
-kind: Cluster
+  kubeletConfiguration:
+    clusterDNS:
+      - "10.96.0.10"
+  labels:
+    kit.k8s.sh/app: ${GUEST_CLUSTER_NAME}-apiserver
+    kit.k8s.sh/control-plane-name: ${GUEST_CLUSTER_NAME}
+  provider:
+    instanceProfile: kit-${SUBSTRATE_CLUSTER_NAME}-tenant-controlplane-node-role
+    subnetSelector:
+      karpenter.sh/discovery: ${SUBSTRATE_CLUSTER_NAME}
+    securityGroupSelector:
+      kubernetes.io/cluster/${SUBSTRATE_CLUSTER_NAME}: owned
+    tags:
+      kit.aws/substrate: ${SUBSTRATE_CLUSTER_NAME}
+  ttlSecondsAfterEmpty: 30
+EOF
+```
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
 metadata:
-  name: ${USER}
-spec: {}
+  name: etcd-nodes
+spec:
+  kubeletConfiguration:
+    clusterDNS:
+      - "10.96.0.10"
+  labels:
+    kit.k8s.sh/app: ${GUEST_CLUSTER_NAME}-etcd
+    kit.k8s.sh/control-plane-name: ${GUEST_CLUSTER_NAME}
+  provider:
+    instanceProfile: kit-${SUBSTRATE_CLUSTER_NAME}-tenant-controlplane-node-role
+    subnetSelector:
+      karpenter.sh/discovery: ${SUBSTRATE_CLUSTER_NAME}
+    securityGroupSelector:
+      kubernetes.io/cluster/${SUBSTRATE_CLUSTER_NAME}: owned
+    tags:
+      kit.aws/substrate: ${SUBSTRATE_CLUSTER_NAME}
+  ttlSecondsAfterEmpty: 30
+EOF
 EOF
+```
+
+### Provision a Kubernetes cluster control plane
 
-kitcli delete substrate ${USER}
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: kit.k8s.sh/v1alpha1
+kind: ControlPlane
+metadata:
+  name: ${GUEST_CLUSTER_NAME} # Desired Cluster name
+spec: {}
+EOF
 ```
 
-## Developing
+### cleanup
+
+- To remove the kubernetes cluster provisioned using kit-operator
+
+```bash
+kubectl delete controlplane ${GUEST_CLUSTER_NAME}
 ```
-alias kitcli="go run ./cmd"
+
+- To clean up the AWS environment
+
+```bash
+kitctl helps provision an AWS infrastructure environment to deploy Kubernetes clusters using kit-operator. It runs a single node kubernetes cluster with all the required controllers like Karpeneter, KIT-operator, ELB controller and EBS CSI Driver to manage the lifecycle of clusters. kitctl also creates the necessary delete
 ```
+
+### Debug logs
+Run the `kitctl helps provision an AWS infrastructure environment to deploy Kubernetes clusters using kit-operator. It runs a single node kubernetes cluster with all the required controllers like Karpeneter, KIT-operator, ELB controller and EBS CSI Driver to manage the lifecycle of clusters. kitctl also creates the necessary` commands with `--debug` flag

substrate/cmd/apply.go → substrate/cmd/bootstrap.go

@@ -15,6 +15,7 @@ limitations under the License.
 package main
 
 import (
+	"os"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -25,18 +26,30 @@ import (
 	"knative.dev/pkg/logging"
 )
 
-func init() {
-	rootCmd.AddCommand(&cobra.Command{
-		Use:   "apply",
-		Short: "Apply an environment for testing. Will reconnect if the environment already exists.",
+func bootstrapCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "bootstrap",
+		Short: "Bootstrap an environment for testing. Will reconnect if the environment already exists.",
 		Long:  ``,
-		Run:   Apply,
-	})
+		Run:   bootstrap,
+	}
 }
 
-func Apply(cmd *cobra.Command, args []string) {
+func bootstrap(cmd *cobra.Command, args []string) {
+	// ignore logs printed to stdout from underlying kubeadm packages
+	if !options.debug {
+		stdout := os.Stdout
+		stderr := os.Stderr
+		os.Stdout, _ = os.Open(os.DevNull)
+		os.Stderr, _ = os.Open(os.DevNull)
+		defer func() {
+			os.Stdout = stdout
+			os.Stderr = stderr
+		}()
+	}
 	ctx := cmd.Context()
 	start := time.Now()
+	// TODO make substrate name configurable
 	name := "test-substrate"
 	if err := substrate.NewController(ctx).Reconcile(ctx, &v1alpha1.Substrate{
 		ObjectMeta: metav1.ObjectMeta{Name: name},
@@ -56,5 +69,5 @@ func Apply(cmd *cobra.Command, args []string) {
 		logging.FromContext(ctx).Error(err.Error())
 		return
 	}
-	logging.FromContext(ctx).Infof("Applied substrate %s after %s", name, time.Since(start))
+	logging.FromContext(ctx).Infof("✅ Bootstrap finished, substrate name %s time taken %s", name, time.Since(start))
 }

substrate/cmd/delete.go

@@ -24,18 +24,16 @@ import (
 	"knative.dev/pkg/logging"
 )
 
-func init() {
-	rootCmd.LocalFlags().StringVar(&options.File, "name", "", "Name for the environment")
-	rootCmd.LocalFlags().StringVarP(&options.File, "file", "f", "", "Configuration file for the environment")
-	rootCmd.AddCommand(&cobra.Command{
+func deleteCommand() *cobra.Command {
+	return &cobra.Command{
 		Use:   "delete",
 		Short: "Delete the environment",
 		Long:  ``,
-		Run:   Delete,
-	})
+		Run:   delete,
+	}
 }
 
-func Delete(cmd *cobra.Command, args []string) {
+func delete(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 	start := time.Now()
 	name := "test-substrate"

substrate/cmd/root.go

@@ -17,33 +17,83 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"io"
+	"os"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"knative.dev/pkg/logging"
 )
 
 func main() {
-	ctx := context.Background()
-	logger, _ := zap.NewDevelopment(zap.WithCaller(false))
-	ctx = logging.WithLogger(ctx, logger.Sugar())
-	runtime.Must(rootCmd.ExecuteContext(ctx))
+	rootCmd := newRootCmd(os.Args[1:])
+	logLevel := zapcore.InfoLevel
+	if options.debug {
+		logLevel = zapcore.DebugLevel
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	logger := zap.New(zapcore.NewCore(zapcore.NewConsoleEncoder(zapcore.EncoderConfig{MessageKey: "message"}),
+		customLogWriteTo(ctx, os.Stdout), zap.LevelEnablerFunc(func(level zapcore.Level) bool {
+			return level >= logLevel
+		}),
+	))
+	runtime.Must(rootCmd.ExecuteContext(logging.WithLogger(ctx, logger.Sugar())))
 }
 
-var rootCmd = &cobra.Command{
-	Use:   "kitctl",
-	Short: "A tool to provision Kubernetes cluster using kit-operator",
-	Long: `kitctl help users provision Kubernetes clustes using kit-operator.
-It also supports configuring the cloud provider environment to get started easily`,
+func newRootCmd(args []string) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "kitctl",
+		Short: "A tool to provision Kubernetes cluster using kit-operator",
+		Long: `kitctl help users provision Kubernetes clustes using kit-operator.
+		It also configures the cloud provider environment to get started easily`,
+	}
+	flags := cmd.PersistentFlags()
+	options.addFlags(flags)
+	if err := flags.Parse(args); err != nil {
+		panic(err)
+	}
+	// Add subcommands
+	cmd.AddCommand(bootstrapCommand())
+	cmd.AddCommand(deleteCommand())
+	return cmd
 }
 
-var options = Options{}
+var options = &Options{}
 
 type Options struct {
-	File string
+	file  string
+	debug bool
+	help  bool
 }
 
-func init() {
-	rootCmd.PersistentFlags().StringVarP(&options.File, "file", "f", "", "Configuration file for the environment")
+func (o *Options) addFlags(fs *pflag.FlagSet) {
+	fs.StringVarP(&o.file, "file", "f", "", "Configuration file for the environment")
+	fs.BoolVarP(&o.debug, "debug", "", false, "enable debug logs")
+	fs.BoolVar(&o.help, "help", false, "help flag")
+}
+
+func customLogWriteTo(ctx context.Context, w io.Writer) *os.File {
+	reader, writer, err := os.Pipe()
+	if err != nil {
+		panic(fmt.Sprintf("failed to create pipe, %v", err))
+	}
+	go func(ctx context.Context) {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				_, err := io.Copy(w, reader)
+				if err != nil {
+					panic(err)
+				}
+			}
+		}
+	}(ctx)
+	return writer
 }

substrate/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/imdario/mergo v0.3.12
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/spf13/cobra v1.3.0
+	github.com/spf13/pflag v1.0.5
 	go.uber.org/multierr v1.7.0
 	go.uber.org/zap v1.19.1
 	helm.sh/helm/v3 v3.8.0

substrate/go.sum

@@ -847,6 +847,7 @@ github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

substrate/pkg/controller/substrate/cluster/addons/kubeproxy.go

@@ -38,7 +38,7 @@ func (k *KubeProxy) Create(ctx context.Context, substrate *v1alpha1.Substrate) (
 	}
 	config := cluster.DefaultClusterConfig(substrate)
 	if err := proxy.EnsureProxyAddon(&config.ClusterConfiguration, &config.LocalAPIEndpoint, client); err != nil {
-		return reconcile.Result{Requeue: true}, fmt.Errorf("ensuring kube-proxy addon, %w", err)
+		return reconcile.Result{}, fmt.Errorf("ensuring kube-proxy addon, %w", err)
 	}
 	return reconcile.Result{}, nil
 }

substrate/pkg/controller/substrate/cluster/address.go

@@ -36,7 +36,7 @@ func (a *Address) Create(ctx context.Context, substrate *v1alpha1.Substrate) (re
 		return reconcile.Result{}, fmt.Errorf("describing addresses, %w", err)
 	}
 	if len(addressesOutput.Addresses) > 0 {
-		logging.FromContext(ctx).Infof("Found address %s", aws.StringValue(addressesOutput.Addresses[0].PublicIp))
+		logging.FromContext(ctx).Debugf("Found address %s", aws.StringValue(addressesOutput.Addresses[0].PublicIp))
 		substrate.Status.Cluster.Address = addressesOutput.Addresses[0].PublicIp
 		return reconcile.Result{}, nil
 	}