feat: logs (#285)

* feat: write in log file

* chore: remove logrus

* feat: stream logs from file

* chore: clean up

extensions/aws/env.go

@@ -15,7 +15,7 @@ import (
 	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
 	"github.com/kubefirst/runtime/pkg/k8s"
 	"github.com/kubefirst/runtime/pkg/vault"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 )
 

extensions/civo/env.go

@@ -15,7 +15,7 @@ import (
 	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
 	"github.com/kubefirst/runtime/pkg/k8s"
 	"github.com/kubefirst/runtime/pkg/vault"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 )
 

extensions/digitalocean/env.go

@@ -15,7 +15,7 @@ import (
 	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
 	"github.com/kubefirst/runtime/pkg/k8s"
 	"github.com/kubefirst/runtime/pkg/vault"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 )
 

extensions/google/env.go

@@ -16,7 +16,7 @@ import (
 	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
 	"github.com/kubefirst/runtime/pkg/k8s"
 	"github.com/kubefirst/runtime/pkg/vault"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -34,7 +34,7 @@ func GetGoogleTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[st
 	envs["TF_VAR_project"] = cl.GoogleAuth.ProjectId
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
-		log.Fatalf("error getting home path: %s", err)
+		log.Fatal().Msgf("error getting home path: %s", err)
 	}
 	envs["GOOGLE_APPLICATION_CREDENTIALS"] = fmt.Sprintf("%s/.k1/application-default-credentials.json", homeDir)
 
@@ -48,7 +48,7 @@ func GetGithubTerraformEnvs(envs map[string]string, cl *pkgtypes.Cluster) map[st
 	envs["TF_VAR_kbot_ssh_public_key"] = cl.GitAuth.PublicKey
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
-		log.Fatalf("error getting home path: %s", err)
+		log.Fatal().Msgf("error getting home path: %s", err)
 	}
 	envs["GOOGLE_APPLICATION_CREDENTIALS"] = fmt.Sprintf("%s/.k1/application-default-credentials.json", homeDir)
 
@@ -65,7 +65,7 @@ func GetGitlabTerraformEnvs(envs map[string]string, gid int, cl *pkgtypes.Cluste
 	envs["TF_VAR_gitlab_owner"] = cl.GitAuth.Owner
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
-		log.Fatalf("error getting home path: %s", err)
+		log.Fatal().Msgf("error getting home path: %s", err)
 	}
 	envs["GOOGLE_APPLICATION_CREDENTIALS"] = fmt.Sprintf("%s/.k1/application-default-credentials.json", homeDir)
 
@@ -79,7 +79,7 @@ func GetUsersTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster
 	envs[fmt.Sprintf("%s_OWNER", strings.ToUpper(cl.GitProvider))] = cl.GitAuth.Owner
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
-		log.Fatalf("error getting home path: %s", err)
+		log.Fatal().Msgf("error getting home path: %s", err)
 	}
 	envs["GOOGLE_APPLICATION_CREDENTIALS"] = fmt.Sprintf("%s/.k1/application-default-credentials.json", homeDir)
 
@@ -104,7 +104,7 @@ func GetVaultTerraformEnvs(clientset *kubernetes.Clientset, cl *pkgtypes.Cluster
 	envs["TF_VAR_cloudflare_api_key"] = cl.CloudflareAuth.Token
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
-		log.Fatalf("error getting home path: %s", err)
+		log.Fatal().Msgf("error getting home path: %s", err)
 	}
 	envs["GOOGLE_APPLICATION_CREDENTIALS"] = fmt.Sprintf("%s/.k1/application-default-credentials.json", homeDir)
 

extensions/terraform/exec.go

@@ -12,22 +12,13 @@ import (
 	"os/exec"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 )
 
 // ExecShellWithVars Exec shell actions supporting:
 //   - On-the-fly logging of result
 //   - Map of Vars loaded
 func ExecShellWithVars(osvars map[string]string, command string, args ...string) error {
-	// Logging handler
-	// Logs to stdout to maintain compatibility with event streaming
-	log.SetFormatter(&log.TextFormatter{
-		FullTimestamp:   true,
-		TimestampFormat: "",
-	})
-	log.SetReportCaller(false)
-	log.SetOutput(os.Stdout)
-
 	for k, v := range osvars {
 		os.Setenv(k, v)
 		suppressedValue := strings.Repeat("*", len(v))

extensions/terraform/terraform.go

@@ -10,19 +10,10 @@ import (
 	"fmt"
 	"os"
 
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 )
 
 func initActionAutoApprove(terraformClientPath string, tfAction, tfEntrypoint string, tfEnvs map[string]string) error {
-	// Logging handler
-	// Logs to stdout to maintain compatibility with event streaming
-	log.SetFormatter(&log.TextFormatter{
-		FullTimestamp:   true,
-		TimestampFormat: "",
-	})
-	log.SetReportCaller(false)
-	log.SetOutput(os.Stdout)
-
 	log.Printf("initActionAutoApprove - action: %s entrypoint: %s", tfAction, tfEntrypoint)
 
 	err := os.Chdir(tfEntrypoint)

extensions/vultr/env.go

@@ -15,7 +15,7 @@ import (
 	pkgtypes "github.com/kubefirst/kubefirst-api/pkg/types"
 	"github.com/kubefirst/runtime/pkg/k8s"
 	"github.com/kubefirst/runtime/pkg/vault"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 	"k8s.io/client-go/kubernetes"
 )
 

go.mod

@@ -29,9 +29,9 @@ require (
 	github.com/kubefirst/metrics-client v0.3.0
 	github.com/kubefirst/runtime v0.3.35
 	github.com/minio/minio-go/v7 v7.0.49
+	github.com/nxadm/tail v1.4.8
 	github.com/otiai10/copy v1.7.0
 	github.com/rs/zerolog v1.29.1
-	github.com/sirupsen/logrus v1.9.0
 	github.com/swaggo/files v1.0.0
 	github.com/swaggo/gin-swagger v1.5.3
 	github.com/swaggo/swag v1.16.1
@@ -47,6 +47,11 @@ require (
 	sigs.k8s.io/aws-iam-authenticator v0.6.7
 )
 
+require (
+	github.com/sirupsen/logrus v1.9.0 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+)
+
 replace (
 	sigs.k8s.io/kustomize/api => sigs.k8s.io/kustomize/api v0.12.1
 	// https://github.com/fluxcd/flux2/issues/3564
@@ -101,6 +106,7 @@ require (
 	github.com/cloudflare/circl v1.1.0 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/digitalocean/godo v1.98.0 // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
@@ -212,11 +218,11 @@ require (
 	github.com/rs/xid v1.4.0 // indirect
 	github.com/russross/blackfriday v1.5.2 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	github.com/segmentio/analytics-go v3.1.0+incompatible // indirect
+	github.com/segmentio/analytics-go v3.1.0+incompatible
 	github.com/segmentio/backo-go v1.0.1 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/skeema/knownhosts v1.1.0 // indirect
-	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/afero v1.9.3
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

go.sum

@@ -315,6 +315,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
+github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
+github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
@@ -738,8 +740,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubefirst/metrics-client v0.2.8 h1:JfaeiBifZD/DpyYW2QVHcrhH/KWA98OmM+7c7M12qMc=
-github.com/kubefirst/metrics-client v0.2.8/go.mod h1:GR7wsMcyYhd+EU67PeuMCBYFE6OJ7P/j5OI5BLOoRMc=
 github.com/kubefirst/metrics-client v0.3.0 h1:zCug82pEzeWhHhpeYQvdhytRNDxrLxX18dPQ5PSxY3s=
 github.com/kubefirst/metrics-client v0.3.0/go.mod h1:GR7wsMcyYhd+EU67PeuMCBYFE6OJ7P/j5OI5BLOoRMc=
 github.com/kubefirst/runtime v0.3.35 h1:wn430Irf0E1vc3X0WX3lYBpyhQ5TN6xxMcargILA9uI=

internal/aws/aws.go

@@ -21,7 +21,7 @@ import (
 	"github.com/kubefirst/kubefirst-api/internal/constants"
 	"github.com/kubefirst/kubefirst-api/internal/env"
 	"github.com/kubefirst/kubefirst-api/internal/utils"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 )
 
 const (
@@ -44,7 +44,7 @@ func NewAws() aws.Config {
 		config.WithSharedConfigProfile(env.AWSProfile),
 	)
 	if err != nil {
-		log.Errorf("Could not create AWS config: %s", err.Error())
+		log.Error().Msgf("Could not create AWS config: %s", err.Error())
 	}
 
 	return awsClient
@@ -69,13 +69,13 @@ func (conf *AWSConfiguration) GetHostedZoneID(domain string) (string, error) {
 func (conf *AWSConfiguration) Route53AlterResourceRecord(r *AWSRoute53AlterResourceRecord) (*route53.ChangeResourceRecordSetsOutput, error) {
 	route53Client := route53.NewFromConfig(conf.Config)
 
-	log.Infof("validating hostedZoneId %s", r.hostedZoneID)
-	log.Infof("validating route53RecordName %s", r.route53RecordName)
+	log.Info().Msgf("validating hostedZoneId %s", r.hostedZoneID)
+	log.Info().Msgf("validating route53RecordName %s", r.route53RecordName)
 	record, err := route53Client.ChangeResourceRecordSets(
 		context.Background(),
 		r.input)
 	if err != nil {
-		log.Warnf("%s", err)
+		log.Warn().Msgf("%s", err)
 		return &route53.ChangeResourceRecordSetsOutput{}, err
 	}
 	return record, nil
@@ -119,7 +119,7 @@ func (conf *AWSConfiguration) Route53ListTXTRecords(hostedZoneId string) ([]AWST
 	}
 	var txtRecords []AWSTXTRecord
 	for _, recordSet := range recordSets.ResourceRecordSets {
-		log.Debugf("Record Name: %s", *recordSet.Name)
+		log.Debug().Msgf("Record Name: %s", *recordSet.Name)
 		if recordSet.Type == route53Types.RRTypeTxt {
 			for _, resourceRecord := range recordSet.ResourceRecords {
 				if recordSet.SetIdentifier != nil && recordSet.Weight != nil {
@@ -172,10 +172,10 @@ func (conf *AWSConfiguration) TestHostedZoneLiveness(hostedZoneName string) (boo
 	// Determine whether or not the record exists, create if it doesn't
 	switch utils.FindStringInSlice(foundRecordNames, route53RecordName) {
 	case true:
-		log.Infof("record %s exists - zone validated", route53RecordName)
+		log.Info().Msgf("record %s exists - zone validated", route53RecordName)
 		return true, nil
 	case false:
-		log.Infof("record %s does not exist, creating...", route53RecordName)
+		log.Info().Msgf("record %s does not exist, creating...", route53RecordName)
 
 		// Construct resource record alter and create record
 		alt := AWSRoute53AlterResourceRecord{
@@ -210,27 +210,27 @@ func (conf *AWSConfiguration) TestHostedZoneLiveness(hostedZoneName string) (boo
 		if err != nil {
 			return false, err
 		}
-		log.Infof("record created and is in state: %s", record.ChangeInfo.Status)
+		log.Info().Msgf("record created and is in state: %s", record.ChangeInfo.Status)
 
 		// Wait for record
 		ch := make(chan bool, 1)
 		retries := 10
 		retryInterval := 10
 		duration := (retries * retryInterval)
-		log.Infof("waiting on %s domain validation record creation for %v seconds...", route53RecordName, duration)
+		log.Info().Msgf("waiting on %s domain validation record creation for %v seconds...", route53RecordName, duration)
 		go func() {
 			for i := 1; i < retries; i++ {
 				ips, err := net.LookupTXT(route53RecordName)
 				if err != nil {
 					ips, err = utils.BackupResolver.LookupTXT(context.Background(), route53RecordName)
 				}
 				if err != nil {
-					log.Warnf("attempt %v of %v resolving %s, retrying in %vs", i, retries, route53RecordName, retryInterval)
+					log.Warn().Msgf("attempt %v of %v resolving %s, retrying in %vs", i, retries, route53RecordName, retryInterval)
 					time.Sleep(time.Duration(int32(retryInterval)) * time.Second)
 				} else {
 					for _, ip := range ips {
 						// todo check ip against route53RecordValue in some capacity so we can pivot the value for testing
-						log.Infof("%s. in TXT record value: %s", route53RecordName, ip)
+						log.Info().Msgf("%s. in TXT record value: %s", route53RecordName, ip)
 						ch <- true
 					}
 				}

internal/controller/argocd.go

@@ -9,29 +9,19 @@ package controller
 import (
 	"context"
 	"fmt"
-	"os"
 
 	argocdapi "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned"
 	awsext "github.com/kubefirst/kubefirst-api/extensions/aws"
 	"github.com/kubefirst/metrics-client/pkg/telemetry"
 	"github.com/kubefirst/runtime/pkg"
 	"github.com/kubefirst/runtime/pkg/argocd"
 	"github.com/kubefirst/runtime/pkg/k8s"
-	log "github.com/sirupsen/logrus"
+	log "github.com/rs/zerolog/log"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // InstallArgoCD
 func (clctrl *ClusterController) InstallArgoCD() error {
-	// Logging handler
-	// Logs to stdout to maintain compatibility with event streaming
-	log.SetFormatter(&log.TextFormatter{
-		FullTimestamp:   true,
-		TimestampFormat: "",
-	})
-	log.SetReportCaller(false)
-	log.SetOutput(os.Stdout)
-
 	cl, err := clctrl.MdbCl.GetCluster(clctrl.ClusterName)
 	if err != nil {
 		return err
@@ -54,7 +44,7 @@ func (clctrl *ClusterController) InstallArgoCD() error {
 		}
 
 		argoCDInstallPath := fmt.Sprintf("github.com:kubefirst/manifests/argocd/cloud?ref=%s", pkg.KubefirstManifestRepoRef)
-		log.Infof("installing argocd")
+		log.Info().Msg("installing argocd")
 
 		telemetry.SendEvent(clctrl.TelemetryEvent, telemetry.ArgoCDInstallStarted, "")
 		err = argocd.ApplyArgoCDKustomize(kcfg.Clientset, argoCDInstallPath)
@@ -68,7 +58,7 @@ func (clctrl *ClusterController) InstallArgoCD() error {
 		// Wait for ArgoCD to be ready
 		_, err = k8s.VerifyArgoCDReadiness(kcfg.Clientset, true, 300)
 		if err != nil {
-			log.Errorf("error waiting for ArgoCD to become ready: %s", err)
+			log.Error().Msgf("error waiting for ArgoCD to become ready: %s", err)
 			return err
 		}
 
@@ -83,15 +73,6 @@ func (clctrl *ClusterController) InstallArgoCD() error {
 
 // InitializeArgoCD
 func (clctrl *ClusterController) InitializeArgoCD() error {
-	// Logging handler
-	// Logs to stdout to maintain compatibility with event streaming
-	log.SetFormatter(&log.TextFormatter{
-		FullTimestamp:   true,
-		TimestampFormat: "",
-	})
-	log.SetReportCaller(false)
-	log.SetOutput(os.Stdout)
-
 	cl, err := clctrl.MdbCl.GetCluster(clctrl.ClusterName)
 	if err != nil {
 		return err
@@ -113,7 +94,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error {
 			}
 		}
 
-		log.Info("Setting argocd username and password credentials")
+		log.Info().Msg("Setting argocd username and password credentials")
 
 		argocd.ArgocdSecretClient = kcfg.Clientset.CoreV1().Secrets("argocd")
 
@@ -122,8 +103,8 @@ func (clctrl *ClusterController) InitializeArgoCD() error {
 			return fmt.Errorf("argocd password not found in secret")
 		}
 
-		log.Info("argocd username and password credentials set successfully")
-		log.Info("getting an argocd auth token")
+		log.Info().Msg("argocd username and password credentials set successfully")
+		log.Info().Msg("getting an argocd auth token")
 
 		var argoCDToken string
 
@@ -150,7 +131,7 @@ func (clctrl *ClusterController) InitializeArgoCD() error {
 			}
 		}
 
-		log.Info("argocd admin auth token set")
+		log.Info().Msg("argocd admin auth token set")
 
 		err = clctrl.MdbCl.UpdateCluster(clctrl.ClusterName, "argocd_password", argocdPassword)
 		if err != nil {
@@ -198,7 +179,7 @@ func (clctrl *ClusterController) DeployRegistryApplication() error {
 			return err
 		}
 
-		log.Info("applying the registry application to argocd")
+		log.Info().Msg("applying the registry application to argocd")
 
 		registryURL, err := clctrl.GetRepoURL()
 		if err != nil {