refactor: replace apex/log with structured logger across multiple ser…

…vices - Replaced all instances of apex/log with a structured logger interface in various services, including Api, Server, Config, and others, to enhance logging consistency and context. - Updated logging calls to utilize the new logger methods, improving error tracking and service monitoring. - Added logger initialization in services and controllers to ensure proper logging setup. - Improved error handling and logging messages for better clarity during service operations. - Removed unused apex/log imports and cleaned up related code for better maintainability.
crawlab-team · Dec 24, 2024 · 3276083 · 3276083
1 parent 67165f5
commit 3276083
Show file tree

Hide file tree

Showing 46 changed files with 617 additions and 490 deletions.
diff --git a/core/apps/api.go b/core/apps/api.go
@@ -3,8 +3,8 @@ package apps
 import (
 	"context"
 	"errors"
-	"github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/middlewares"
 	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/gin-gonic/gin"
@@ -25,6 +25,7 @@ type Api struct {
 	ln          net.Listener
 	srv         *http.Server
 	initialized bool
+	interfaces.Logger
 }
 
 func (app *Api) Init() {
@@ -59,14 +60,14 @@ func (app *Api) Start() {
 	if err != nil {
 		panic(err)
 	}
-	log.Infof("api server listening on %s", address)
+	app.Infof("api server listening on %s", address)
 
 	// serve
 	if err := http.Serve(app.ln, app.app); err != nil {
 		if !errors.Is(err, http.ErrServerClosed) {
-			log.Errorf("run api server error: %v", err)
+			app.Errorf("run api server error: %v", err)
 		} else {
-			log.Info("api server graceful down")
+			app.Info("api server graceful down")
 		}
 	}
 }
@@ -80,7 +81,7 @@ func (app *Api) Stop() {
 	defer cancel()
 
 	if err := app.srv.Shutdown(ctx); err != nil {
-		log.Errorf("shutdown api server error: %v", err)
+		app.Errorf("shutdown api server error: %v", err)
 	}
 }
 
@@ -100,7 +101,8 @@ func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (
 
 func newApi() *Api {
 	api := &Api{
-		app: gin.New(),
+		app:    gin.New(),
+		Logger: utils.NewLogger("Api"),
 	}
 	api.Init()
 	return api

diff --git a/core/apps/server.go b/core/apps/server.go
@@ -2,7 +2,6 @@ package apps
 
 import (
 	"fmt"
-	"github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/node/service"
 	"github.com/crawlab-team/crawlab/core/utils"
@@ -16,6 +15,9 @@ type Server struct {
 	// modules
 	nodeSvc interfaces.NodeService
 	api     *Api
+
+	// internals
+	interfaces.Logger
 }
 
 func (app *Server) Init() {
@@ -53,7 +55,7 @@ func (app *Server) GetNodeService() interfaces.NodeService {
 }
 
 func (app *Server) logNodeInfo() {
-	log.Infof("current node type: %s", utils.GetNodeType())
+	app.Infof("current node type: %s", utils.GetNodeType())
 }
 
 func (app *Server) initPprof() {
@@ -66,7 +68,9 @@ func (app *Server) initPprof() {
 
 func newServer() App {
 	// server
-	svr := &Server{}
+	svr := &Server{
+		Logger: utils.NewLogger("Server"),
+	}
 
 	// master modules
 	if utils.IsMaster() {

diff --git a/core/apps/utils.go b/core/apps/utils.go
@@ -1,11 +1,11 @@
 package apps
 
 import (
-	"fmt"
-	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab/trace"
+	"github.com/crawlab-team/crawlab/core/utils"
 )
 
+var utilsLogger = utils.NewLogger("AppsUtils")
+
 func Start(app App) {
 	start(app)
 }
@@ -19,10 +19,9 @@ func start(app App) {
 
 func initModule(name string, fn func() error) (err error) {
 	if err := fn(); err != nil {
-		log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
-		_ = trace.TraceError(err)
+		utilsLogger.Errorf("init %s error: %v", name, err)
 		panic(err)
 	}
-	log.Info(fmt.Sprintf("initialized %s successfully", name))
+	utilsLogger.Infof("initialized %s successfully", name)
 	return nil
 }
diff --git a/core/config/config.go b/core/config/config.go
@@ -2,6 +2,8 @@ package config
 
 import (
 	"errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils"
 	"strings"
 	"sync"
 
@@ -12,6 +14,7 @@ import (
 
 type Config struct {
 	Name string
+	interfaces.Logger
 }
 
 func (c *Config) Init() {
@@ -43,7 +46,7 @@ func (c *Config) Init() {
 	if err := viper.ReadInConfig(); err != nil {
 		var configFileNotFoundError viper.ConfigFileNotFoundError
 		if errors.As(err, &configFileNotFoundError) {
-			log.Warn("No config file found. Using default values.")
+			c.Warn("No config file found. Using default values.")
 		}
 	}
 
@@ -54,7 +57,7 @@ func (c *Config) Init() {
 func (c *Config) WatchConfig() {
 	viper.WatchConfig()
 	viper.OnConfigChange(func(e fsnotify.Event) {
-		log.Infof("Config file changed: %s", e.Name)
+		c.Infof("Config file changed: %s", e.Name)
 	})
 }
 
@@ -78,7 +81,9 @@ func (c *Config) initLogLevel() {
 }
 
 func newConfig() *Config {
-	return &Config{}
+	return &Config{
+		Logger: utils.NewLogger("Config"),
+	}
 }
 
 var _config *Config

diff --git a/core/controllers/base_file.go b/core/controllers/base_file.go
@@ -3,7 +3,6 @@ package controllers
 import (
 	"errors"
 	"fmt"
-	log2 "github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/fs"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/utils"
@@ -138,28 +137,28 @@ func PostBaseFileSaveFiles(rootPath string, c *gin.Context) {
 		go func(path string) {
 			file, err := c.FormFile(path)
 			if err != nil {
-				log2.Warnf("invalid file header: %s", path)
-				log2.Error(err.Error())
+				logger.Warnf("invalid file header: %s", path)
+				logger.Error(err.Error())
 				wg.Done()
 				return
 			}
 			f, err := file.Open()
 			if err != nil {
-				log2.Warnf("unable to open file: %s", path)
-				log2.Error(err.Error())
+				logger.Warnf("unable to open file: %s", path)
+				logger.Error(err.Error())
 				wg.Done()
 				return
 			}
 			fileData, err := io.ReadAll(f)
 			if err != nil {
-				log2.Warnf("unable to read file: %s", path)
-				log2.Error(err.Error())
+				logger.Warnf("unable to read file: %s", path)
+				logger.Error(err.Error())
 				wg.Done()
 				return
 			}
 			if err := fsSvc.Save(path, fileData); err != nil {
-				log2.Warnf("unable to save file: %s", path)
-				log2.Error(err.Error())
+				logger.Warnf("unable to save file: %s", path)
+				logger.Error(err.Error())
 				wg.Done()
 				return
 			}

diff --git a/core/controllers/spider.go b/core/controllers/spider.go
@@ -9,15 +9,13 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/fs"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/models/service"
 	"github.com/crawlab-team/crawlab/core/spider/admin"
 	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/crawlab-team/crawlab/db/generic"
 	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/crawlab-team/crawlab/trace"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -398,7 +396,7 @@ func DeleteSpiderById(c *gin.Context) {
 				// delete task logs
 				logPath := filepath.Join(utils.GetTaskLogPath(), id)
 				if err := os.RemoveAll(logPath); err != nil {
-					log.Warnf("failed to remove task log directory: %s", logPath)
+					logger.Warnf("failed to remove task log directory: %s", logPath)
 				}
 				wg.Done()
 			}(id.Hex())
@@ -416,12 +414,12 @@ func DeleteSpiderById(c *gin.Context) {
 			// delete spider directory
 			fsSvc, err := getSpiderFsSvcById(id)
 			if err != nil {
-				log.Errorf("failed to get spider fs service: %s", err.Error())
+				logger.Errorf("failed to get spider fs service: %v", err)
 				return
 			}
 			err = fsSvc.Delete(".")
 			if err != nil {
-				log.Errorf("failed to delete spider directory: %s", err.Error())
+				logger.Errorf("failed to delete spider directory: %v", err)
 				return
 			}
 		}()
@@ -503,7 +501,7 @@ func DeleteSpiderList(c *gin.Context) {
 				// delete task logs
 				logPath := filepath.Join(utils.GetTaskLogPath(), id)
 				if err := os.RemoveAll(logPath); err != nil {
-					log.Warnf("failed to remove task log directory: %s", logPath)
+					logger.Warnf("failed to remove task log directory: %s", logPath)
 				}
 				wg.Done()
 			}(id.Hex())
@@ -532,14 +530,12 @@ func DeleteSpiderList(c *gin.Context) {
 				// Delete spider directory
 				fsSvc, err := getSpiderFsSvcById(s.Id)
 				if err != nil {
-					log.Errorf("failed to get spider fs service: %s", err.Error())
-					trace.PrintError(err)
+					logger.Errorf("failed to get spider fs service: %v", err)
 					return
 				}
 				err = fsSvc.Delete(".")
 				if err != nil {
-					log.Errorf("failed to delete spider directory: %s", err.Error())
-					trace.PrintError(err)
+					logger.Errorf("failed to delete spider directory: %v", err)
 					return
 				}
 			}(&spiders[i])
@@ -719,8 +715,7 @@ func getSpiderFsSvc(s *models.Spider) (svc interfaces.FsService, err error) {
 func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsService, err error) {
 	s, err := service.NewModelService[models.Spider]().GetById(id)
 	if err != nil {
-		log.Errorf("failed to get spider: %s", err.Error())
-		trace.PrintError(err)
+		logger.Errorf("failed to get spider: %v", err)
 		return nil, err
 	}
 	return getSpiderFsSvc(s)

diff --git a/core/controllers/task.go b/core/controllers/task.go
@@ -2,7 +2,6 @@ package controllers
 
 import (
 	"errors"
-	log2 "github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/models/models"
@@ -191,12 +190,12 @@ func DeleteTaskById(c *gin.Context) {
 		// delete task stat
 		_, err = service.NewModelService[models.TaskStat]().GetById(id)
 		if err != nil {
-			log2.Warnf("delete task stat error: %s", err.Error())
+			logger.Warnf("delete task stat error: %s", err.Error())
 			return nil
 		}
 		err = service.NewModelService[models.TaskStat]().DeleteById(id)
 		if err != nil {
-			log2.Warnf("delete task stat error: %s", err.Error())
+			logger.Warnf("delete task stat error: %s", err.Error())
 			return nil
 		}
 
@@ -209,7 +208,7 @@ func DeleteTaskById(c *gin.Context) {
 	// delete task logs
 	logPath := filepath.Join(utils.GetTaskLogPath(), id.Hex())
 	if err := os.RemoveAll(logPath); err != nil {
-		log2.Warnf("failed to remove task log directory: %s", logPath)
+		logger.Warnf("failed to remove task log directory: %s", logPath)
 	}
 
 	HandleSuccess(c)
@@ -240,7 +239,7 @@ func DeleteList(c *gin.Context) {
 				"$in": payload.Ids,
 			},
 		}); err != nil {
-			log2.Warnf("delete task stat error: %s", err.Error())
+			logger.Warnf("delete task stat error: %s", err.Error())
 			return nil
 		}
 
@@ -258,7 +257,7 @@ func DeleteList(c *gin.Context) {
 			// delete task logs
 			logPath := filepath.Join(utils.GetTaskLogPath(), id)
 			if err := os.RemoveAll(logPath); err != nil {
-				log2.Warnf("failed to remove task log directory: %s", logPath)
+				logger.Warnf("failed to remove task log directory: %s", logPath)
 			}
 			wg.Done()
 		}(id.Hex())

diff --git a/core/controllers/utils_logger.go b/core/controllers/utils_logger.go
@@ -0,0 +1,5 @@
+package controllers
+
+import "github.com/crawlab-team/crawlab/core/utils"
+
+var logger = utils.NewLogger("Controllers")
diff --git a/core/controllers/ws_writer.go b/core/controllers/ws_writer.go
@@ -1,8 +1,6 @@
 package controllers
 
 import (
-	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab/trace"
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"io"
@@ -16,7 +14,7 @@ type WsWriter struct {
 }
 
 func (w *WsWriter) Write(data []byte) (n int, err error) {
-	log.Infof("websocket write: %s", string(data))
+	logger.Infof("websocket write: %s", string(data))
 	err = w.conn.WriteMessage(websocket.TextMessage, data)
 	if err != nil {
 		return 0, err
@@ -47,8 +45,7 @@ func NewWsWriter(c *gin.Context) (writer *WsWriter, err error) {
 
 	conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
 	if err != nil {
-		log.Errorf("websocket open connection error: %v", err)
-		trace.PrintError(err)
+		logger.Errorf("websocket open connection error: %v", err)
 	}
 
 	return &WsWriter{