refactor: remove db module and update imports to core/mongo

- Deleted the db module, consolidating database-related functionality into the core/mongo package for better organization and maintainability. - Updated all import paths across the codebase to replace references to the removed db module with core/mongo. - Cleaned up unused code and dependencies, enhancing overall project clarity and reducing complexity. - This refactor improves the structure of the codebase by centralizing database operations and simplifying module management.
crawlab-team · Dec 25, 2024 · dc59599 · dc59599
1 parent a28ffbf
commit dc59599
Show file tree

Hide file tree

Showing 47 changed files with 126 additions and 254 deletions.
diff --git a/backend/go.mod b/backend/go.mod
@@ -4,7 +4,6 @@ go 1.22.9
 
 replace (
 	github.com/crawlab-team/crawlab/core => ../core
-	github.com/crawlab-team/crawlab/db => ../db
 	github.com/crawlab-team/crawlab/grpc => ../grpc
 	github.com/crawlab-team/crawlab/trace => ../trace
 	github.com/crawlab-team/crawlab/vcs => ../vcs
@@ -29,7 +28,6 @@ require (
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
-	github.com/crawlab-team/crawlab/db v0.0.0 // indirect
 	github.com/crawlab-team/crawlab/grpc v0.0.0 // indirect
 	github.com/crawlab-team/crawlab/trace v0.0.0 // indirect
 	github.com/crawlab-team/crawlab/vcs v0.0.0 // indirect

diff --git a/core/controllers/base.go b/core/controllers/base.go
@@ -4,7 +4,7 @@ import (
 	"errors"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/db/mongo"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/controllers/base_test.go b/core/controllers/base_test.go
@@ -8,13 +8,13 @@ import (
 	"github.com/crawlab-team/crawlab/core/middlewares"
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/user"
 	"github.com/spf13/viper"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/assert"
 )

diff --git a/core/controllers/filter.go b/core/controllers/filter.go
@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/crawlab-team/crawlab/core/entity"
-	"github.com/crawlab-team/crawlab/db/mongo"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	mongo2 "go.mongodb.org/mongo-driver/mongo"

diff --git a/core/controllers/project.go b/core/controllers/project.go
@@ -4,7 +4,7 @@ import (
 	"errors"
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/db/mongo"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/controllers/spider.go b/core/controllers/spider.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/models/models"
+	mongo2 "github.com/crawlab-team/crawlab/core/mongo"
 	"math"
 	"os"
 	"path/filepath"
@@ -14,12 +15,10 @@ import (
 	"github.com/crawlab-team/crawlab/core/models/service"
 	"github.com/crawlab-team/crawlab/core/spider/admin"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/generic"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
-	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
 func GetSpiderById(c *gin.Context) {
@@ -29,7 +28,7 @@ func GetSpiderById(c *gin.Context) {
 		return
 	}
 	s, err := service.NewModelService[models.Spider]().GetById(id)
-	if errors.Is(err, mongo2.ErrNoDocuments) {
+	if errors.Is(err, mongo.ErrNoDocuments) {
 		HandleErrorNotFound(c, err)
 		return
 	}
@@ -41,7 +40,7 @@ func GetSpiderById(c *gin.Context) {
 	// stat
 	s.Stat, err = service.NewModelService[models.SpiderStat]().GetById(s.Id)
 	if err != nil {
-		if !errors.Is(err, mongo2.ErrNoDocuments) {
+		if !errors.Is(err, mongo.ErrNoDocuments) {
 			HandleErrorInternalServerError(c, err)
 			return
 		}
@@ -51,7 +50,7 @@ func GetSpiderById(c *gin.Context) {
 	if s.ColName == "" && !s.ColId.IsZero() {
 		col, err := service.NewModelService[models.DataCollection]().GetById(s.ColId)
 		if err != nil {
-			if !errors.Is(err, mongo2.ErrNoDocuments) {
+			if !errors.Is(err, mongo.ErrNoDocuments) {
 				HandleErrorInternalServerError(c, err)
 				return
 			}
@@ -64,7 +63,7 @@ func GetSpiderById(c *gin.Context) {
 	if utils.IsPro() && !s.GitId.IsZero() {
 		s.Git, err = service.NewModelService[models.Git]().GetById(s.GitId)
 		if err != nil {
-			if !errors.Is(err, mongo2.ErrNoDocuments) {
+			if !errors.Is(err, mongo.ErrNoDocuments) {
 				HandleErrorInternalServerError(c, err)
 				return
 			}
@@ -100,13 +99,13 @@ func getSpiderListWithStats(c *gin.Context) {
 	sort := MustGetSortOption(c)
 
 	// get list
-	spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo.FindOptions{
+	spiders, err := service.NewModelService[models.Spider]().GetMany(query, &mongo2.FindOptions{
 		Sort:  sort,
 		Skip:  pagination.Size * (pagination.Page - 1),
 		Limit: pagination.Size,
 	})
 	if err != nil {
-		if err.Error() != mongo2.ErrNoDocuments.Error() {
+		if err.Error() != mongo.ErrNoDocuments.Error() {
 			HandleErrorInternalServerError(c, err)
 		}
 		return
@@ -347,7 +346,7 @@ func DeleteSpiderById(c *gin.Context) {
 		return
 	}
 
-	if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
+	if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) {
 		// delete spider
 		err = service.NewModelService[models.Spider]().DeleteById(id)
 		if err != nil {
@@ -448,7 +447,7 @@ func DeleteSpiderList(c *gin.Context) {
 		return
 	}
 
-	if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
+	if err := mongo2.RunTransaction(func(context mongo.SessionContext) (err error) {
 		// delete spiders
 		if err := service.NewModelService[models.Spider]().DeleteMany(bson.M{
 			"_id": bson.M{
@@ -683,11 +682,11 @@ func GetSpiderResults(c *gin.Context) {
 	pagination := MustGetPagination(c)
 	query := getResultListQuery(c)
 
-	col := mongo.GetMongoCol(s.ColName)
+	col := mongo2.GetMongoCol(s.ColName)
 
 	var results []bson.M
-	err = col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(&generic.ListOptions{
-		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+	err = col.Find(mongo2.GetMongoQuery(query), mongo2.GetMongoOpts(&mongo2.ListOptions{
+		Sort:  []mongo2.ListSort{{"_id", mongo2.SortDirectionDesc}},
 		Skip:  pagination.Size * (pagination.Page - 1),
 		Limit: pagination.Size,
 	})).All(&results)
@@ -696,7 +695,7 @@ func GetSpiderResults(c *gin.Context) {
 		return
 	}
 
-	total, err := mongo.GetMongoCol(s.ColName).Count(utils.GetMongoQuery(query))
+	total, err := mongo2.GetMongoCol(s.ColName).Count(mongo2.GetMongoQuery(query))
 	if err != nil {
 		HandleErrorInternalServerError(c, err)
 		return

diff --git a/core/controllers/task.go b/core/controllers/task.go
@@ -6,11 +6,11 @@ import (
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
+	mongo3 "github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/spider/admin"
 	"github.com/crawlab-team/crawlab/core/task/log"
 	"github.com/crawlab-team/crawlab/core/task/scheduler"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -80,7 +80,7 @@ func GetTaskList(c *gin.Context) {
 	sort := MustGetSortOption(c)
 
 	// get tasks
-	tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo.FindOptions{
+	tasks, err := service.NewModelService[models.Task]().GetMany(query, &mongo3.FindOptions{
 		Sort:  sort,
 		Skip:  pagination.Size * (pagination.Page - 1),
 		Limit: pagination.Size,
@@ -176,7 +176,7 @@ func DeleteTaskById(c *gin.Context) {
 	}
 
 	// delete in db
-	if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) {
+	if err := mongo3.RunTransaction(func(context mongo2.SessionContext) (err error) {
 		// delete task
 		_, err = service.NewModelService[models.Task]().GetById(id)
 		if err != nil {
@@ -223,7 +223,7 @@ func DeleteList(c *gin.Context) {
 		return
 	}
 
-	if err := mongo.RunTransaction(func(context mongo2.SessionContext) error {
+	if err := mongo3.RunTransaction(func(context mongo2.SessionContext) error {
 		// delete tasks
 		if err := service.NewModelService[models.Task]().DeleteMany(bson.M{
 			"_id": bson.M{

diff --git a/core/controllers/user.go b/core/controllers/user.go
@@ -3,12 +3,12 @@ package controllers
 import (
 	"errors"
 	"fmt"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"regexp"
 
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/controllers/utils_filter.go b/core/controllers/utils_filter.go
@@ -5,8 +5,8 @@ import (
 	errors2 "errors"
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/generic"
 	"github.com/gin-gonic/gin"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -125,13 +125,13 @@ func MustGetFilterAll(c *gin.Context) (res bool) {
 	return res
 }
 
-func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
+func getResultListQuery(c *gin.Context) (q mongo.ListQuery) {
 	f, err := GetFilter(c)
 	if err != nil {
 		return q
 	}
 	for _, cond := range f.Conditions {
-		q = append(q, generic.ListQueryCondition{
+		q = append(q, mongo.ListQueryCondition{
 			Key:   cond.Key,
 			Op:    cond.Op,
 			Value: utils.NormalizeObjectId(cond.Value),

diff --git a/core/export/csv_service.go b/core/export/csv_service.go
@@ -9,8 +9,8 @@ import (
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/entity"
 	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/hashicorp/go-uuid"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/export/csv_service_test.go b/core/export/csv_service_test.go
@@ -4,7 +4,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/db/mongo"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/export/json_service.go b/core/export/json_service.go
@@ -8,8 +8,8 @@ import (
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/entity"
 	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/hashicorp/go-uuid"
 	mongo2 "go.mongodb.org/mongo-driver/mongo"
 	"os"

diff --git a/core/go.mod b/core/go.mod
@@ -3,7 +3,6 @@ module github.com/crawlab-team/crawlab/core
 go 1.22.9
 
 replace (
-	github.com/crawlab-team/crawlab/db => ../db
 	github.com/crawlab-team/crawlab/grpc => ../grpc
 	github.com/crawlab-team/crawlab/trace => ../trace
 	github.com/crawlab-team/crawlab/vcs => ../vcs
@@ -14,7 +13,6 @@ require (
 	github.com/ReneKroon/ttlcache v1.7.0
 	github.com/apex/log v1.9.0
 	github.com/cenkalti/backoff/v4 v4.3.0
-	github.com/crawlab-team/crawlab/db v0.0.0
 	github.com/crawlab-team/crawlab/grpc v0.0.0
 	github.com/crawlab-team/crawlab/trace v0.0.0
 	github.com/crawlab-team/crawlab/vcs v0.0.0

diff --git a/core/grpc/server/dependency_service_server.go b/core/grpc/server/dependency_service_server.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/cenkalti/backoff/v4"
 	"github.com/crawlab-team/crawlab/core/interfaces"
+	mongo2 "github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/utils"
 	"io"
 	"sync"
@@ -14,7 +15,6 @@ import (
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
-	mongo2 "github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/crawlab-team/crawlab/grpc"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/grpc/server/model_base_service_server.go b/core/grpc/server/model_base_service_server.go
@@ -3,14 +3,14 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/mongo"
 	"github.com/crawlab-team/crawlab/core/utils"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"reflect"
 	"sync"
 
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/crawlab-team/crawlab/grpc"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

diff --git a/core/grpc/server/task_service_server.go b/core/grpc/server/task_service_server.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	mongo3 "github.com/crawlab-team/crawlab/core/mongo"
 	"io"
 	"strings"
 	"sync"
@@ -17,7 +18,6 @@ import (
 	"github.com/crawlab-team/crawlab/core/notification"
 	"github.com/crawlab-team/crawlab/core/task/stats"
 	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/crawlab-team/crawlab/grpc"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -147,14 +147,14 @@ func (svr TaskServiceServer) FetchTask(ctx context.Context, request *grpc.TaskSe
 		return nil, err
 	}
 	var tid primitive.ObjectID
-	opts := &mongo.FindOptions{
+	opts := &mongo3.FindOptions{
 		Sort: bson.D{
 			{"priority", 1},
 			{"_id", 1},
 		},
 		Limit: 1,
 	}
-	if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) {
+	if err := mongo3.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) {
 		// fetch task for the given node
 		t, err := service.NewModelService[models.Task]().GetOne(bson.M{
 			"node_id": n.Id,

diff --git a/core/interfaces/result_service.go b/core/interfaces/result_service.go
diff --git a/core/interfaces/result_service_registry.go b/core/interfaces/result_service_registry.go