From 1a3411b91ff43bef97f762f537afb722d52f77d1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 16 Jun 2024 12:24:48 +0800 Subject: [PATCH] feat: migrated git from spider --- core/controllers/router_v2.go | 27 +-- core/controllers/spider_v2.go | 429 ---------------------------------- 2 files changed, 1 insertion(+), 455 deletions(-) diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 91e42ae8b..908368bdf 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -58,7 +58,6 @@ func InitRoutes(app *gin.Engine) (err error) { RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]()) RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]()) RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) - RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]()) RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]()) RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) @@ -172,31 +171,7 @@ func InitRoutes(app *gin.Engine) (err error) { Path: "/:id/run", HandlerFunc: PostSpiderRun, }, - Action{ - Method: http.MethodGet, - Path: "/:id/git", - HandlerFunc: GetSpiderGit, - }, - Action{ - Method: http.MethodGet, - Path: "/:id/git/remote-refs", - HandlerFunc: GetSpiderGitRemoteRefs, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/checkout", - HandlerFunc: PostSpiderGitCheckout, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/pull", - HandlerFunc: PostSpiderGitPull, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/commit", - HandlerFunc: PostSpiderGitCommit, - }, + Action{ Method: http.MethodGet, Path: "/:id/data-source", diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 9e81c6a28..11f1c4913 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -5,19 +5,13 @@ import ( "fmt" log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" - "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" "github.com/gin-gonic/gin" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -26,7 +20,6 @@ import ( "math" "os" "path/filepath" - "strings" "sync" ) @@ -756,266 +749,6 @@ func PostSpiderRun(c *gin.Context) { HandleSuccessWithData(c, taskIds) } -func GetSpiderGit(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // branches - branches, err := gitClient.GetBranches() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if branches == nil || len(branches) == 0 && currentBranch != "" { - branches = []vcs.GitRef{{Name: currentBranch}} - } - - // changes - changes, err := gitClient.GetStatus() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // logs - logs, err := gitClient.GetLogsWithRefs() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // ignore - ignore, err := getSpiderGitIgnore(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // git - _git, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) - return - } - } - - // response - res := bson.M{ - "current_branch": currentBranch, - "branches": branches, - "changes": changes, - "logs": logs, - "ignore": ignore, - "git": _git, - } - - HandleSuccessWithData(c, res) -} - -func GetSpiderGitRemoteRefs(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // remote name - remoteName := c.Query("remote") - if remoteName == "" { - remoteName = vcs.GitRemoteNameOrigin - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // refs - refs, err := gitClient.GetRemoteRefs(remoteName) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, refs) -} - -func PostSpiderGitCheckout(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload struct { - Paths []string `json:"paths"` - CommitMessage string `json:"commit_message"` - Branch string `json:"branch"` - Tag string `json:"tag"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // branch to pull - var branch string - if payload.Branch == "" { - // by default current branch - branch, err = gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - // payload branch - branch = payload.Branch - } - - // checkout - if err := gitSpiderCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func PostSpiderGitPull(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload struct { - Paths []string `json:"paths"` - CommitMessage string `json:"commit_message"` - Branch string `json:"branch"` - Tag string `json:"tag"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git - g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // attempt to sync git - adminSvc, err := admin.GetSpiderAdminServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - _ = adminSvc.SyncGitOne(g) - - HandleSuccess(c) -} - -func PostSpiderGitCommit(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // add - for _, p := range payload.Paths { - if err := gitClient.Add(p); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // commit - if err := gitClient.Commit(payload.CommitMessage); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // push - if err := gitClient.Push( - vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), - ); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - func GetSpiderDataSource(c *gin.Context) { // spider id id, err := primitive.ObjectIDFromHex(c.Param("id")) @@ -1107,168 +840,6 @@ func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 { return fsSvc } -func getSpiderGitClient(id primitive.ObjectID) (client *vcs.GitClient, err error) { - // git - g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - if !errors.Is(err, mongo2.ErrNoDocuments) { - return nil, trace.TraceError(err) - } - return nil, nil - } - - // git client - workspacePath := viper.GetString("workspace") - client, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) - if err != nil { - return nil, err - } - - // set auth - utils.InitGitClientAuthV2(g, client) - - // remote name - remoteName := vcs.GitRemoteNameOrigin - - // update remote - r, err := client.GetRemote(remoteName) - if errors.Is(err, git.ErrRemoteNotFound) { - // remote not exists, create - if _, err := client.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } else if err == nil { - // remote exists, update if different - if g.Url != r.Config().URLs[0] { - if err := client.DeleteRemote(remoteName); err != nil { - return nil, trace.TraceError(err) - } - if _, err := client.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } - client.SetRemoteUrl(g.Url) - } else { - // error - return nil, trace.TraceError(err) - } - - // check if head reference exists - _, err = client.GetRepository().Head() - if err == nil { - return client, nil - } - - // align master/main branch - alignSpiderGitBranch(client) - - return client, nil -} - -func alignSpiderGitBranch(gitClient *vcs.GitClient) { - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - trace.PrintError(err) - return - } - - // skip if current branch is not master - if currentBranch != vcs.GitBranchNameMaster { - return - } - - // remote refs - refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) - if err != nil { - trace.PrintError(err) - return - } - - // main branch - defaultRemoteBranch, err := getSpiderDefaultRemoteBranch(refs) - if err != nil || defaultRemoteBranch == "" { - return - } - - // move branch - if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { - trace.PrintError(err) - } -} - -func getSpiderDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { - // remote branch name - for _, r := range refs { - if r.Type != vcs.GitRefTypeBranch { - continue - } - - if r.Name == vcs.GitBranchNameMain { - defaultRemoteBranchName = r.Name - break - } - - if r.Name == vcs.GitBranchNameMaster { - defaultRemoteBranchName = r.Name - continue - } - - if defaultRemoteBranchName == "" { - defaultRemoteBranchName = r.Name - continue - } - } - - return defaultRemoteBranchName, nil -} - -func getSpiderGitIgnore(id primitive.ObjectID) (ignore []string, err error) { - workspacePath := viper.GetString("workspace") - filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") - if !utils.Exists(filePath) { - return nil, nil - } - data, err := os.ReadFile(filePath) - if err != nil { - return nil, trace.TraceError(err) - } - ignore = strings.Split(string(data), "\n") - return ignore, nil -} - -func gitSpiderCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { - if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { - return trace.TraceError(err) - } - - // pull - return spiderGitPull(gitClient, remote, branch) -} - -func spiderGitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { - // pull - if err := gitClient.Pull( - vcs.WithRemoteNamePull(remote), - vcs.WithBranchNamePull(branch), - ); err != nil { - return trace.TraceError(err) - } - - // reset - if err := gitClient.Reset(); err != nil { - return trace.TraceError(err) - } - - return nil -} - func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { modelSvc := service.NewModelServiceV2[models.DataCollectionV2]() if s.ColId.IsZero() {