From 059711a533f5447f10939a06fff519be47b2cb4b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 5 Jul 2024 12:57:09 +0800 Subject: [PATCH] fix: unable to sync files from spiders created from sub-folder of git repo --- core/task/handler/runner_v2.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 46789bbfd..885b1c018 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -18,7 +18,7 @@ import ( service2 "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" "github.com/spf13/viper" @@ -414,7 +414,7 @@ func (r *RunnerV2) syncFiles() (err error) { } } else { log.Infof("File needs to be synchronized: %s", path) - _err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(r.cwd, path), masterFile) + _err := r.downloadFile(masterURL+"/download?path="+filepath.Join(workingDir, path), filepath.Join(r.cwd, path), masterFile) if _err != nil { log.Errorf("Error downloading file: %v", _err) err = errors.Join(err, _err) @@ -441,6 +441,10 @@ func (r *RunnerV2) downloadFile(url string, filePath string, fileInfo *entity.Fs log.Errorf("Error getting file response: %v", err) return err } + if resp.StatusCode != http.StatusOK { + log.Errorf("Error downloading file: %s", resp.Status) + return errors.New(resp.Status) + } defer resp.Body.Close() // create directory if not exists