diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 151cc946e..000000000 --- a/.gitmodules +++ /dev/null @@ -1,15 +0,0 @@ -[submodule "backend/core"] - path = backend/core - url = https://github.com/crawlab-team/crawlab-core -[submodule "backend/db"] - path = backend/db - url = https://github.com/crawlab-team/crawlab-db -[submodule "backend/fs"] - path = backend/fs - url = https://github.com/crawlab-team/crawlab-fs -[submodule "backend/vcs"] - path = backend/vcs - url = https://github.com/crawlab-team/crawlab-vcs -[submodule "backend/log"] - path = backend/log - url = https://github.com/crawlab-team/crawlab-log diff --git a/backend/go.mod b/backend/go.mod index 4908d6da6..861f29191 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -20,13 +20,13 @@ require ( github.com/cenkalti/backoff/v4 v4.1.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.3 // indirect - github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 // indirect - github.com/crawlab-team/crawlab-fs v0.6.3 // indirect + github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 // indirect + github.com/crawlab-team/crawlab/fs v0.6.3 // indirect github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c // indirect github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 // indirect - github.com/crawlab-team/go-trace v0.1.1 // indirect + github.com/crawlab-team/crawlab/vcs v0.1.1 // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect - github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e // indirect + github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect diff --git a/backend/go.mod.dev b/backend/go.mod.dev index 9aa361a4d..f45ffe255 100644 --- a/backend/go.mod.dev +++ b/backend/go.mod.dev @@ -5,14 +5,14 @@ go 1.16 replace ( github.com/crawlab-team/crawlab-core => ../../crawlab-core github.com/crawlab-team/crawlab-vcs => ../../crawlab-vcs - github.com/crawlab-team/crawlab-fs => ../../crawlab-fs - github.com/crawlab-team/crawlab-db => ../../crawlab-db + github.com/crawlab-team/crawlab/fs => ../../crawlab-fs + github.com/crawlab-team/crawlab/db => ../../crawlab-db ) require ( github.com/apex/log v1.9.0 github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 - github.com/crawlab-team/go-trace v0.1.1 + github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 diff --git a/backend/go.mod.local b/backend/go.mod.local index e36ac67c1..ff0ab04c5 100644 --- a/backend/go.mod.local +++ b/backend/go.mod.local @@ -5,14 +5,14 @@ go 1.16 replace ( github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs - github.com/crawlab-team/crawlab-fs => /libs/crawlab-team/crawlab-fs - github.com/crawlab-team/crawlab-db => /libs/crawlab-team/crawlab-db + github.com/crawlab-team/crawlab/fs => /libs/crawlab-team/crawlab-fs + github.com/crawlab-team/crawlab/db => /libs/crawlab-team/crawlab-db ) require ( github.com/apex/log v1.9.0 github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 - github.com/crawlab-team/go-trace v0.1.1 + github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 diff --git a/backend/go.sum b/backend/go.sum index b73089f2e..4e7bd4b37 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -155,24 +155,24 @@ github.com/crawlab-team/crawlab-core v0.6.3-0.20231021045242-07956209f653 h1:uOC github.com/crawlab-team/crawlab-core v0.6.3-0.20231021045242-07956209f653/go.mod h1:HNuAjSVVZpHhpyUP4k1F2YKxEiarZESFolSyx7YjgZ8= github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203 h1:nyANfzoPgTSYJxuTye1uj44An8Cjou9QmcKRES7Gdwg= github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203/go.mod h1:HNuAjSVVZpHhpyUP4k1F2YKxEiarZESFolSyx7YjgZ8= -github.com/crawlab-team/crawlab-db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= -github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73 h1:xIgfVPa3ZJWC72Y57oHS41n4jRtGZPn1YDEYBgMj2EU= -github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= -github.com/crawlab-team/crawlab-fs v0.6.3 h1:mS91sYu+tOPavjYvt4CZ8YwY5okEiwCAuyx/5RIbXJY= -github.com/crawlab-team/crawlab-fs v0.6.3/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= +github.com/crawlab-team/crawlab/db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= +github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= +github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= +github.com/crawlab-team/crawlab/fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73 h1:xIgfVPa3ZJWC72Y57oHS41n4jRtGZPn1YDEYBgMj2EU= +github.com/crawlab-team/crawlab/fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= +github.com/crawlab-team/crawlab/fs v0.6.3 h1:mS91sYu+tOPavjYvt4CZ8YwY5okEiwCAuyx/5RIbXJY= +github.com/crawlab-team/crawlab/fs v0.6.3/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c h1:jX0iax3WHwomWGQVWrCTy8a4zYDsKKyuspP3+04XCcU= github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c/go.mod h1:Bq2Pm967EYWbjhP5Ghc4DV2LZgbOLMzLftJXDJYz/gs= github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 h1:A/XSUuGgGMn+z+lFd2ye2ClgIKhDZYUerhOL5jePQhU= github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I= -github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= -github.com/crawlab-team/go-trace v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= -github.com/crawlab-team/go-trace v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= +github.com/crawlab-team/crawlab/vcs v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= +github.com/crawlab-team/crawlab/vcs v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= +github.com/crawlab-team/crawlab/vcs v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= -github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= +github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= +github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/core/apps/docker.go b/core/apps/docker.go index 948a5cff9..33ad57031 100644 --- a/core/apps/docker.go +++ b/core/apps/docker.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/imroc/req" "github.com/spf13/viper" "os" diff --git a/core/apps/utils.go b/core/apps/utils.go index 69f880a45..cd10edab3 100644 --- a/core/apps/utils.go +++ b/core/apps/utils.go @@ -19,7 +19,7 @@ import ( taskstats "github.com/crawlab-team/crawlab/core/task/stats" "github.com/crawlab-team/crawlab/core/user" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func Start(app App) { diff --git a/core/color/service.go b/core/color/service.go index 3b4cbadc4..cf32a535d 100644 --- a/core/color/service.go +++ b/core/color/service.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "math/rand" "strconv" "strings" diff --git a/core/config/config.go b/core/config/config.go index 733d03e79..1caa14d7f 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -3,7 +3,7 @@ package config import ( "bytes" "github.com/apex/log" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/fsnotify/fsnotify" "github.com/spf13/viper" "strings" diff --git a/core/controllers/data_source.go b/core/controllers/data_source.go index 167dc284a..a4b905655 100644 --- a/core/controllers/data_source.go +++ b/core/controllers/data_source.go @@ -10,7 +10,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/controllers/delegate_list.go b/core/controllers/delegate_list.go index 1d2bd8711..403a89301 100644 --- a/core/controllers/delegate_list.go +++ b/core/controllers/delegate_list.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/delegate" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/demo.go b/core/controllers/demo.go index 9c25aab4a..7a09d9889 100644 --- a/core/controllers/demo.go +++ b/core/controllers/demo.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "net/http" ) diff --git a/core/controllers/node.go b/core/controllers/node.go index 1ea748b26..236ebafcf 100644 --- a/core/controllers/node.go +++ b/core/controllers/node.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/delegate" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "github.com/google/uuid" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/spider.go b/core/controllers/spider.go index 29ede12b4..f05da3729 100644 --- a/core/controllers/spider.go +++ b/core/controllers/spider.go @@ -3,7 +3,6 @@ package controllers import ( "bytes" "fmt" - vcs "github.com/crawlab-team/crawlab-vcs" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/entity" @@ -15,7 +14,8 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" + "github.com/crawlab-team/crawlab/vcs" "github.com/gin-gonic/gin" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 9214ab1f7..9e81c6a28 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" log2 "github.com/apex/log" - vcs "github.com/crawlab-team/crawlab-vcs" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/fs" @@ -14,7 +13,8 @@ import ( "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" + "github.com/crawlab-team/crawlab/vcs" "github.com/gin-gonic/gin" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" diff --git a/core/controllers/test/base.go b/core/controllers/test/base.go index fe8b18356..1e3c2906a 100644 --- a/core/controllers/test/base.go +++ b/core/controllers/test/base.go @@ -4,7 +4,7 @@ import ( "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/routes" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gavv/httpexpect/v2" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" diff --git a/core/controllers/user.go b/core/controllers/user.go index e243b02c7..5c22a7293 100644 --- a/core/controllers/user.go +++ b/core/controllers/user.go @@ -11,7 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/utils_http.go b/core/controllers/utils_http.go index 60ce95eba..4726f2886 100644 --- a/core/controllers/utils_http.go +++ b/core/controllers/utils_http.go @@ -3,7 +3,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "net/http" ) diff --git a/core/ds/cockroachdb.go b/core/ds/cockroachdb.go index c7777eea9..44aca2630 100644 --- a/core/ds/cockroachdb.go +++ b/core/ds/cockroachdb.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/core/ds/es.go b/core/ds/es.go index f078a54aa..e3910d195 100644 --- a/core/ds/es.go +++ b/core/ds/es.go @@ -14,7 +14,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esapi" "github.com/google/uuid" diff --git a/core/ds/kafka.go b/core/ds/kafka.go index 0d8a416cb..4931b6326 100644 --- a/core/ds/kafka.go +++ b/core/ds/kafka.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/segmentio/kafka-go" "go.mongodb.org/mongo-driver/bson/primitive" "time" diff --git a/core/ds/mssql.go b/core/ds/mssql.go index 4adc73896..f73e4be65 100644 --- a/core/ds/mssql.go +++ b/core/ds/mssql.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/core/ds/postgresql.go b/core/ds/postgresql.go index 06aac13f4..174d004be 100644 --- a/core/ds/postgresql.go +++ b/core/ds/postgresql.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/core/ds/service.go b/core/ds/service.go index cf57a92bc..9991e4d56 100644 --- a/core/ds/service.go +++ b/core/ds/service.go @@ -12,7 +12,7 @@ import ( "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/utils" utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "sync" diff --git a/core/ds/sql.go b/core/ds/sql.go index 3fd1bc18c..1fa7b100e 100644 --- a/core/ds/sql.go +++ b/core/ds/sql.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" utils2 "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/upper/db/v4" "time" ) diff --git a/core/ds/sqlite.go b/core/ds/sqlite.go index 43fdb6f72..9e682cd5b 100644 --- a/core/ds/sqlite.go +++ b/core/ds/sqlite.go @@ -5,7 +5,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/core/entity/grpc_base_service_message.go b/core/entity/grpc_base_service_message.go index 6d51b1eff..243f74203 100644 --- a/core/entity/grpc_base_service_message.go +++ b/core/entity/grpc_base_service_message.go @@ -3,7 +3,7 @@ package entity import ( "encoding/json" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) type GrpcBaseServiceMessage struct { diff --git a/core/entity/grpc_delegate_message.go b/core/entity/grpc_delegate_message.go index 2760e7309..dd8e44255 100644 --- a/core/entity/grpc_delegate_message.go +++ b/core/entity/grpc_delegate_message.go @@ -3,7 +3,7 @@ package entity import ( "encoding/json" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) type GrpcDelegateMessage struct { diff --git a/core/entity/result.go b/core/entity/result.go index db0970c7f..dd8cc6903 100644 --- a/core/entity/result.go +++ b/core/entity/result.go @@ -3,7 +3,7 @@ package entity import ( "encoding/json" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/core/event/service.go b/core/event/service.go index 8fdf93397..892cea9aa 100644 --- a/core/event/service.go +++ b/core/event/service.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/thoas/go-funk" "regexp" ) diff --git a/core/export/csv_service.go b/core/export/csv_service.go index 6444a3063..666cc331c 100644 --- a/core/export/csv_service.go +++ b/core/export/csv_service.go @@ -12,7 +12,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/hashicorp/go-uuid" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/export/json_service.go b/core/export/json_service.go index 44ea0169d..5d56268d7 100644 --- a/core/export/json_service.go +++ b/core/export/json_service.go @@ -11,7 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/hashicorp/go-uuid" mongo2 "go.mongodb.org/mongo-driver/mongo" "os" diff --git a/core/go.mod b/core/go.mod index 6f43f36a1..35a77a516 100644 --- a/core/go.mod +++ b/core/go.mod @@ -3,20 +3,24 @@ module github.com/crawlab-team/crawlab/core go 1.22 replace ( - github.com/crawlab-team/crawlab/grpc => ../grpc github.com/crawlab-team/crawlab/db => ../db + github.com/crawlab-team/crawlab/grpc => ../grpc + github.com/crawlab-team/crawlab/template-parser => ../template-parser + github.com/crawlab-team/crawlab/trace => ../trace + github.com/crawlab-team/crawlab/vcs => ../vcs + github.com/crawlab-team/crawlab/fs => ../fs ) require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/db v0.0.0 - github.com/crawlab-team/crawlab-fs v0.6.3-2 + github.com/crawlab-team/crawlab/fs v0.6.3-2 + github.com/crawlab-team/crawlab/db v0.6.0-1 github.com/crawlab-team/crawlab/grpc v0.0.0 - github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 - github.com/crawlab-team/go-trace v0.1.1 - github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e + github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e + github.com/crawlab-team/crawlab/trace v0.1.0 + github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.5.1 @@ -53,7 +57,7 @@ require ( ) require ( - cloud.google.com/go v0.99.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -67,7 +71,6 @@ require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/crawlab-team/goseaweedfs v0.6.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect @@ -87,7 +90,7 @@ require ( github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/gorilla/css v1.0.0 // indirect @@ -110,8 +113,8 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect github.com/magiconair/properties v1.8.5 // indirect @@ -127,7 +130,7 @@ require ( github.com/olekukonko/tablewriter v0.0.1 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect @@ -137,7 +140,7 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/smartystreets/assertions v1.0.0 // indirect - github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -167,13 +170,13 @@ require ( github.com/ztrue/tracerr v0.4.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.12.0 // indirect + golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.13.0 // indirect - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect + golang.org/x/tools v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/core/go.sum b/core/go.sum index 0e42c1e31..a6e708a6d 100644 --- a/core/go.sum +++ b/core/go.sum @@ -3,6 +3,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,6 +16,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -34,9 +36,12 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -47,6 +52,7 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -101,13 +107,11 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= @@ -143,28 +147,17 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/crawlab/db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= -github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab-fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/jK8p0L9HhA= -github.com/crawlab-team/crawlab-fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0= -github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 h1:A/XSUuGgGMn+z+lFd2ye2ClgIKhDZYUerhOL5jePQhU= -github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I= -github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= -github.com/crawlab-team/go-trace v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= -github.com/crawlab-team/go-trace v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= +github.com/crawlab-team/crawlab/fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/jK8p0L9HhA= +github.com/crawlab-team/crawlab/fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0= +github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= +github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= -github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= -github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -174,8 +167,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= @@ -203,7 +194,6 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -246,8 +236,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -259,14 +247,12 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -300,10 +286,9 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -336,6 +321,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -351,21 +337,17 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -379,27 +361,22 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= @@ -474,9 +451,6 @@ github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -490,16 +464,15 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -517,17 +490,14 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -552,7 +522,6 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -560,17 +529,13 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= @@ -585,27 +550,23 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= @@ -614,35 +575,30 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -689,24 +645,19 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= @@ -738,7 +689,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -750,7 +700,6 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -771,10 +720,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= @@ -788,7 +735,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= @@ -811,11 +757,9 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -861,7 +805,7 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -869,6 +813,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= @@ -913,16 +858,14 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -954,6 +897,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -1010,7 +954,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1056,6 +999,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1063,6 +1007,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1082,7 +1027,6 @@ golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1118,7 +1062,6 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1130,7 +1073,6 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1142,7 +1084,6 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1177,6 +1118,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1185,8 +1127,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1269,7 +1211,9 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1296,8 +1240,9 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1357,19 +1302,16 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/core/grpc/client/client.go b/core/grpc/client/client.go index 15a6c5e34..6767e8d74 100644 --- a/core/grpc/client/client.go +++ b/core/grpc/client/client.go @@ -13,7 +13,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" diff --git a/core/grpc/client/client_v2.go b/core/grpc/client/client_v2.go index 8a55a3049..fd40cc9c4 100644 --- a/core/grpc/client/client_v2.go +++ b/core/grpc/client/client_v2.go @@ -13,7 +13,7 @@ import ( nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/utils" grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" diff --git a/core/grpc/client/pool.go b/core/grpc/client/pool.go index 5ecd8dadb..6559610ce 100644 --- a/core/grpc/client/pool.go +++ b/core/grpc/client/pool.go @@ -3,7 +3,7 @@ package client import ( "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/emirpasic/gods/lists/arraylist" "math/rand" ) diff --git a/core/grpc/server/message_server.go b/core/grpc/server/message_server.go index d39893415..ea29df090 100644 --- a/core/grpc/server/message_server.go +++ b/core/grpc/server/message_server.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/service" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "io" ) diff --git a/core/grpc/server/model_base_service_binder.go b/core/grpc/server/model_base_service_binder.go index 20244dbf1..c9f7331e9 100644 --- a/core/grpc/server/model_base_service_binder.go +++ b/core/grpc/server/model_base_service_binder.go @@ -5,7 +5,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func NewModelBaseServiceBinder(req *grpc.Request) (b *ModelBaseServiceBinder) { diff --git a/core/grpc/server/model_base_service_server.go b/core/grpc/server/model_base_service_server.go index 7f373affc..661c9505a 100644 --- a/core/grpc/server/model_base_service_server.go +++ b/core/grpc/server/model_base_service_server.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) type ModelBaseServiceServer struct { diff --git a/core/grpc/server/server.go b/core/grpc/server/server.go index 6e7e83994..5639e6416 100644 --- a/core/grpc/server/server.go +++ b/core/grpc/server/server.go @@ -12,7 +12,7 @@ import ( "github.com/crawlab-team/crawlab/core/grpc/middlewares" "github.com/crawlab-team/crawlab/core/interfaces" grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/grpc-ecosystem/go-grpc-middleware" grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" "github.com/grpc-ecosystem/go-grpc-middleware/recovery" diff --git a/core/grpc/server/server_v2.go b/core/grpc/server/server_v2.go index 3174bdafe..ebcb871b7 100644 --- a/core/grpc/server/server_v2.go +++ b/core/grpc/server/server_v2.go @@ -11,7 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" diff --git a/core/grpc/server/task_server.go b/core/grpc/server/task_server.go index fc3230b09..b6d085923 100644 --- a/core/grpc/server/task_server.go +++ b/core/grpc/server/task_server.go @@ -16,7 +16,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index 5edfbdf44..f1c5585e1 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -16,7 +16,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/grpc/server/utils_handle.go b/core/grpc/server/utils_handle.go index c44ac20f0..d860360a3 100644 --- a/core/grpc/server/utils_handle.go +++ b/core/grpc/server/utils_handle.go @@ -3,7 +3,7 @@ package server import ( "encoding/json" "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func HandleError(err error) (res *grpc.Response, err2 error) { diff --git a/core/interfaces/fs_service.go b/core/interfaces/fs_service.go index f856c978a..001e978d5 100644 --- a/core/interfaces/fs_service.go +++ b/core/interfaces/fs_service.go @@ -1,8 +1,8 @@ package interfaces import ( - cfs "github.com/crawlab-team/crawlab-fs" - vcs "github.com/crawlab-team/crawlab-vcs" + cfs "github.com/crawlab-team/crawlab/fs" + vcs "github.com/crawlab-team/crawlab/vcs" ) type FsService interface { diff --git a/core/models/client/binder_basic.go b/core/models/client/binder_basic.go index 06e675a74..4d4b47164 100644 --- a/core/models/client/binder_basic.go +++ b/core/models/client/binder_basic.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func NewBasicBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelBinder) { diff --git a/core/models/client/binder_list.go b/core/models/client/binder_list.go index 4f81980c6..fe5788044 100644 --- a/core/models/client/binder_list.go +++ b/core/models/client/binder_list.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func NewListBinder(id interfaces.ModelId, res *grpc.Response) (b interfaces.GrpcModelListBinder) { diff --git a/core/models/client/model_base_service.go b/core/models/client/model_base_service.go index 0beeb218a..6ee6b9dd2 100644 --- a/core/models/client/model_base_service.go +++ b/core/models/client/model_base_service.go @@ -11,7 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" errors2 "github.com/pkg/errors" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" diff --git a/core/models/client/model_delegate.go b/core/models/client/model_delegate.go index a42db1cd3..d59524ca4 100644 --- a/core/models/client/model_delegate.go +++ b/core/models/client/model_delegate.go @@ -10,7 +10,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" ) diff --git a/core/models/delegate/model.go b/core/models/delegate/model.go index f27b0b186..3e9d8139e 100644 --- a/core/models/delegate/model.go +++ b/core/models/delegate/model.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/errors" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/models/models/utils_tag.go b/core/models/models/utils_tag.go index bdcb80461..c5c070c3e 100644 --- a/core/models/models/utils_tag.go +++ b/core/models/models/utils_tag.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func convertInterfacesToTags(tags []interfaces.Tag) (res []Tag) { diff --git a/core/models/service/base_service.go b/core/models/service/base_service.go index 087e942ca..20d2362a0 100644 --- a/core/models/service/base_service.go +++ b/core/models/service/base_service.go @@ -10,7 +10,7 @@ import ( models2 "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "reflect" diff --git a/core/models/service/binder_list.go b/core/models/service/binder_list.go index ade6cdf15..a7a8f50d5 100644 --- a/core/models/service/binder_list.go +++ b/core/models/service/binder_list.go @@ -5,7 +5,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" ) func NewListBinder(id interfaces.ModelId, fr *mongo.FindResult) (b interfaces.ModelListBinder) { diff --git a/core/models/service/tag_service.go b/core/models/service/tag_service.go index 362b3fafa..9013b0221 100644 --- a/core/models/service/tag_service.go +++ b/core/models/service/tag_service.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/delegate" models2 "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go index 03b47ba9e..7e05adde4 100644 --- a/core/node/config/config_service.go +++ b/core/node/config/config_service.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "os" "path" ) diff --git a/core/node/service/master_service.go b/core/node/service/master_service.go index e2d42139b..3ade5da19 100644 --- a/core/node/service/master_service.go +++ b/core/node/service/master_service.go @@ -18,7 +18,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index f6da2f533..1ca7a52aa 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -21,7 +21,7 @@ import ( "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/node/service/worker_service.go b/core/node/service/worker_service.go index 3df9895ab..cf91625ab 100644 --- a/core/node/service/worker_service.go +++ b/core/node/service/worker_service.go @@ -11,7 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "time" ) diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index 155c35dd4..0134c54db 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -12,7 +12,7 @@ import ( "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "time" ) diff --git a/core/notification/mobile.go b/core/notification/mobile.go index cd36641b0..90a238705 100644 --- a/core/notification/mobile.go +++ b/core/notification/mobile.go @@ -2,7 +2,7 @@ package notification import ( "errors" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/imroc/req" "strings" ) diff --git a/core/notification/service.go b/core/notification/service.go index 97beba449..37a7f93d0 100644 --- a/core/notification/service.go +++ b/core/notification/service.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" - parser "github.com/crawlab-team/template-parser" + parser "github.com/crawlab-team/crawlab/template-parser" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 0184e2cf3..6db6a01a5 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" mongo2 "github.com/crawlab-team/crawlab/db/mongo" - parser "github.com/crawlab-team/template-parser" + parser "github.com/crawlab-team/crawlab/template-parser" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" diff --git a/core/process/daemon.go b/core/process/daemon.go index 0a5395c42..058880dd2 100644 --- a/core/process/daemon.go +++ b/core/process/daemon.go @@ -5,7 +5,7 @@ import ( "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/sys_exec" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "math/rand" "os/exec" "time" diff --git a/core/process/manage.go b/core/process/manage.go index d2b050136..0869ac33e 100644 --- a/core/process/manage.go +++ b/core/process/manage.go @@ -1,7 +1,7 @@ package process import ( - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "os/exec" "regexp" "runtime" diff --git a/core/result/service.go b/core/result/service.go index 3acd63219..d1e946851 100644 --- a/core/result/service.go +++ b/core/result/service.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" "sync" ) diff --git a/core/result/service_mongo.go b/core/result/service_mongo.go index 13765beeb..504e28270 100644 --- a/core/result/service_mongo.go +++ b/core/result/service_mongo.go @@ -1,6 +1,7 @@ package result import ( + "github.com/crawlab-team/crawlab/trace" "time" "github.com/crawlab-team/crawlab/core/constants" @@ -10,7 +11,6 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/schedule/logger.go b/core/schedule/logger.go index fa6bf8340..f1e18156d 100644 --- a/core/schedule/logger.go +++ b/core/schedule/logger.go @@ -3,7 +3,7 @@ package schedule import ( "fmt" "github.com/apex/log" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/robfig/cron/v3" "strings" ) diff --git a/core/schedule/service.go b/core/schedule/service.go index f4b978da5..1238700bf 100644 --- a/core/schedule/service.go +++ b/core/schedule/service.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/robfig/cron/v3" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go index c1cffbefc..e7c4f51d3 100644 --- a/core/schedule/service_v2.go +++ b/core/schedule/service_v2.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/robfig/cron/v3" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/spider/admin/service.go b/core/spider/admin/service.go index 83d6ac9e8..b4d13480b 100644 --- a/core/spider/admin/service.go +++ b/core/spider/admin/service.go @@ -3,7 +3,6 @@ package admin import ( "context" "github.com/apex/log" - vcs "github.com/crawlab-team/crawlab-vcs" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/container" @@ -12,7 +11,8 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" + "github.com/crawlab-team/crawlab/vcs" "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/spf13/viper" diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index 73a845965..c62a5fee9 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -3,7 +3,6 @@ package admin import ( "context" "github.com/apex/log" - vcs "github.com/crawlab-team/crawlab-vcs" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" @@ -13,7 +12,8 @@ import ( "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" + "github.com/crawlab-team/crawlab/vcs" "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/spf13/viper" diff --git a/core/sys_exec/sys_exec.go b/core/sys_exec/sys_exec.go index 397e4357e..010270848 100644 --- a/core/sys_exec/sys_exec.go +++ b/core/sys_exec/sys_exec.go @@ -2,7 +2,7 @@ package sys_exec import ( "bufio" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" "os/exec" "time" diff --git a/core/task/base.go b/core/task/base.go index 265186a3a..90de0ac22 100644 --- a/core/task/base.go +++ b/core/task/base.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/delegate" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" ) diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go index b848bce14..b39d9445c 100644 --- a/core/task/handler/runner.go +++ b/core/task/handler/runner.go @@ -20,7 +20,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" "github.com/sirupsen/logrus" "github.com/spf13/viper" diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 5818f25c9..7424a2c18 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -19,7 +19,7 @@ import ( "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" "github.com/sirupsen/logrus" "github.com/spf13/viper" diff --git a/core/task/handler/service.go b/core/task/handler/service.go index 84a929ada..69a99884a 100644 --- a/core/task/handler/service.go +++ b/core/task/handler/service.go @@ -13,7 +13,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/delegate" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index 397d5f112..ad0dcdd9c 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -13,7 +13,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "sync" diff --git a/core/task/log/file_driver.go b/core/task/log/file_driver.go index 590a1c9a5..852c0b21c 100644 --- a/core/task/log/file_driver.go +++ b/core/task/log/file_driver.go @@ -6,7 +6,7 @@ import ( "errors" "github.com/apex/log" "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "io" "os" diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go index b7bdbb080..3644aa172 100644 --- a/core/task/scheduler/service.go +++ b/core/task/scheduler/service.go @@ -11,7 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/task" "github.com/crawlab-team/crawlab/db/mongo" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index b5b6cfa4c..fbf8ad406 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -10,7 +10,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" diff --git a/core/task/stats/service.go b/core/task/stats/service.go index ee78dc70b..b6964c297 100644 --- a/core/task/stats/service.go +++ b/core/task/stats/service.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/task" "github.com/crawlab-team/crawlab/core/task/log" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "sync" diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go index b198c95d9..b74b34411 100644 --- a/core/task/stats/service_v2.go +++ b/core/task/stats/service_v2.go @@ -7,7 +7,7 @@ import ( nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/task/log" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "sync" diff --git a/core/user/service.go b/core/user/service.go index db0878906..7294df4f9 100644 --- a/core/user/service.go +++ b/core/user/service.go @@ -10,7 +10,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "github.com/golang-jwt/jwt/v5" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/user/service_v2.go b/core/user/service_v2.go index c5b31c175..7784003c5 100644 --- a/core/user/service_v2.go +++ b/core/user/service_v2.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "github.com/golang-jwt/jwt/v5" "go.mongodb.org/mongo-driver/bson" diff --git a/core/utils/backoff.go b/core/utils/backoff.go index 53658a8d7..6010acfa9 100644 --- a/core/utils/backoff.go +++ b/core/utils/backoff.go @@ -3,7 +3,7 @@ package utils import ( "github.com/apex/log" "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "time" ) diff --git a/core/utils/demo.go b/core/utils/demo.go index 74c8439df..bf6676476 100644 --- a/core/utils/demo.go +++ b/core/utils/demo.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" ) diff --git a/core/utils/di.go b/core/utils/di.go index 8c467228e..8d69a733f 100644 --- a/core/utils/di.go +++ b/core/utils/di.go @@ -1,7 +1,7 @@ package utils import ( - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "go.uber.org/dig" "os" diff --git a/core/utils/es.go b/core/utils/es.go index 5345a093f..6636d0da4 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esapi" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/utils/git.go b/core/utils/git.go index 1065aad59..475d2314c 100644 --- a/core/utils/git.go +++ b/core/utils/git.go @@ -1,10 +1,10 @@ package utils import ( - vcs "github.com/crawlab-team/crawlab-vcs" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" + vcs "github.com/crawlab-team/crawlab/vcs" ) func InitGitClientAuth(g interfaces.Git, gitClient *vcs.GitClient) { diff --git a/core/utils/helpers.go b/core/utils/helpers.go index 338166f80..3f63f12ba 100644 --- a/core/utils/helpers.go +++ b/core/utils/helpers.go @@ -1,7 +1,7 @@ package utils import ( - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "io" "reflect" "unsafe" diff --git a/core/utils/http.go b/core/utils/http.go index 3b2c0f667..b0e48ebff 100644 --- a/core/utils/http.go +++ b/core/utils/http.go @@ -3,7 +3,7 @@ package utils import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "net/http" ) diff --git a/db/go.mod b/db/go.mod index 522e6fa65..861550fc0 100644 --- a/db/go.mod +++ b/db/go.mod @@ -5,43 +5,48 @@ go 1.22 require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/go-trace v0.1.0 + github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 - github.com/spf13/viper v1.7.1 - github.com/stretchr/testify v1.6.1 + github.com/spf13/viper v1.10.0 + github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.15.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.4.7 // indirect - github.com/golang/snappy v0.0.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/lib/pq v1.1.1 // indirect - github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e // indirect - github.com/magiconair/properties v1.8.1 // indirect - github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/lib/pq v1.10.4 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-sqlite3 v1.14.9 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml v1.7.0 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/afero v1.1.2 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/spf13/pflag v1.0.3 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/spf13/afero v1.10.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - github.com/ztrue/tracerr v0.3.0 // indirect + github.com/ztrue/tracerr v0.4.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - gopkg.in/ini.v1 v1.51.0 // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/db/mongo/client.go b/db/mongo/client.go index 5bcd39071..e5c03f90d 100644 --- a/db/mongo/client.go +++ b/db/mongo/client.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/apex/log" "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" diff --git a/db/mongo/col.go b/db/mongo/col.go index fda004fe3..67f3b3479 100644 --- a/db/mongo/col.go +++ b/db/mongo/col.go @@ -3,6 +3,7 @@ package mongo import ( "context" "github.com/crawlab-team/crawlab/db/errors" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" diff --git a/db/mongo/db.go b/db/mongo/db.go index 8ce143cfe..1b29790de 100644 --- a/db/mongo/db.go +++ b/db/mongo/db.go @@ -1,7 +1,7 @@ package mongo import ( - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/mongo" ) diff --git a/db/mongo/transaction.go b/db/mongo/transaction.go index 60ded40f1..fe327b270 100644 --- a/db/mongo/transaction.go +++ b/db/mongo/transaction.go @@ -2,7 +2,7 @@ package mongo import ( "context" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/mongo" ) diff --git a/db/redis/client.go b/db/redis/client.go index 21bed3ff7..97dcbef39 100644 --- a/db/redis/client.go +++ b/db/redis/client.go @@ -5,6 +5,7 @@ import ( "github.com/crawlab-team/crawlab/db" "github.com/crawlab-team/crawlab/db/errors" "github.com/crawlab-team/crawlab/db/utils" + "github.com/crawlab-team/crawlab/trace" "github.com/gomodule/redigo/redis" "reflect" "strings" diff --git a/db/redis/pool.go b/db/redis/pool.go index 05ca20414..cc08388a0 100644 --- a/db/redis/pool.go +++ b/db/redis/pool.go @@ -1,7 +1,7 @@ package redis import ( - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/gomodule/redigo/redis" "github.com/spf13/viper" "time" diff --git a/db/sql/sql.go b/db/sql/sql.go index 1ae076473..0a35ffc62 100644 --- a/db/sql/sql.go +++ b/db/sql/sql.go @@ -3,7 +3,7 @@ package sql import ( "errors" "fmt" - "github.com/crawlab-team/go-trace" + "github.com/crawlab-team/crawlab/trace" "github.com/jmoiron/sqlx" ) diff --git a/fs/.editorconfig b/fs/.editorconfig new file mode 100644 index 000000000..aeab87c1f --- /dev/null +++ b/fs/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*.go] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/fs/.github/workflows/test.yml b/fs/.github/workflows/test.yml new file mode 100644 index 000000000..35f99598d --- /dev/null +++ b/fs/.github/workflows/test.yml @@ -0,0 +1,54 @@ +name: Test and coverage + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup Go environment + uses: actions/setup-go@v2.1.3 + with: + # The Go version to download (if necessary) and use. Supports semver spec and ranges. + go-version: 1.15 + +# - name: Download Binary Files +# run: | +# mkdir -p $GITHUB_WORKSPACE/seaweedfs +# curl https://github.com/chrislusf/seaweedfs/releases/download/2.48/linux_amd64.tar.gz -o $GITHUB_WORKSPACE/seaweedfs/linux_amd64.tar.gz +# cd $GITHUB_WORKSPACE/seaweedfs +# ls -l +# tar -zxf linux_amd64.tar.gz + - name: Download Binary Files + uses: fabriciobastian/download-release-asset-action@v1.0.6 + with: + # A specific release version. Defaults to latest + version: 2.48 # default is latest + # Relative path to the repository in the format user/repo e.g.: myuser/my-repository + repository: chrislusf/seaweedfs # default is + # The name of the asset to download from the release + file: linux_amd64.tar.gz + # Path to the directory where to download the asset + out: seaweedfs # optional, default is . + + - name: Extract Binary Files + run: | + cd $GITHUB_WORKSPACE/seaweedfs + tar -zxf linux_amd64.tar.gz + + - name: Validate Binary Files + run: | + cd $GITHUB_WORKSPACE/seaweedfs + ls -l weed + + - name: Run Tests + run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic -coverpkg github.com/crawlab-team/crawlab/fs + + - name: Codecov + uses: codecov/codecov-action@v1.5.0 + with: + # Repository upload token - get it from codecov.io. Required only for private repositories + token: ${{ secrets.CODECOV_TOKEN }} + # Comma-separated list of files to upload diff --git a/fs/.gitignore b/fs/.gitignore new file mode 100644 index 000000000..76d7ddabb --- /dev/null +++ b/fs/.gitignore @@ -0,0 +1,23 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +vendor/ +tmp/ +.idea/ +.DS_Store +coverage.txt + +filerldb2 +seaweedfs +*.txt \ No newline at end of file diff --git a/fs/Dockerfile b/fs/Dockerfile new file mode 100644 index 000000000..48bbe6df0 --- /dev/null +++ b/fs/Dockerfile @@ -0,0 +1,8 @@ +FROM golang:1.15 + +WORKDIR /app +ADD ./go.mod /app +ADD ./go.sum /app +RUN go mod download + +CMD ["sh", "./bin/test.sh"] diff --git a/fs/LICENSE b/fs/LICENSE new file mode 100644 index 000000000..e41311d27 --- /dev/null +++ b/fs/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Crawlab Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/fs/README.md b/fs/README.md new file mode 100644 index 000000000..a12f48c13 --- /dev/null +++ b/fs/README.md @@ -0,0 +1,5 @@ +# crawlab-fs + +[![codecov](https://codecov.io/gh/crawlab-team/crawlab-fs/branch/main/graph/badge.svg?token=KOXJPXWAKI)](https://codecov.io/gh/crawlab-team/crawlab-fs) + +Backend file system module for Crawlab diff --git a/fs/bin/start.sh b/fs/bin/start.sh new file mode 100755 index 000000000..822ddf1c1 --- /dev/null +++ b/fs/bin/start.sh @@ -0,0 +1,24 @@ +#!/bin/sh +if [ -e ./tmp ]; then + : +else + mkdir ./tmp +fi + +if [ -x /usr/local/bin/weed ]; then + weed server \ + -dir ./tmp \ + -master.dir ./tmp \ + -volume.dir.idx ./tmp \ + -ip localhost \ + -ip.bind 0.0.0.0 \ + -filer +else + ./seaweedfs/weed server \ + -dir ./tmp \ + -master.dir ./tmp \ + -volume.dir.idx ./tmp \ + -ip localhost \ + -ip.bind 0.0.0.0 \ + -filer +fi diff --git a/fs/bin/stop.sh b/fs/bin/stop.sh new file mode 100755 index 000000000..2c93c25cf --- /dev/null +++ b/fs/bin/stop.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +kill -9 `ps axu|grep weed|grep -v grep|awk '{print \$2}'|xargs` \ No newline at end of file diff --git a/fs/constants.go b/fs/constants.go new file mode 100644 index 000000000..5fbfb17a7 --- /dev/null +++ b/fs/constants.go @@ -0,0 +1,19 @@ +package fs + +import "os" + +const ( + FilerResponseNotFoundErrorMessage = "response status code: 404" + FilerStatusNotFoundErrorMessage = "Status:404 Not Found" +) + +const ( + DefaultDirMode = os.FileMode(0766) + DefaultFileMode = os.FileMode(0666) +) + +const ( + MethodUpdateFile = "update-file" + MethodUploadFile = "upload-file" + MethodUploadDir = "upload-dir" +) diff --git a/fs/docker-compose.yml b/fs/docker-compose.yml new file mode 100644 index 000000000..804b0b370 --- /dev/null +++ b/fs/docker-compose.yml @@ -0,0 +1,13 @@ +version: '2' + +services: + server: + image: chrislusf/seaweedfs # use a remote image + container_name: seaweedfs-server + restart: always +# command: "server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer -encryptVolumeData" + command: "server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer" + ports: + - 8888:8888 + #volumes: + # - /data/seaweedfs:/data diff --git a/fs/errors.go b/fs/errors.go new file mode 100644 index 000000000..182dfd921 --- /dev/null +++ b/fs/errors.go @@ -0,0 +1,5 @@ +package fs + +import "errors" + +var ErrorFsNotExists = errors.New("not exists") diff --git a/fs/go.mod b/fs/go.mod new file mode 100644 index 000000000..bc989c3de --- /dev/null +++ b/fs/go.mod @@ -0,0 +1,17 @@ +module github.com/crawlab-team/crawlab/fs + +go 1.22 + +replace ( + github.com/crawlab-team/crawlab/trace => ../trace +) + +require ( + github.com/apex/log v1.9.0 + github.com/cenkalti/backoff/v4 v4.1.0 + github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 + github.com/emirpasic/gods v1.18.1 // indirect + github.com/google/uuid v1.1.1 + github.com/pkg/errors v0.9.1 // indirect + github.com/stretchr/testify v1.6.1 +) diff --git a/fs/go.sum b/fs/go.sum new file mode 100644 index 000000000..dff61a29c --- /dev/null +++ b/fs/go.sum @@ -0,0 +1,102 @@ +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= +github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= +github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936 h1:c4SgTj2baDqD2UYa1eCpj3ukOF3mXOjvOCP4cWwgfyw= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= +github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y= +github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/fs/interface.go b/fs/interface.go new file mode 100644 index 000000000..0bbaca2bb --- /dev/null +++ b/fs/interface.go @@ -0,0 +1,31 @@ +package fs + +import ( + "github.com/crawlab-team/goseaweedfs" + "time" +) + +type Manager interface { + Init() (err error) + Close() (err error) + ListDir(remotePath string, isRecursive bool) (files []goseaweedfs.FilerFileInfo, err error) + UploadFile(localPath, remotePath string, args ...interface{}) (err error) + UploadDir(localPath, remotePath string, args ...interface{}) (err error) + DownloadFile(remotePath, localPath string, args ...interface{}) (err error) + DownloadDir(remotePath, localPath string, args ...interface{}) (err error) + DeleteFile(remotePath string) (err error) + DeleteDir(remotePath string) (err error) + SyncLocalToRemote(localPath, remotePath string, args ...interface{}) (err error) + SyncRemoteToLocal(remotePath, localPath string, args ...interface{}) (err error) + GetFile(remotePath string, args ...interface{}) (data []byte, err error) + GetFileInfo(remotePath string) (file *goseaweedfs.FilerFileInfo, err error) + UpdateFile(remotePath string, data []byte, args ...interface{}) (err error) + Exists(remotePath string, args ...interface{}) (ok bool, err error) + SetFilerUrl(url string) + SetFilerAuthKey(authKey string) + SetTimeout(timeout time.Duration) + SetWorkerNum(num int) + SetRetryInterval(interval time.Duration) + SetRetryNum(num int) + SetMaxQps(qps int) +} diff --git a/fs/lib/copy/copy.go b/fs/lib/copy/copy.go new file mode 100644 index 000000000..1a89e5971 --- /dev/null +++ b/fs/lib/copy/copy.go @@ -0,0 +1,111 @@ +package copy + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "syscall" +) + +func CopyDirectory(scrDir, dest string) error { + entries, err := ioutil.ReadDir(scrDir) + if err != nil { + return err + } + for _, entry := range entries { + sourcePath := filepath.Join(scrDir, entry.Name()) + destPath := filepath.Join(dest, entry.Name()) + + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath) + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := CreateIfNotExists(destPath, 0755); err != nil { + return err + } + if err := CopyDirectory(sourcePath, destPath); err != nil { + return err + } + case os.ModeSymlink: + if err := CopySymLink(sourcePath, destPath); err != nil { + return err + } + default: + if err := Copy(sourcePath, destPath); err != nil { + return err + } + } + + if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + isSymlink := entry.Mode()&os.ModeSymlink != 0 + if !isSymlink { + if err := os.Chmod(destPath, entry.Mode()); err != nil { + return err + } + } + } + return nil +} + +func Copy(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + defer in.Close() + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func Exists(filePath string) bool { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return false + } + + return true +} + +func CreateIfNotExists(dir string, perm os.FileMode) error { + if Exists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func CopySymLink(source, dest string) error { + link, err := os.Readlink(source) + if err != nil { + return err + } + return os.Symlink(link, dest) +} diff --git a/fs/options.go b/fs/options.go new file mode 100644 index 000000000..ce07b2456 --- /dev/null +++ b/fs/options.go @@ -0,0 +1,47 @@ +package fs + +import "time" + +type Option func(m Manager) + +func WithFilerUrl(url string) Option { + return func(m Manager) { + m.SetFilerUrl(url) + } +} + +func WithFilerAuthKey(authKey string) Option { + return func(m Manager) { + m.SetFilerAuthKey(authKey) + } +} + +func WithTimeout(timeout time.Duration) Option { + return func(m Manager) { + m.SetTimeout(timeout) + } +} + +func WithWorkerNum(num int) Option { + return func(m Manager) { + m.SetWorkerNum(num) + } +} + +func WithRetryInterval(interval time.Duration) Option { + return func(m Manager) { + m.SetRetryInterval(interval) + } +} + +func WithRetryNum(num int) Option { + return func(m Manager) { + m.SetRetryNum(num) + } +} + +func WithMaxQps(qps int) Option { + return func(m Manager) { + m.SetMaxQps(qps) + } +} diff --git a/fs/seaweedfs_manager.go b/fs/seaweedfs_manager.go new file mode 100644 index 000000000..bd3034a29 --- /dev/null +++ b/fs/seaweedfs_manager.go @@ -0,0 +1,722 @@ +package fs + +import ( + "bytes" + "errors" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab/trace" + "github.com/crawlab-team/goseaweedfs" + "github.com/emirpasic/gods/queues/linkedlistqueue" + "github.com/google/uuid" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +type seaweedFsManagerFn func(params seaweedFsManagerParams) (res seaweedFsManagerResults) + +type seaweedFsManagerHandle struct { + params seaweedFsManagerParams + fn seaweedFsManagerFn + resChan chan seaweedFsManagerResults +} + +type seaweedFsManagerParams struct { + localPath string + remotePath string + isRecursive bool + collection string + ttl string + urlValues url.Values + data []byte +} + +type seaweedFsManagerResults struct { + files []goseaweedfs.FilerFileInfo + file *goseaweedfs.FilerFileInfo + data []byte + ok bool + err error +} + +type SeaweedFsManager struct { + // settings variables + filerUrl string + timeout time.Duration + authKey string + workerNum int + retryNum uint64 + retryInterval time.Duration + maxQps int + + // internals + f *goseaweedfs.Filer + q *linkedlistqueue.Queue + cr int + ch chan seaweedFsManagerHandle + closed bool +} + +func (m *SeaweedFsManager) Init() (err error) { + // filer options + var filerOpts []goseaweedfs.FilerOption + + // auth key + if m.authKey != "" { + filerOpts = append(filerOpts, goseaweedfs.WithFilerAuthKey(m.authKey)) + } + + // handle channel + m.ch = make(chan seaweedFsManagerHandle, m.workerNum) + + // filer instance + m.f, err = goseaweedfs.NewFiler(m.filerUrl, &http.Client{Timeout: m.timeout}, filerOpts...) + if err != nil { + return trace.TraceError(err) + } + + // start async + go m.start() + + return nil +} + +func (m *SeaweedFsManager) Close() (err error) { + m.closed = true + if err := m.f.Close(); err != nil { + return trace.TraceError(err) + } + return nil +} + +func (m *SeaweedFsManager) ListDir(remotePath string, isRecursive bool) (files []goseaweedfs.FilerFileInfo, err error) { + params := seaweedFsManagerParams{ + remotePath: remotePath, + isRecursive: isRecursive, + } + res := m.process(params, m.listDir) + return res.files, res.err +} + +func (m *SeaweedFsManager) ListDirRecursive(remotePath string) (files []goseaweedfs.FilerFileInfo, err error) { + params := seaweedFsManagerParams{ + remotePath: remotePath, + } + res := m.process(params, m.listDirRecursive) + return res.files, res.err +} + +func (m *SeaweedFsManager) UploadFile(localPath, remotePath string, args ...interface{}) (err error) { + localPath, err = filepath.Abs(localPath) + if err != nil { + return trace.TraceError(err) + } + collection, ttl := getCollectionAndTtlFromArgs(args...) + params := seaweedFsManagerParams{ + localPath: localPath, + remotePath: remotePath, + collection: collection, + ttl: ttl, + } + res := m.process(params, m.uploadFile) + return res.err +} + +func (m *SeaweedFsManager) UploadDir(localPath, remotePath string, args ...interface{}) (err error) { + localPath, err = filepath.Abs(localPath) + if err != nil { + return trace.TraceError(err) + } + collection, ttl := getCollectionAndTtlFromArgs(args...) + params := seaweedFsManagerParams{ + localPath: localPath, + remotePath: remotePath, + collection: collection, + ttl: ttl, + } + res := m.process(params, m.uploadDir) + return res.err +} + +func (m *SeaweedFsManager) DownloadFile(remotePath, localPath string, args ...interface{}) (err error) { + localPath, err = filepath.Abs(localPath) + if err != nil { + return trace.TraceError(err) + } + collection, ttl := getCollectionAndTtlFromArgs(args...) + urlValues := getUrlValuesFromArgs(args...) + params := seaweedFsManagerParams{ + localPath: localPath, + remotePath: remotePath, + collection: collection, + ttl: ttl, + urlValues: urlValues, + } + res := m.process(params, m.downloadFile) + return res.err +} + +func (m *SeaweedFsManager) DownloadDir(remotePath, localPath string, args ...interface{}) (err error) { + localPath, err = filepath.Abs(localPath) + if err != nil { + return trace.TraceError(err) + } + params := seaweedFsManagerParams{ + remotePath: remotePath, + } + res := m.process(params, m.downloadDir) + return res.err +} + +func (m *SeaweedFsManager) DeleteFile(remotePath string) (err error) { + params := seaweedFsManagerParams{ + remotePath: remotePath, + } + res := m.process(params, m.deleteFile) + return res.err +} + +func (m *SeaweedFsManager) DeleteDir(remotePath string) (err error) { + params := seaweedFsManagerParams{ + remotePath: remotePath, + } + res := m.process(params, m.deleteDir) + return res.err +} + +func (m *SeaweedFsManager) SyncLocalToRemote(localPath, remotePath string, args ...interface{}) (err error) { + localPath, err = filepath.Abs(localPath) + if err != nil { + return trace.TraceError(err) + } + collection, ttl := getCollectionAndTtlFromArgs(args...) + params := seaweedFsManagerParams{ + localPath: localPath, + remotePath: remotePath, + collection: collection, + ttl: ttl, + } + res := m.process(params, m.syncLocalToRemote) + return res.err +} + +func (m *SeaweedFsManager) SyncRemoteToLocal(remotePath, localPath string, args ...interface{}) (err error) { + collection, ttl := getCollectionAndTtlFromArgs(args...) + params := seaweedFsManagerParams{ + localPath: localPath, + remotePath: remotePath, + collection: collection, + ttl: ttl, + } + res := m.process(params, m.syncRemoteToLocal) + return res.err +} + +func (m *SeaweedFsManager) GetFile(remotePath string, args ...interface{}) (data []byte, err error) { + urlValues := getUrlValuesFromArgs(args...) + params := seaweedFsManagerParams{ + remotePath: remotePath, + urlValues: urlValues, + } + res := m.process(params, m.getFile) + return res.data, res.err +} + +func (m *SeaweedFsManager) GetFileInfo(remotePath string) (file *goseaweedfs.FilerFileInfo, err error) { + params := seaweedFsManagerParams{ + remotePath: remotePath, + } + res := m.process(params, m.getFileInfo) + return res.file, res.err +} + +func (m *SeaweedFsManager) UpdateFile(remotePath string, data []byte, args ...interface{}) (err error) { + collection, ttl := getCollectionAndTtlFromArgs(args...) + params := seaweedFsManagerParams{ + remotePath: remotePath, + collection: collection, + ttl: ttl, + data: data, + } + res := m.process(params, m.updateFile) + return res.err +} + +func (m *SeaweedFsManager) Exists(remotePath string, args ...interface{}) (ok bool, err error) { + _, err = m.GetFile(remotePath, args...) + if err == nil { + // exists + return true, nil + } + if strings.Contains(err.Error(), FilerStatusNotFoundErrorMessage) { + // not exists + return false, nil + } + return ok, trace.TraceError(err) +} + +func (m *SeaweedFsManager) SetFilerUrl(url string) { + m.filerUrl = url +} + +func (m *SeaweedFsManager) SetFilerAuthKey(authKey string) { + m.authKey = authKey +} + +func (m *SeaweedFsManager) SetTimeout(timeout time.Duration) { + m.timeout = timeout +} + +func (m *SeaweedFsManager) SetWorkerNum(num int) { + m.workerNum = num +} + +func (m *SeaweedFsManager) SetRetryInterval(interval time.Duration) { + m.retryInterval = interval +} + +func (m *SeaweedFsManager) SetRetryNum(num int) { + m.retryNum = uint64(num) +} + +func (m *SeaweedFsManager) SetMaxQps(qps int) { + m.maxQps = qps +} + +func (m *SeaweedFsManager) newHandle(params seaweedFsManagerParams, fn seaweedFsManagerFn) (handle seaweedFsManagerHandle) { + return seaweedFsManagerHandle{ + params: params, + fn: fn, + resChan: make(chan seaweedFsManagerResults), + } +} + +func (m *SeaweedFsManager) start() { + for { + if m.closed { + return + } + handle := <-m.ch + go func() { + if err := backoff.Retry(func() error { + res := handle.fn(handle.params) + if res.err != nil { + return res.err + } + handle.resChan <- res + return nil + }, backoff.WithMaxRetries( + backoff.NewConstantBackOff(m.retryInterval), m.retryNum), + ); err != nil { + handle.resChan <- m.error(err) + } + m.wait() + }() + } +} + +func (m *SeaweedFsManager) process(params seaweedFsManagerParams, fn seaweedFsManagerFn) (res seaweedFsManagerResults) { + handle := m.newHandle(params, fn) + //log.Infof("handle: %v", handle) + m.ch <- handle + res = <-handle.resChan + return +} + +func (m *SeaweedFsManager) error(err error) (res seaweedFsManagerResults) { + if err != nil { + trace.PrintError(err) + } + return seaweedFsManagerResults{err: err} +} + +func (m *SeaweedFsManager) wait() { + ms := float32(1) / float32(m.maxQps) * 1e3 + d := time.Duration(ms) * time.Millisecond + time.Sleep(d) +} + +func (m *SeaweedFsManager) getCollectionAndTtlArgsFromParams(params seaweedFsManagerParams) (args []interface{}) { + if params.collection != "" { + args = append(args, params.collection) + } + if params.ttl != "" { + args = append(args, params.ttl) + } + return args +} + +func (m *SeaweedFsManager) listDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + var err error + if params.isRecursive { + res.files, err = m.ListDirRecursive(params.remotePath) + } else { + res.files, err = m.f.ListDir(params.remotePath) + } + if err != nil { + return m.error(err) + } + return +} + +func (m *SeaweedFsManager) listDirRecursive(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + entries, err := m.f.ListDir(params.remotePath) + if err != nil { + return m.error(err) + } + for _, file := range entries { + file = goseaweedfs.GetFileWithExtendedFields(file) + if file.IsDir { + file.Children, err = m.ListDirRecursive(file.FullPath) + if err != nil { + return m.error(err) + } + } + res.files = append(res.files, file) + } + return +} + +func (m *SeaweedFsManager) uploadFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + r, err := m.f.UploadFile(params.localPath, params.remotePath, params.collection, params.ttl) + if err != nil { + return m.error(err) + } + if r.Error != "" { + err = errors.New(r.Error) + return m.error(err) + } + return +} + +func (m *SeaweedFsManager) uploadDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + args := m.getCollectionAndTtlArgsFromParams(params) + if strings.HasSuffix(params.localPath, "/") { + params.localPath = params.localPath[:(len(params.localPath) - 1)] + } + if !strings.HasPrefix(params.remotePath, "/") { + params.remotePath = "/" + params.remotePath + } + files, err := goseaweedfs.ListFilesRecursive(params.localPath) + if err != nil { + return m.error(err) + } + for _, info := range files { + newFilePath := params.remotePath + strings.Replace(info.Path, params.localPath, "", -1) + if err := m.UploadFile(info.Path, newFilePath, args...); err != nil { + return m.error(err) + } + } + return +} + +func (m *SeaweedFsManager) downloadFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + err := m.f.Download(params.remotePath, params.urlValues, func(reader io.Reader) error { + data, err := ioutil.ReadAll(reader) + if err != nil { + return trace.TraceError(err) + } + dirPath := filepath.Dir(params.localPath) + _, err = os.Stat(dirPath) + if err != nil { + // if not exists, create a new directory + if err := os.MkdirAll(dirPath, DefaultDirMode); err != nil { + return trace.TraceError(err) + } + } + fileMode := DefaultFileMode + fileInfo, err := os.Stat(params.localPath) + if err == nil { + // if file already exists, save file mode and remove it + fileMode = fileInfo.Mode() + if err := os.Remove(params.localPath); err != nil { + return trace.TraceError(err) + } + } + if err := ioutil.WriteFile(params.localPath, data, fileMode); err != nil { + return trace.TraceError(err) + } + return nil + }) + if err != nil { + return m.error(err) + } + return +} + +func (m *SeaweedFsManager) downloadDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + args := m.getCollectionAndTtlArgsFromParams(params) + var files []goseaweedfs.FilerFileInfo + files, res.err = m.ListDir(params.remotePath, true) + for _, file := range files { + if file.IsDir { + if err := m.DownloadDir(file.FullPath, path.Join(params.localPath, file.Name), args...); err != nil { + return m.error(err) + } + } else { + if err := m.DownloadFile(file.FullPath, path.Join(params.localPath, file.Name), args...); err != nil { + return m.error(err) + } + } + } + return +} + +func (m *SeaweedFsManager) deleteFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + if err := m.f.DeleteFile(params.remotePath); err != nil { + return m.error(err) + } + return +} + +func (m *SeaweedFsManager) deleteDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + if err := m.f.DeleteDir(params.remotePath); err != nil { + return m.error(err) + } + return +} + +func (m *SeaweedFsManager) syncLocalToRemote(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + // args + args := m.getCollectionAndTtlArgsFromParams(params) + + // raise error if local path does not exist + if _, err := os.Stat(params.localPath); err != nil { + return m.error(err) + } + + // get files and maps + localFiles, remoteFiles, localFilesMap, remoteFilesMap, err := getFilesAndFilesMaps(m.f, params.localPath, params.remotePath) + if err != nil { + return m.error(err) + } + + // compare remote files with local files and delete files absent in local files + for _, remoteFile := range remoteFiles { + // attempt to get corresponding local file + _, ok := localFilesMap[remoteFile.FullPath] + + if !ok { + // file does not exist on local, delete + if remoteFile.IsDir { + if err := m.DeleteDir(remoteFile.FullPath); err != nil { + return m.error(err) + } + } else { + if err := m.DeleteFile(remoteFile.FullPath); err != nil { + return m.error(err) + } + } + } + } + + // compare local files with remote files and upload files with difference + for _, localFile := range localFiles { + // skip .git + if IsGitFile(localFile) { + continue + } + + // corresponding remote file path + fileRemotePath := fmt.Sprintf("%s%s", params.remotePath, strings.Replace(localFile.Path, params.localPath, "", -1)) + + // attempt to get corresponding remote file + remoteFile, ok := remoteFilesMap[fileRemotePath] + + if !ok { + // file does not exist on remote, upload + if err := m.UploadFile(localFile.Path, fileRemotePath, args...); err != nil { + return m.error(err) + } + } else { + // file exists on remote, upload if md5sum values are different + if remoteFile.Md5 != localFile.Md5 { + if err := m.UploadFile(localFile.Path, fileRemotePath, args...); err != nil { + return m.error(err) + } + } + } + } + + return +} + +func (m *SeaweedFsManager) syncRemoteToLocal(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + // args + args := m.getCollectionAndTtlArgsFromParams(params) + + // create directory if local path does not exist + if _, err := os.Stat(params.localPath); err != nil { + if err := os.MkdirAll(params.localPath, os.ModePerm); err != nil { + return m.error(err) + } + } + + // get files and maps + localFiles, remoteFiles, localFilesMap, remoteFilesMap, err := getFilesAndFilesMaps(m.f, params.localPath, params.remotePath) + if err != nil { + return m.error(err) + } + + // compare local files with remote files and delete files absent on remote + for _, localFile := range localFiles { + // skip .git + if IsGitFile(localFile) { + continue + } + + // corresponding remote file path + fileRemotePath := fmt.Sprintf("%s%s", params.remotePath, strings.Replace(localFile.Path, params.localPath, "", -1)) + + // attempt to get corresponding remote file + _, ok := remoteFilesMap[fileRemotePath] + + if !ok { + // file does not exist on remote, upload + if err := os.Remove(localFile.Path); err != nil { + return m.error(err) + } + } + } + + // compare remote files with local files and download if files with difference + for _, remoteFile := range remoteFiles { + // directory + if remoteFile.IsDir { + localDirRelativePath := strings.Replace(remoteFile.FullPath, params.remotePath, "", 1) + localDirPath := fmt.Sprintf("%s%s", params.localPath, localDirRelativePath) + if err := m.SyncRemoteToLocal(remoteFile.FullPath, localDirPath); err != nil { + return m.error(err) + } + continue + } + + // local file path + localFileRelativePath := strings.Replace(remoteFile.FullPath, params.remotePath, "", 1) + localFilePath := fmt.Sprintf("%s%s", params.localPath, localFileRelativePath) + + // attempt to get corresponding local file + localFile, ok := localFilesMap[remoteFile.FullPath] + + if !ok { + // file does not exist on local, download + if err := m.DownloadFile(remoteFile.FullPath, localFilePath); err != nil { + return m.error(err) + } + } else { + // file exists on remote, download if md5sum values are different + if remoteFile.Md5 != localFile.Md5 { + if err := m.DownloadFile(remoteFile.FullPath, localFilePath, args...); err != nil { + return m.error(err) + } + } + } + } + + return +} + +func (m *SeaweedFsManager) getFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + var buf bytes.Buffer + res.err = m.f.Download(params.remotePath, params.urlValues, func(reader io.Reader) error { + _, err := io.Copy(&buf, reader) + if err != nil { + return trace.TraceError(err) + } + return nil + }) + res.data = buf.Bytes() + return +} + +func (m *SeaweedFsManager) getFileInfo(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + arr := strings.Split(params.remotePath, "/") + dirName := strings.Join(arr[:(len(arr)-1)], "/") + files, err := m.f.ListDir(dirName) + if err != nil { + return m.error(err) + } + for _, f := range files { + if f.FullPath == params.remotePath { + res.file = &f + return + } + } + return m.error(ErrorFsNotExists) +} + +func (m *SeaweedFsManager) updateFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) { + tmpRootDir := os.TempDir() + tmpDirPath := path.Join(tmpRootDir, ".seaweedfs") + if _, err := os.Stat(tmpDirPath); err != nil { + if err := os.MkdirAll(tmpDirPath, os.ModePerm); err != nil { + return m.error(err) + } + } + tmpFilePath := path.Join(tmpDirPath, fmt.Sprintf(".%s", uuid.New().String())) + if _, err := os.Stat(tmpFilePath); err == nil { + if err := os.Remove(tmpFilePath); err != nil { + return m.error(err) + } + } + if err := ioutil.WriteFile(tmpFilePath, params.data, os.ModePerm); err != nil { + return m.error(err) + } + params2 := seaweedFsManagerParams{ + localPath: tmpFilePath, + remotePath: params.remotePath, + collection: params.collection, + ttl: params.ttl, + } + if res := m.uploadFile(params2); res.err != nil { + return m.error(res.err) + } + if err := os.Remove(tmpFilePath); err != nil { + return m.error(err) + } + return +} + +func NewSeaweedFsManager(opts ...Option) (m2 Manager, err error) { + // manager + m := &SeaweedFsManager{ + filerUrl: "http://localhost:8888", + timeout: 5 * time.Minute, + workerNum: 1, + retryInterval: 500 * time.Millisecond, + retryNum: 3, + maxQps: 5, + q: linkedlistqueue.New(), + } + + // apply options + for _, opt := range opts { + opt(m) + } + + // initialize + if err := m.Init(); err != nil { + return nil, err + } + + return m, nil +} + +var _seaweedFsManager Manager + +func GetSeaweedFsManager(opts ...Option) (m2 Manager, err error) { + if _seaweedFsManager == nil { + _seaweedFsManager, err = NewSeaweedFsManager(opts...) + if err != nil { + return nil, err + } + } + return _seaweedFsManager, nil +} diff --git a/fs/test/base.go b/fs/test/base.go new file mode 100644 index 000000000..214f8b1b0 --- /dev/null +++ b/fs/test/base.go @@ -0,0 +1,56 @@ +package test + +import ( + fs "github.com/crawlab-team/crawlab/fs" + "os" + "testing" + "time" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + m fs.Manager +} + +func (t *Test) Setup(t2 *testing.T) { + t.Cleanup() + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = T.m.DeleteDir("/test") + + // wait to avoid caching + time.Sleep(200 * time.Millisecond) +} + +func NewTest() (res *Test, err error) { + // test + t := &Test{} + + // filer url + filerUrl := os.Getenv("CRAWLAB_FILER_URL") + if filerUrl == "" { + filerUrl = "http://localhost:8888" + } + + // manager + t.m, err = fs.NewSeaweedFsManager( + fs.WithFilerUrl(filerUrl), + fs.WithTimeout(10*time.Second), + ) + if err != nil { + return nil, err + } + + return t, nil +} diff --git a/fs/test/bindata.go b/fs/test/bindata.go new file mode 100644 index 000000000..f90c2792f --- /dev/null +++ b/fs/test/bindata.go @@ -0,0 +1,269 @@ +// Code generated for package test by go-bindata DO NOT EDIT. (@generated) +// sources: +// bin/start.sh +// bin/stop.sh +package test + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +// Name return file name +func (fi bindataFileInfo) Name() string { + return fi.name +} + +// Size return file size +func (fi bindataFileInfo) Size() int64 { + return fi.size +} + +// Mode return file mode +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} + +// Mode return file modify time +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} + +// IsDir return file whether a directory +func (fi bindataFileInfo) IsDir() bool { + return fi.mode&os.ModeDir != 0 +} + +// Sys return file is sys mode +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _binStartSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x8f\x4b\x0e\xc2\x30\x0c\x44\xf7\x3e\xc5\x20\xd6\x4d\x58\xc3\x51\x80\x45\x4b\x1c\xd5\x22\x69\xab\x38\x2d\x3d\x3e\x6a\xa0\x7c\xc4\x05\x90\x57\x7e\xa3\x19\x8f\xb7\x1b\xdb\x48\x67\xb5\x25\xf1\x38\xa2\x62\x18\x9b\xe3\x80\xf3\x01\xb9\xe5\x8e\x80\x3d\x71\x50\x26\x20\x5e\x9d\xa4\x87\x4c\x5e\xe8\x69\x98\x61\x47\x4d\x36\xf4\x97\x3a\x94\xa8\x1b\xb3\xfb\xb0\x97\x55\x39\x4d\x9c\x70\x22\x00\xa8\x5e\x31\x2b\x88\xb5\x66\x4e\xe6\x87\x4f\x7d\x18\x23\x2f\xdc\x88\x9b\xbf\x35\x19\x50\x6e\xb6\xbd\xe6\x37\x33\x8d\x74\x0e\x3b\x53\x66\xc5\x5e\x02\xa7\xf5\x0b\x63\x95\xeb\xa5\x94\x57\xfb\x37\xdd\xbc\xd0\x3d\x00\x00\xff\xff\x09\xef\x85\x28\x89\x01\x00\x00") + +func binStartShBytes() ([]byte, error) { + return bindataRead( + _binStartSh, + "bin/start.sh", + ) +} + +func binStartSh() (*asset, error) { + bytes, err := binStartShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "bin/start.sh", size: 393, mode: os.FileMode(493), modTime: time.Unix(1621153670, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _binStopSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x52\x56\xd4\x4f\xca\xcc\xd3\x2f\xce\xe0\xe2\xca\xce\xcc\xc9\x51\xd0\xb5\x54\x48\x28\x28\x56\x48\xac\x28\xad\x49\x2f\x4a\x2d\x50\x28\x4f\x4d\x4d\x81\xb0\x74\xcb\x14\x40\x74\x4d\x62\x79\xb6\x82\x7a\x75\x41\x51\x66\x5e\x89\x42\x8c\x8a\x51\xad\x7a\x4d\x45\x62\x51\x7a\x71\x02\x20\x00\x00\xff\xff\x6b\x5f\x02\x48\x4a\x00\x00\x00") + +func binStopShBytes() ([]byte, error) { + return bindataRead( + _binStopSh, + "bin/stop.sh", + ) +} + +func binStopSh() (*asset, error) { + bytes, err := binStopShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "bin/stop.sh", size: 74, mode: os.FileMode(493), modTime: time.Unix(1621154552, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "bin/start.sh": binStartSh, + "bin/stop.sh": binStopSh, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "bin": &bintree{nil, map[string]*bintree{ + "start.sh": &bintree{binStartSh, map[string]*bintree{}}, + "stop.sh": &bintree{binStopSh, map[string]*bintree{}}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/fs/test/main_test.go b/fs/test/main_test.go new file mode 100644 index 000000000..866e8ac85 --- /dev/null +++ b/fs/test/main_test.go @@ -0,0 +1,21 @@ +package test + +import ( + "testing" +) + +func TestMain(m *testing.M) { + // before test + //if err := StartTestSeaweedFs(); err != nil { + // panic(err) + //} + + // test + m.Run() + + // close + _ = T.m.Close() + + // after test + //_ = StopTestSeaweedFs() +} diff --git a/fs/test/seaweedfs_test.go b/fs/test/seaweedfs_test.go new file mode 100644 index 000000000..6881ce3cd --- /dev/null +++ b/fs/test/seaweedfs_test.go @@ -0,0 +1,473 @@ +package test + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/fs" + "github.com/crawlab-team/crawlab/fs/lib/copy" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "strings" + "sync" + "testing" + "time" +) + +func TestNewSeaweedFsManager(t *testing.T) { + _, err := fs.NewSeaweedFsManager() + require.Nil(t, err) +} + +func TestSeaweedFsManager_ListDir(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data/nested", "/test/data/nested") + require.Nil(t, err) + + valid := false + files, err := T.m.ListDir("/test/data", true) + require.Nil(t, err) + for _, f1 := range files { + if f1.Name == "nested" && f1.Children != nil { + for _, f2 := range f1.Children { + if f2.Name == "nested_test_data.txt" { + valid = true + } + } + } + } + require.True(t, valid) +} + +func TestSeaweedFsManager_UploadFile(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt") + require.Nil(t, err) + + files, err := T.m.ListDir("/test/data", true) + require.Nil(t, err) + valid := false + for _, file := range files { + if file.Name == "test_data.txt" { + valid = true + } + } + require.True(t, valid) +} + +func TestSeaweedFsManager_UploadDir(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data/nested", "/test/data/nested") + require.Nil(t, err) + + valid := false + files, err := T.m.ListDir("/test/data", true) + require.Nil(t, err) + for _, f1 := range files { + if f1.Name == "nested" && f1.Children != nil { + for _, f2 := range f1.Children { + if f2.Name == "nested_test_data.txt" { + valid = true + } + } + } + } + require.True(t, valid) +} + +func TestSeaweedFsManager_GetFile(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt") + require.Nil(t, err) + + data, err := T.m.GetFile("/test/data/test_data.txt") + require.Equal(t, "this is a test data", string(data)) +} + +func TestSeaweedFsManager_DownloadFile(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt") + require.Nil(t, err) + + err = T.m.DownloadFile("/test/data/test_data.txt", "./tmp/test_data.txt") + require.Nil(t, err) + + data, err := ioutil.ReadFile("./tmp/test_data.txt") + require.Nil(t, err) + require.NotEmpty(t, data) +} + +func TestSeaweedFsManager_DownloadDir(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data/nested", "/test/data/nested") + require.Nil(t, err) + + err = T.m.DownloadDir("/test/data", "./tmp/data") + require.Nil(t, err) + + data, err := ioutil.ReadFile("./data/nested/nested_test_data.txt") + require.Nil(t, err) + require.NotEmpty(t, data) +} + +func TestSeaweedFsManager_DeleteFile(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt") + require.Nil(t, err) + + err = T.m.DeleteFile("/test/data/test_data.txt") + require.Nil(t, err) + + files, err := T.m.ListDir("/test/data", true) + require.Nil(t, err) + require.Equal(t, 0, len(files)) +} + +func TestSeaweedFsManager_DeleteDir(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data", "/test/data") + require.Nil(t, err) + + err = T.m.DeleteDir("/test/data/nested") + require.Nil(t, err) + + files, err := T.m.ListDir("/test/data", true) + require.Nil(t, err) + valid := true + for _, file := range files { + if file.Name == "nested" && file.IsDir { + valid = false + } + } + require.True(t, valid) +} + +func TestSeaweedFsManager_SyncLocalToRemote(t *testing.T) { + var err error + T.Setup(t) + + err = copy.CopyDirectory("./data", "./tmp/data") + require.Nil(t, err) + + err = T.m.SyncLocalToRemote("./tmp/data", "/test/data") + require.Nil(t, err) + + data, err := T.m.GetFile("/test/data/test_data.txt") + require.Nil(t, err) + require.Equal(t, "this is a test data", string(data)) + + data, err = T.m.GetFile("/test/data/nested/nested_test_data.txt") + require.Nil(t, err) + require.Equal(t, "this is nested test data", string(data)) + + err = ioutil.WriteFile("./tmp/data/test_data.txt", []byte("this is changed data"), os.ModePerm) + require.Nil(t, err) + + err = T.m.SyncLocalToRemote("./tmp/data", "/test/data") + require.Nil(t, err) + + data, err = T.m.GetFile("/test/data/test_data.txt") + require.Equal(t, "this is changed data", string(data)) + + err = os.Remove("./tmp/data/test_data.txt") + require.Nil(t, err) + + err = T.m.SyncLocalToRemote("./tmp/data", "/test/data") + require.Nil(t, err) + + valid := true + files, err := T.m.ListDir("/test/data", true) + for _, file := range files { + if file.Name == "test_data.txt" { + valid = false + } + } + require.True(t, valid) + + // check if directory is deleted after sync + err = ioutil.WriteFile("./tmp/test.txt", []byte("test"), os.ModePerm) + require.Nil(t, err) + err = T.m.UploadFile("./tmp/test.txt", "/test/data/folder1/test.txt") + require.Nil(t, err) + time.Sleep(1 * time.Second) + err = T.m.SyncLocalToRemote("./tmp/data", "/test/data") + require.Nil(t, err) + valid = true + files, err = T.m.ListDir("/test/data", true) + for _, file := range files { + if strings.Contains(file.FullPath, "folder1") { + valid = false + } + } + require.True(t, valid) +} + +func TestSeaweedFsManager_SyncRemoteToLocal(t *testing.T) { + var err error + T.Setup(t) + + if _, err := os.Stat("./tmp/data"); err == nil { + err = os.RemoveAll("./tmp/data") + require.Nil(t, err) + } + + err = T.m.UploadDir("./data", "/test/data") + require.Nil(t, err) + + err = T.m.SyncRemoteToLocal("/test/data", "./tmp/data") + require.Nil(t, err) + + data, err := ioutil.ReadFile("./tmp/data/test_data.txt") + require.Nil(t, err) + require.Equal(t, "this is a test data", string(data)) + + data, err = ioutil.ReadFile("./tmp/data/nested/nested_test_data.txt") + require.Nil(t, err) + require.Equal(t, "this is nested test data", string(data)) + + err = T.m.UpdateFile("/test/data/test_data.txt", []byte("this is changed data")) + require.Nil(t, err) + + err = T.m.SyncRemoteToLocal("/test/data", "./tmp/data") + require.Nil(t, err) + + data, err = ioutil.ReadFile("./tmp/data/test_data.txt") + require.Equal(t, "this is changed data", string(data)) + + err = T.m.DeleteFile("/test/data/test_data.txt") + require.Nil(t, err) + + err = T.m.SyncRemoteToLocal("/test/data", "./tmp/data") + require.Nil(t, err) + + _, err = os.Stat("./tmp/data/test_data.txt") + require.NotNil(t, err) +} + +func TestSeaweedFsManager_UpdateFile(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data", "/test/data") + require.Nil(t, err) + + err = T.m.UpdateFile("/test/data/test_data.txt", []byte("this is changed data")) + require.Nil(t, err) + + data, err := T.m.GetFile("/test/data/test_data.txt") + require.Nil(t, err) + require.Equal(t, "this is changed data", string(data)) +} + +func TestSeaweedFsManager_Exists(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data", "/test/data") + require.Nil(t, err) + + ok, err := T.m.Exists("/test/data/test_data.txt") + require.Nil(t, err) + require.True(t, ok) + + ok, err = T.m.Exists("/test/data/test_data_404.txt") + require.Nil(t, err) + require.False(t, ok) +} + +func TestSeaweedFsManager_ListDirPressure(t *testing.T) { + var err error + T.Setup(t) + + err = T.m.UploadDir("./data/nested", "/test/data/nested") + require.Nil(t, err) + + n := int(1e3) + doneNum := 0 + errNum := 0 + startTs := time.Now() + + wg := sync.WaitGroup{} + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + _, err := T.m.ListDir("test/data", true) + wg.Done() + if err != nil { + errNum++ + } + doneNum++ + log.Infof("list dir: %d/%d", doneNum, n) + require.Nil(t, err) + }(i) + } + wg.Wait() + + endTs := time.Now() + duration := endTs.Sub(startTs).Milliseconds() + + fmt.Println(fmt.Sprintf("total: %d", n)) + fmt.Println(fmt.Sprintf("errors: %d", errNum)) + fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n))) + fmt.Println(fmt.Sprintf("duration: %dms", duration)) +} + +func TestSeaweedFsManager_UploadFilePressure(t *testing.T) { + var err error + T.Setup(t) + + n := int(1e3) + doneNum := 0 + errNum := 0 + startTs := time.Now() + + wg := sync.WaitGroup{} + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + err = T.m.UploadFile("./data/test_data.txt", fmt.Sprintf("/test/data/test_data_%d.txt", i)) + wg.Done() + if err != nil { + errNum++ + } + doneNum++ + log.Infof("upload file: %d/%d", doneNum, n) + require.Nil(t, err) + }(i) + } + wg.Wait() + + endTs := time.Now() + duration := endTs.Sub(startTs).Milliseconds() + + fmt.Println(fmt.Sprintf("total: %d", n)) + fmt.Println(fmt.Sprintf("errors: %d", errNum)) + fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n))) + fmt.Println(fmt.Sprintf("duration: %dms", duration)) +} + +func TestSeaweedFsManager_UploadDirPressure(t *testing.T) { + var err error + T.Setup(t) + + n := int(1e3) + doneNum := 0 + errNum := 0 + startTs := time.Now() + + wg := sync.WaitGroup{} + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + err = T.m.UploadDir("./data/nested", "/test/data") + wg.Done() + if err != nil { + errNum++ + } + doneNum++ + log.Infof("upload dir: %d/%d", doneNum, n) + require.Nil(t, err) + }(i) + } + wg.Wait() + + endTs := time.Now() + duration := endTs.Sub(startTs).Milliseconds() + + fmt.Println(fmt.Sprintf("total: %d", n)) + fmt.Println(fmt.Sprintf("errors: %d", errNum)) + fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n))) + fmt.Println(fmt.Sprintf("duration: %dms", duration)) +} + +func TestSeaweedFsManager_SyncRemoteToLocalPressure(t *testing.T) { + var err error + T.Setup(t) + + if _, err := os.Stat("./tmp/data"); err == nil { + err = os.RemoveAll("./tmp/data") + require.Nil(t, err) + } + + err = T.m.UploadDir("./data", "/test/data") + require.Nil(t, err) + + n := int(1e3) + doneNum := 0 + errNum := 0 + startTs := time.Now() + + wg := sync.WaitGroup{} + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + err = T.m.SyncRemoteToLocal("/test/data", fmt.Sprintf("./tmp/data_%d", i)) + wg.Done() + if err != nil { + errNum++ + } + doneNum++ + log.Infof("updated: %d/%d", doneNum, n) + require.Nil(t, err) + }(i) + } + wg.Wait() + + endTs := time.Now() + duration := endTs.Sub(startTs).Milliseconds() + + fmt.Println(fmt.Sprintf("total: %d", n)) + fmt.Println(fmt.Sprintf("errors: %d", errNum)) + fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n))) + fmt.Println(fmt.Sprintf("duration: %dms", duration)) +} + +func TestSeaweedFsManager_UpdateFilePressure(t *testing.T) { + var err error + T.Setup(t) + + n := int(5e3) + doneNum := 0 + errNum := 0 + startTs := time.Now() + + wg := sync.WaitGroup{} + wg.Add(n) + for i := 0; i < n; i++ { + go func(i int) { + err = T.m.UpdateFile(fmt.Sprintf("/test/data/test_data_%5d.txt", i), []byte(fmt.Sprintf("this is test data: %5d", i))) + wg.Done() + if err != nil { + errNum++ + } + doneNum++ + log.Infof("updated: %d/%d", doneNum, n) + require.Nil(t, err) + }(i) + } + wg.Wait() + + endTs := time.Now() + duration := endTs.Sub(startTs).Milliseconds() + + fmt.Println(fmt.Sprintf("total: %d", n)) + fmt.Println(fmt.Sprintf("errors: %d", errNum)) + fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n))) + fmt.Println(fmt.Sprintf("duration: %dms", duration)) +} diff --git a/fs/test/utils.go b/fs/test/utils.go new file mode 100644 index 000000000..58bd49dfa --- /dev/null +++ b/fs/test/utils.go @@ -0,0 +1,124 @@ +package test + +import ( + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab/trace" + "github.com/google/uuid" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "time" +) + +func init() { + var err error + TmpDir, err = filepath.Abs("tmp") + if err != nil { + panic(err) + } + if _, err := os.Stat(TmpDir); err != nil { + if err := os.MkdirAll(TmpDir, os.ModePerm); err != nil { + panic(err) + } + } + //TmpDir = getTmpDir() +} + +var TmpDir string + +func StartTestSeaweedFs() (err error) { + // skip if CRAWLAB_IGNORE_WEED is set true + if os.Getenv("CRAWLAB_IGNORE_WEED") != "" { + return nil + } + + // write to start.sh and stop.sh + if err := writeShFiles(TmpDir); err != nil { + return trace.TraceError(err) + } + + // run weed + go runCmd(exec.Command("sh", "./start.sh"), TmpDir) + + // wait for containers to be ready + time.Sleep(5 * time.Second) + f := func() error { + _, err := T.m.ListDir("/", true) + if err != nil { + return err + } + return nil + } + b := backoff.WithMaxRetries(backoff.NewConstantBackOff(5*time.Second), 5) + nt := func(err error, duration time.Duration) { + log.Infof("seaweedfs services not ready, re-attempt in %.1f seconds", duration.Seconds()) + } + err = backoff.RetryNotify(f, b, nt) + if err != nil { + return trace.TraceError(err) + } + + return nil +} + +func StopTestSeaweedFs() (err error) { + // skip if CRAWLAB_IGNORE_WEED is set true + if os.Getenv("CRAWLAB_IGNORE_WEED") != "" { + return nil + } + + // stop seaweedfs + if err := runCmd(exec.Command("sh", "./stop.sh"), TmpDir); err != nil { + return trace.TraceError(err) + } + time.Sleep(5 * time.Second) + + // remove tmp folder + if err := os.RemoveAll(TmpDir); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func writeShFiles(dirPath string) (err error) { + fileNames := []string{ + "start.sh", + "stop.sh", + } + + for _, fileName := range fileNames { + data, err := Asset("bin/" + fileName) + if err != nil { + return trace.TraceError(err) + } + filePath := path.Join(dirPath, fileName) + if err := ioutil.WriteFile(filePath, data, os.FileMode(0766)); err != nil { + return trace.TraceError(err) + } + } + + return nil +} + +func runCmd(cmd *exec.Cmd, dirPath string) (err error) { + log.Infof("running cmd: %v", cmd) + cmd.Dir = dirPath + //cmd.Stdout = os.Stdout + //cmd.Stderr = os.Stdout + return cmd.Run() +} + +func getTmpDir() string { + id, _ := uuid.NewUUID() + tmpDir := path.Join(os.TempDir(), id.String()) + if _, err := os.Stat(tmpDir); err != nil { + if err := os.MkdirAll(tmpDir, os.FileMode(0766)); err != nil { + panic(err) + } + } + return tmpDir +} diff --git a/fs/utils.go b/fs/utils.go new file mode 100644 index 000000000..a34382bae --- /dev/null +++ b/fs/utils.go @@ -0,0 +1,89 @@ +package fs + +import ( + "fmt" + "github.com/crawlab-team/goseaweedfs" + "net/url" + "path/filepath" + "regexp" + "strings" +) + +func IsGitFile(file goseaweedfs.FileInfo) (res bool) { + // skip .git + res, err := regexp.MatchString("/?\\.git/", file.Path) + if err != nil { + return false + } + return res +} + +func getCollectionAndTtlFromArgs(args ...interface{}) (collection, ttl string) { + if len(args) > 0 { + collection = args[0].(string) + } + if len(args) > 1 { + ttl = args[1].(string) + } + return +} + +func getUrlValuesFromArgs(args ...interface{}) (values url.Values) { + if len(args) > 0 { + values = args[0].(url.Values) + } + return values +} + +func getFilesAndFilesMaps(f *goseaweedfs.Filer, localPath, remotePath string) (localFiles []goseaweedfs.FileInfo, remoteFiles []goseaweedfs.FilerFileInfo, localFilesMap map[string]goseaweedfs.FileInfo, remoteFilesMap map[string]goseaweedfs.FilerFileInfo, err error) { + // declare maps + localFilesMap = map[string]goseaweedfs.FileInfo{} + remoteFilesMap = map[string]goseaweedfs.FilerFileInfo{} + + // cache local files info + localFiles, err = goseaweedfs.ListLocalFilesRecursive(localPath) + if err != nil { + return localFiles, remoteFiles, localFilesMap, remoteFilesMap, err + } + for _, file := range localFiles { + fileRemotePath := fmt.Sprintf("%s%s", remotePath, strings.Replace(file.Path, localPath, "", -1)) + localFilesMap[fileRemotePath] = file + + // directory + dirRemotePath := filepath.Dir(fileRemotePath) + _, ok := localFilesMap[dirRemotePath] + if !ok { + localFilesMap[dirRemotePath] = goseaweedfs.FileInfo{ + Name: filepath.Base(dirRemotePath), + Path: dirRemotePath, + } + } + } + + // cache remote files info + remoteFiles, err = f.ListDirRecursive(remotePath) + if err != nil { + if err.Error() != FilerResponseNotFoundErrorMessage { + return localFiles, remoteFiles, localFilesMap, remoteFilesMap, err + } + err = nil + } + remoteFiles = getFlattenRemoteFiles(remoteFiles) + for _, file := range remoteFiles { + remoteFilesMap[file.FullPath] = file + } + + return +} + +func getFlattenRemoteFiles(files []goseaweedfs.FilerFileInfo) (flattenFiles []goseaweedfs.FilerFileInfo) { + flattenFiles = []goseaweedfs.FilerFileInfo{} + for _, file := range files { + flattenFiles = append(flattenFiles, file) + + if file.IsDir { + flattenFiles = append(flattenFiles, getFlattenRemoteFiles(file.Children)...) + } + } + return +} diff --git a/go.work b/go.work index af710c989..24bc5dbc4 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,10 @@ go 1.22 use ( core - grpc db + grpc + template-parser + trace + vcs + fs ) diff --git a/go.work.sum b/go.work.sum index 7052a5823..7e136fd5a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,5 +1,3 @@ -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= @@ -58,10 +56,6 @@ cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= -cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.13.0 h1:6Vs/YnDG5STGjlWMEjN/xtmft7MrOTOnOZYUZtGTx0w= cloud.google.com/go/contactcenterinsights v1.13.0/go.mod h1:ieq5d5EtHsu8vhe2y3amtZ+BE+AQwX5qAy7cpo0POsI= cloud.google.com/go/container v1.31.0 h1:MAaNH7VRNPWEhvqOypq2j+7ONJKrKzon4v9nS3nLZe0= @@ -224,7 +218,7 @@ cloud.google.com/go/spanner v1.56.0/go.mod h1:DndqtUKQAt3VLuV2Le+9Y3WTnq5cNKrnLb cloud.google.com/go/speech v1.21.1 h1:nuFc+Kj5B8de75nN4FdPyUbI2SiBoHZG6BLurXL56Q0= cloud.google.com/go/speech v1.21.1/go.mod h1:E5GHZXYQlkqWQwY5xRSLHw2ci5NMQNG52FfMU1aZrIA= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= cloud.google.com/go/storagetransfer v1.10.4 h1:dy4fL3wO0VABvzM05ycMUPFHxTPbJz9Em8ikAJVqSbI= @@ -258,11 +252,10 @@ cloud.google.com/go/websecurityscanner v1.6.5/go.mod h1:QR+DWaxAz2pWooylsBF854/I cloud.google.com/go/workflows v1.12.4 h1:uHNmUiatTbPQ4H1pabwfzpfEYD4BBnqDHqMm2IesOh4= cloud.google.com/go/workflows v1.12.4/go.mod h1:yQ7HUqOkdJK4duVtMeBCAOPiN1ZF1E9pAMX51vpwB/w= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= @@ -274,6 +267,7 @@ github.com/aphistic/sweet v0.2.0 h1:I4z+fAUqvKfvZV/CHi5dV0QuwbmIvYYFDjG0Ss5QpAs= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= +github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4= github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -304,14 +298,14 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/crawlab-team/crawlab-db v0.6.0-1 h1:Is3a3q1epc7mFP4iDIr9nXlcHSW5IL6jXKQ3YEsXpVo= -github.com/crawlab-team/crawlab-db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= @@ -330,11 +324,15 @@ github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= @@ -342,8 +340,6 @@ github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -355,7 +351,6 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= @@ -372,7 +367,6 @@ github.com/googleapis/gax-go/v2 v2.12.1/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7 github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -403,6 +397,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weK github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= @@ -410,10 +407,6 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= @@ -435,18 +428,18 @@ github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWe github.com/mmcloughlin/avo v0.5.0 h1:nAco9/aI9Lg2kiuROBY6BhCI/z0t5jEvJfjWbL8qXLU= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/prometheus/client_golang v1.4.0 h1:YVIb/fVcOTMSqtqZWSKnHpSLBxu8DKgxq8z6RuBZwqI= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= @@ -457,6 +450,8 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMoqGRA7G7paDNDqTXE30mXGqzzybrfo05w= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= @@ -465,22 +460,21 @@ github.com/sagikazarmark/crypt v0.3.0 h1:TV5DVog+pihN4Rr0rN1IClv4ePpkzdg9sPrw7WD github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= +github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129 h1:ORfYNrFCM8HMZkU8QFX9Z8M6NWMlQnamJpSI5s0l4cM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CYQUINdZMHdvTdXtPOY2ainKl4IoMcpAXEf2xj3Q= github.com/smartystreets/gunit v1.0.0 h1:RyPDUFcJbvtXlhJPk7v+wnxZRY2EUokhEYl2EJOPToI= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tj/go-buffer v1.1.0 h1:Lo2OsPHlIxXF24zApe15AbK3bJLAOvkkxEA6Ux4c47M= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2 h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9mdZU6shPewugMyH5+GV6LNgW8w= github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= +github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= @@ -526,10 +520,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= @@ -540,11 +531,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -557,14 +545,12 @@ golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5H golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -574,9 +560,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqG golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= @@ -590,8 +574,7 @@ google.golang.org/api v0.164.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeq google.golang.org/api v0.166.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA= google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= diff --git a/template-parser/.gitignore b/template-parser/.gitignore new file mode 100644 index 000000000..ec5b69a88 --- /dev/null +++ b/template-parser/.gitignore @@ -0,0 +1,18 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +.idea +.DS_Store \ No newline at end of file diff --git a/template-parser/LICENSE b/template-parser/LICENSE new file mode 100644 index 000000000..2764ea24c --- /dev/null +++ b/template-parser/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Crawlab Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/template-parser/README.md b/template-parser/README.md new file mode 100644 index 000000000..a03104ff2 --- /dev/null +++ b/template-parser/README.md @@ -0,0 +1 @@ +# template-parser \ No newline at end of file diff --git a/template-parser/entity.go b/template-parser/entity.go new file mode 100644 index 000000000..d2951418a --- /dev/null +++ b/template-parser/entity.go @@ -0,0 +1,6 @@ +package parser + +// type: model / attribute + +// {{task.spider.name}} +// => diff --git a/template-parser/func.go b/template-parser/func.go new file mode 100644 index 000000000..ec464b716 --- /dev/null +++ b/template-parser/func.go @@ -0,0 +1,13 @@ +package parser + +func Parse(template string, args ...interface{}) (content string, err error) { + return ParseGeneral(template, args...) +} + +func ParseGeneral(template string, args ...interface{}) (content string, err error) { + p, _ := NewGeneralParser() + if err := p.Parse(template); err != nil { + return "", err + } + return p.Render(args...) +} diff --git a/template-parser/general_parser.go b/template-parser/general_parser.go new file mode 100644 index 000000000..ba08fd684 --- /dev/null +++ b/template-parser/general_parser.go @@ -0,0 +1,208 @@ +package parser + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/robertkrimen/otto" + "regexp" + "strings" +) + +type GeneralParser struct { + tagPattern string + tagRegexp *regexp.Regexp + mathPattern string + mathRegexp *regexp.Regexp + template string + indexes [][]int + matches [][]string + placeholders []string + variables []Variable + vm *otto.Otto +} + +const VariableNameResult = "result" +const ValueNameNA = "N/A" + +func (p *GeneralParser) Parse(template string) (err error) { + p.template = template + p.indexes = p.tagRegexp.FindAllStringIndex(template, -1) + p.matches = p.tagRegexp.FindAllStringSubmatch(template, -1) + for _, arr := range p.matches { + p.placeholders = append(p.placeholders, arr[1]) + } + + return nil +} + +func (p *GeneralParser) Render(args ...interface{}) (content string, err error) { + // render tag content + content, err = p.renderTagContent(args...) + if err != nil { + return content, err + } + + // render math content + content, err = p.renderMathContent(content) + if err != nil { + return content, err + } + + return content, nil +} + +func (p *GeneralParser) renderTagContent(args ...interface{}) (content string, err error) { + // validate + if len(args) == 0 { + return "", errors.New("no arguments") + } + + // first argument + arg := args[0] + + // content + content = p.template + + // old strings + var oldStrList []string + for _, index := range p.indexes { + // old string + oldStr := content[index[0]:index[1]] + oldStrList = append(oldStrList, oldStr) + } + + // iterate placeholders + for i, placeholder := range p.placeholders { + // variable + v, err := NewVariable(arg, placeholder) + if err != nil { + return "", err + } + + // value + value, err := v.GetValue() + if err != nil || value == nil { + value = ValueNameNA + } + + // old string + oldStr := oldStrList[i] + + // new string + var newStr string + switch value.(type) { + case string: + newStr = value.(string) + default: + newStrBytes, err := json.Marshal(value) + if err != nil { + return "", err + } + newStr = string(newStrBytes) + } + + // replace old string with new string + content = strings.Replace(content, oldStr, newStr, 1) + } + + return content, nil +} + +func (p *GeneralParser) renderMathContent(inputContent string) (content string, err error) { + content = inputContent + indexes := p.mathRegexp.FindAllStringIndex(inputContent, -1) + matches := p.mathRegexp.FindAllStringSubmatch(inputContent, -1) + + // old strings + var oldStrList []string + for _, index := range indexes { + // old string + oldStr := content[index[0]:index[1]] + oldStrList = append(oldStrList, oldStr) + } + + // iterate matches + for i, m := range matches { + // js script to run to get evaluate result + script := fmt.Sprintf("%s = %s; %s", VariableNameResult, m[1], VariableNameResult) + + // replace NA + script = strings.ReplaceAll(script, ValueNameNA, "NaN") + + // value + value, err := p.vm.Run(script) + if err != nil { + return "", err + } + + // old string + oldStr := oldStrList[i] + + // new string + newStr := value.String() + + // replace old string with new string + content = strings.Replace(content, oldStr, newStr, 1) + } + + return content, nil +} + +func (p *GeneralParser) GetPlaceholders() (placeholders []string) { + return p.placeholders +} + +func NewGeneralParser() (p Parser, err error) { + // tag regexp + tagPrefix := "\\{\\{" + tagSuffix := "\\}\\}" + tagBasicChars := "\\$\\.\\w_" + tagAssociateChars := "\\[\\]:" + tagPattern := fmt.Sprintf( + "%s *([%s%s]+) *%s", + tagPrefix, + tagBasicChars, + tagAssociateChars, + tagSuffix, + ) + tagRegexp, err := regexp.Compile(tagPattern) + if err != nil { + return nil, err + } + + // math regexp + mathPrefix := "\\{#" + mathSuffix := "#\\}" + mathBasicChars := " \\(\\)" + mathOpChars := "\\+\\-\\*/%" + mathNumChars := "\\d\\." + mathSpecialChars := "(?:NaN|null)" + mathPattern := fmt.Sprintf( + "%s *([%s%s%s%s]+) *%s", + mathPrefix, + mathBasicChars, + mathOpChars, + mathNumChars, + mathSpecialChars, + mathSuffix, + ) + mathRegexp, err := regexp.Compile(mathPattern) + if err != nil { + return nil, err + } + + // math vm + vm := otto.New() + + // parser + p = &GeneralParser{ + tagPattern: tagPattern, + tagRegexp: tagRegexp, + mathPattern: mathPattern, + mathRegexp: mathRegexp, + vm: vm, + } + + return p, nil +} diff --git a/template-parser/go.mod b/template-parser/go.mod new file mode 100644 index 000000000..7af02e620 --- /dev/null +++ b/template-parser/go.mod @@ -0,0 +1,47 @@ +module github.com/crawlab-team/crawlab/template-parser + +go 1.22 + +replace github.com/crawlab-team/crawlab/db => ../db + +require ( + github.com/crawlab-team/crawlab/db v0.6.0-1 + github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 + github.com/stretchr/testify v1.9.0 + go.mongodb.org/mongo-driver v1.15.0 +) + +require ( + github.com/apex/log v1.9.0 // indirect + github.com/cenkalti/backoff/v4 v4.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.10.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.10.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/sourcemap.v1 v1.0.5 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/template-parser/go.sum b/template-parser/go.sum new file mode 100644 index 000000000..e7103492b --- /dev/null +++ b/template-parser/go.sum @@ -0,0 +1,249 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= +github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= +github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/template-parser/interfaces.go b/template-parser/interfaces.go new file mode 100644 index 000000000..9a68b7787 --- /dev/null +++ b/template-parser/interfaces.go @@ -0,0 +1,8 @@ +package parser + +type Entity interface { + GetType() string + SetType(string) + GetName() string + SetName(string) +} diff --git a/template-parser/parser.go b/template-parser/parser.go new file mode 100644 index 000000000..a1fce8a63 --- /dev/null +++ b/template-parser/parser.go @@ -0,0 +1,6 @@ +package parser + +type Parser interface { + Parse(template string) (err error) + Render(args ...interface{}) (content string, err error) +} diff --git a/template-parser/test/general_parser_test.go b/template-parser/test/general_parser_test.go new file mode 100644 index 000000000..f4c1f1e87 --- /dev/null +++ b/template-parser/test/general_parser_test.go @@ -0,0 +1,141 @@ +package test + +import ( + "fmt" + "github.com/crawlab-team/crawlab/db/mongo" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "testing" +) + +func TestGeneralParser(t *testing.T) { + p, _ := parser.NewGeneralParser() + content := `The task {{ $.node }} (enabled: {{$.node.enabled}}) has completed. +Yours, {{$.user[create]}}` + err := p.Parse(content) + fmt.Println(p.(*parser.GeneralParser).GetPlaceholders()) + require.Nil(t, err) +} + +func TestGeneralParser_Parse(t *testing.T) { + var err error + t.Cleanup(cleanup) + + nodeId := primitive.NewObjectID() + _, err = mongo.GetMongoCol("nodes").Insert(bson.M{ + "_id": nodeId, + "name": "Test Node", + "enabled": true, + "settings": bson.M{ + "max_runners": 8, + }, + }) + require.Nil(t, err) + spiderId := primitive.NewObjectID() + _, err = mongo.GetMongoCol("spiders").Insert(bson.M{ + "_id": spiderId, + }) + require.Nil(t, err) + _, err = mongo.GetMongoCol("spider_stats").Insert(bson.M{ + "_id": spiderId, + "result_count": 5000, + }) + require.Nil(t, err) + userId := primitive.NewObjectID() + _, err = mongo.GetMongoCol("users").Insert(bson.M{ + "_id": userId, + "no": 1001, + "username": "Test Username", + }) + require.Nil(t, err) + userIdUpdate := primitive.NewObjectID() + _, err = mongo.GetMongoCol("users").Insert(bson.M{ + "_id": userIdUpdate, + "no": 1002, + "username": "Test2 Username", + }) + require.Nil(t, err) + + taskId := primitive.NewObjectID() + task := bson.M{ + "_id": taskId, + "node_id": nodeId, + "spider_id": spiderId, + } + _, err = mongo.GetMongoCol("task_stats").Insert(bson.M{ + "_id": taskId, + "result_count": 100, + }) + require.Nil(t, err) + _, err = mongo.GetMongoCol("artifacts").Insert(bson.M{ + "_id": taskId, + "_sys": bson.M{ + "create_uid": userId, + "update_uid": userIdUpdate, + }, + }) + + p, _ := parser.NewGeneralParser() + template := `The task on node {{ $.node.name }} (enabled: {{$.node.enabled}}, max_runners: {{$.node.settings.max_runners}}) has completed. +Task Result Count: {{ $.:task_stat.result_count }} +Spider Result Count: {{ $.spider:stat.result_count }} +Yours, {{$.user.username}} ({{$.user.no}}) and {{$.user[update].username}} ({{$.user[update].no}})` + err = p.Parse(template) + require.Nil(t, err) + + content, err := p.Render(task) + require.Nil(t, err) + require.Equal(t, `The task on node Test Node (enabled: true, max_runners: 8) has completed. +Task Result Count: 100 +Spider Result Count: 5000 +Yours, Test Username (1001) and Test2 Username (1002)`, content) +} + +func TestGeneralParser_ParseMath(t *testing.T) { + var err error + t.Cleanup(cleanup) + + taskId := primitive.NewObjectID() + task := bson.M{ + "_id": taskId, + } + _, err = mongo.GetMongoCol("task_stats").Insert(bson.M{ + "_id": taskId, + "wait_duration": 20000, + "runtime_duration": 80000, + "total_duration": 100000, + "result_count": 500, + }) + require.Nil(t, err) + + p, _ := parser.NewGeneralParser() + template := `The task has completed. +Wait Duration: {# {{ $.:task_stat.wait_duration }} / 1000 #} +Runtime Duration: {# {{$.:task_stat.runtime_duration}} / 1000 #} +Total Duration: {# ({{$.:task_stat.wait_duration}} + {{$.:task_stat.runtime_duration}}) / 1000 #} +Result Count: {{$.:task_stat.result_count}} +Avg Results per Sec: {# {{$.:task_stat.result_count}} / ({{$.:task_stat.total_duration}} / 1000) #} +` + err = p.Parse(template) + require.Nil(t, err) + + content, err := p.Render(task) + require.Nil(t, err) + require.Equal(t, `The task has completed. +Wait Duration: 20 +Runtime Duration: 80 +Total Duration: 100 +Result Count: 500 +Avg Results per Sec: 5 +`, content) +} + +func cleanup() { + _ = mongo.GetMongoCol("nodes").Delete(nil) + _ = mongo.GetMongoCol("spiders").Delete(nil) + _ = mongo.GetMongoCol("spider_stats").Delete(nil) + _ = mongo.GetMongoCol("tasks").Delete(nil) + _ = mongo.GetMongoCol("task_stats").Delete(nil) + _ = mongo.GetMongoCol("users").Delete(nil) +} diff --git a/template-parser/variable.go b/template-parser/variable.go new file mode 100644 index 000000000..4e2a2a18f --- /dev/null +++ b/template-parser/variable.go @@ -0,0 +1,266 @@ +package parser + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/crawlab-team/crawlab/db/mongo" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "regexp" + "strings" +) + +var userRegexp, _ = regexp.Compile("^user(?:\\[(\\w+)\\])?$") +var extRegexp, _ = regexp.Compile("^(\\w+)?:(\\w+)") + +type Variable struct { + root interface{} + tokens []string + doc bson.M +} + +func (v *Variable) GetValue() (value interface{}, err error) { + return v.getNodeByIndex(len(v.tokens) - 1) +} + +func (v *Variable) getNodeByIndex(index int) (result interface{}, err error) { + node := v.doc + for i := 0; i < index && i < len(v.tokens); i++ { + nextIndex := i + 1 + if nextIndex < len(v.tokens)-1 { + // root or intermediate node + } else { + // value + return v.getNextValue(node, i), nil + } + + // next node + node, err = v.getNextNode(node, i) + if err != nil { + return nil, err + } + if node == nil { + return nil, nil + } + } + return node, nil +} + +func (v *Variable) getNextNode(currentNode bson.M, currentIndex int) (nextNode bson.M, err error) { + // next index and token + nextIndex := currentIndex + 1 + nextToken := v.tokens[nextIndex] + + // attempt to get attribute in current node + nextNodeRes, ok := currentNode[nextToken] + if ok { + nextNode, ok = nextNodeRes.(bson.M) + if ok { + return nextNode, nil + } + } + + // if next token is user or user[] + if userRegexp.MatchString(nextToken) { + return v.getNextNodeUserAction(currentNode, nextIndex, nextToken) + } + + // if next token is : + if extRegexp.MatchString(nextToken) { + return v.getNextNodeExt(currentNode, nextIndex, nextToken) + } + + // model + model := nextToken + + // next id + nextId, err := v._getNodeModelId(currentNode, model, nextIndex) + if err != nil { + return nil, err + } + + // mongo collection name + colName := fmt.Sprintf("%ss", model) + + // get next node from mongo collection + if err := mongo.GetMongoCol(colName).FindId(nextId).One(&nextNode); err != nil { + return nil, err + } + + return nextNode, nil +} + +func (v *Variable) getNextNodeUserAction(currentNode bson.M, nextIndex int, nextToken string) (nextNode bson.M, err error) { + // action + matches := userRegexp.FindStringSubmatch(nextToken) + action := "create" + if len(matches) > 1 && matches[1] != "" { + action = matches[1] + } + + // id + id, err := v._getCurrentNodeId(currentNode, nextIndex) + if err != nil { + return nil, err + } + + // artifact + var artifact bson.M + if err := mongo.GetMongoCol("artifacts").FindId(id).One(&artifact); err != nil { + return nil, err + } + + // artifact._sys + sysRes, ok := artifact["_sys"] + if !ok { + return nil, errors.New(fmt.Sprintf("_sys not exists in artifact of %s", strings.Join(v.tokens[:nextIndex], "."))) + } + sys, ok := sysRes.(bson.M) + if !ok { + return nil, errors.New(fmt.Sprintf("_sys is invalid in artifact of %s", strings.Join(v.tokens[:nextIndex], "."))) + } + + // _uid + uidRes, _ := sys[action+"_uid"] + uid, _ := uidRes.(primitive.ObjectID) + if err := mongo.GetMongoCol("users").FindId(uid).One(&nextNode); err != nil { + return nil, err + } + + return nextNode, nil +} + +func (v *Variable) getNextNodeExt(currentNode bson.M, nextIndex int, nextToken string) (nextNode bson.M, err error) { + matches := extRegexp.FindStringSubmatch(nextToken) + var model, ext string + var mode int + if matches[1] == "" { + // :_ + mode = 0 + arr := strings.Split(matches[2], "_") + model = arr[0] + ext = arr[1] + } else { + // : + mode = 1 + model = matches[1] + ext = matches[2] + } + + // id + var id primitive.ObjectID + switch mode { + case 0: + // id = currentNode._id + id, err = v._getCurrentNodeId(currentNode, nextIndex) + if err != nil { + return nil, err + } + case 1: + // id = currentNode._id + id, err = v._getNodeModelId(currentNode, model, nextIndex) + if err != nil { + return nil, err + } + default: + return nil, errors.New(fmt.Sprintf("invalid mode: %d", mode)) + } + + // ext + colName := fmt.Sprintf("%s_%ss", model, ext) + if err := mongo.GetMongoCol(colName).FindId(id).One(&nextNode); err != nil { + return nil, err + } + + return nextNode, nil +} + +func (v *Variable) getNextValue(currentNode bson.M, currentIndex int) (nextValue interface{}) { + // next index and token + nextIndex := currentIndex + 1 + nextToken := v.tokens[nextIndex] + + // next value + nextValue, _ = currentNode[nextToken] + + return nextValue +} + +func (v *Variable) _getCurrentNodeId(currentNode bson.M, nextIndex int) (id primitive.ObjectID, err error) { + idRes, ok := currentNode["_id"] + if !ok { + return id, errors.New(fmt.Sprintf("%s is not available in %s", "_id", strings.Join(v.tokens[:nextIndex], "."))) + } + switch idRes.(type) { + case string: + idStr, ok := idRes.(string) + if !ok { + return id, errors.New(fmt.Sprintf("%s is not ObjectId in %s", "_id", strings.Join(v.tokens[:nextIndex], "."))) + } + id, err = primitive.ObjectIDFromHex(idStr) + if err != nil { + return id, errors.New(fmt.Sprintf("%s is not ObjectId in %s", "_id", strings.Join(v.tokens[:nextIndex], "."))) + } + return id, nil + case primitive.ObjectID: + return idRes.(primitive.ObjectID), nil + default: + return id, errors.New(fmt.Sprintf("%s is not ObjectId in %s", "_id", strings.Join(v.tokens[:nextIndex], "."))) + } +} + +func (v *Variable) _getNodeModelId(currentNode bson.M, model string, nextIndex int) (id primitive.ObjectID, err error) { + nextIdKey := fmt.Sprintf("%s_id", model) + nextIdRes, ok := currentNode[nextIdKey] + if !ok { + return id, errors.New(fmt.Sprintf("%s is not available in %s", nextIdKey, strings.Join(v.tokens[:nextIndex], "."))) + } + nextId, ok := nextIdRes.(primitive.ObjectID) + if !ok { + nextIdStr, ok := nextIdRes.(string) + if !ok { + return id, errors.New(fmt.Sprintf("%s is not ObjectId in %s", nextIdKey, strings.Join(v.tokens[:nextIndex], "."))) + } + nextId, err = primitive.ObjectIDFromHex(nextIdStr) + if err != nil { + return id, err + } + } + if nextId.IsZero() { + return id, nil + } + + return nextId, nil +} + +func NewVariable(root interface{}, placeholder string) (v *Variable, err error) { + // validate + if placeholder == "" { + return nil, errors.New("empty placeholder") + } + if !strings.HasPrefix(placeholder, "$") { + return nil, errors.New("not start with $") + } + + // tokens + tokens := strings.Split(placeholder, ".") + + // document + data, err := json.Marshal(root) + if err != nil { + return nil, err + } + var doc bson.M + if err := json.Unmarshal(data, &doc); err != nil { + return nil, err + } + + v = &Variable{ + root: root, + tokens: tokens, + doc: doc, + } + + return v, nil +} diff --git a/trace b/trace new file mode 160000 index 000000000..188cd58fe --- /dev/null +++ b/trace @@ -0,0 +1 @@ +Subproject commit 188cd58fec6e9cbee13060594f065bf811dea212 diff --git a/vcs/.github/workflows/test.yml b/vcs/.github/workflows/test.yml new file mode 100644 index 000000000..dd57dea5d --- /dev/null +++ b/vcs/.github/workflows/test.yml @@ -0,0 +1,35 @@ +name: Test + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup Go environment + uses: actions/setup-go@v2.1.3 + with: + go-version: 1.18 + + - name: Configure Git + run: | + git config --global user.name "Marvin Zhang" + git config --global user.email "tikazyq@163.com" + + - name: Write Credentials + run: | + echo '${{ secrets.CREDENTIAL_JSON }}' > $GITHUB_WORKSPACE/credentials.json + echo '${{ secrets.CREDENTIAL_JSON }}' > $GITHUB_WORKSPACE/test/credentials.json + ls -l $GITHUB_WORKSPACE/test + mkdir -p $GITHUB_WORKSPACE/.ssh + echo "${{ secrets.SSH_KEY }}" > $GITHUB_WORKSPACE/id_rsa + ls -la $GITHUB_WORKSPACE + echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE" + + - name: Install + run: go mod tidy + + - name: Run Tests + run: go test ./... -race diff --git a/vcs/.gitignore b/vcs/.gitignore new file mode 100644 index 000000000..3c32c3fec --- /dev/null +++ b/vcs/.gitignore @@ -0,0 +1,22 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +.DS_Store +.idea/ +tmp/ +test/credentials.json +id_rsa* +credentials.json +coverage.txt \ No newline at end of file diff --git a/vcs/LICENSE b/vcs/LICENSE new file mode 100644 index 000000000..e41311d27 --- /dev/null +++ b/vcs/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Crawlab Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vcs/README.md b/vcs/README.md new file mode 100644 index 000000000..427990385 --- /dev/null +++ b/vcs/README.md @@ -0,0 +1,2 @@ +# crawlab-vcs +Version Control System (VCS) for Crawlab diff --git a/vcs/constants.go b/vcs/constants.go new file mode 100644 index 000000000..7ac534a6b --- /dev/null +++ b/vcs/constants.go @@ -0,0 +1,37 @@ +package vcs + +const ( + GitRemoteNameOrigin = "origin" + GitRemoteNameUpstream = "upstream" + GitRemoteNameCrawlab = "crawlab" +) +const GitDefaultRemoteName = GitRemoteNameOrigin + +const ( + GitBranchNameMaster = "master" + GitBranchNameMain = "main" + GitBranchNameRelease = "release" + GitBranchNameTest = "test" + GitBranchNameDevelop = "develop" +) +const GitDefaultBranchName = GitBranchNameMaster + +type GitAuthType int + +const ( + GitAuthTypeNone GitAuthType = iota + GitAuthTypeHTTP + GitAuthTypeSSH +) + +type GitInitType int + +const ( + GitInitTypeFs GitInitType = iota + GitInitTypeMem +) + +const ( + GitRefTypeBranch = "branch" + GitRefTypeTag = "tag" +) diff --git a/vcs/entity.go b/vcs/entity.go new file mode 100644 index 000000000..509e3784b --- /dev/null +++ b/vcs/entity.go @@ -0,0 +1,36 @@ +package vcs + +import ( + "time" +) + +type GitOptions struct { + checkout []GitCheckoutOption +} + +type GitRef struct { + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + Hash string `json:"hash"` + Timestamp time.Time `json:"timestamp"` +} + +type GitLog struct { + Hash string `json:"hash"` + Msg string `json:"msg"` + AuthorName string `json:"author_name"` + AuthorEmail string `json:"author_email"` + Timestamp time.Time `json:"timestamp"` + Refs []GitRef `json:"refs"` +} + +type GitFileStatus struct { + Path string `json:"path"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + Staging string `json:"staging"` + Worktree string `json:"worktree"` + Extra string `json:"extra"` + Children []GitFileStatus `json:"children"` +} diff --git a/vcs/errors.go b/vcs/errors.go new file mode 100644 index 000000000..fe56bb07d --- /dev/null +++ b/vcs/errors.go @@ -0,0 +1,16 @@ +package vcs + +import "errors" + +var ( + ErrInvalidArgsLength = errors.New("invalid arguments length") + ErrUnsupportedType = errors.New("unsupported type") + ErrInvalidAuthType = errors.New("invalid auth type") + ErrInvalidOptions = errors.New("invalid options") + ErrRepoAlreadyExists = errors.New("repo already exists") + ErrInvalidRepoPath = errors.New("invalid repo path") + ErrUnableToGetCurrentBranch = errors.New("unable to get current branch") + ErrUnableToCloneWithEmptyRemoteUrl = errors.New("unable to clone with empty remote url") + ErrInvalidHeadRef = errors.New("invalid head ref") + ErrNoMatchedRemoteBranch = errors.New("no matched remote branch") +) diff --git a/vcs/git.go b/vcs/git.go new file mode 100644 index 000000000..3ed5a4570 --- /dev/null +++ b/vcs/git.go @@ -0,0 +1,1019 @@ +package vcs + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/trace" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/go-git/go-git/v5/storage/memory" + "golang.org/x/crypto/ssh" + "io/ioutil" + "os" + "path" + "regexp" + "sort" + "strings" +) + +var headRefRegexp, _ = regexp.Compile("^ref: (.*)") + +type GitClient struct { + // settings + path string + remoteUrl string + isMem bool + authType GitAuthType + username string + password string + privateKey string + privateKeyPath string + defaultBranch string + + // internals + r *git.Repository +} + +func (c *GitClient) Init() (err error) { + initType := c.getInitType() + switch initType { + case GitInitTypeFs: + err = c.initFs() + case GitInitTypeMem: + err = c.initMem() + } + if err != nil { + return err + } + + // if remote url is not empty and no remote exists + // create default remote and pull from remote url + remotes, err := c.r.Remotes() + if err != nil { + return err + } + if c.remoteUrl != "" && len(remotes) == 0 { + // attempt to get default remote + if _, err := c.r.Remote(GitRemoteNameOrigin); err != nil { + if err != git.ErrRemoteNotFound { + return trace.TraceError(err) + } + err = nil + + // create default remote + if err := c.createRemote(GitRemoteNameOrigin, c.remoteUrl); err != nil { + return err + } + + //// pull + //opts := []GitPullOption{ + // WithRemoteNamePull(GitRemoteNameOrigin), + //} + //if err := c.Pull(opts...); err != nil { + // return err + //} + } + } + + return nil +} + +func (c *GitClient) Dispose() (err error) { + switch c.getInitType() { + case GitInitTypeFs: + if err := os.RemoveAll(c.path); err != nil { + return trace.TraceError(err) + } + case GitInitTypeMem: + GitMemStorages.Delete(c.path) + GitMemFileSystem.Delete(c.path) + } + return nil +} + +func (c *GitClient) Checkout(opts ...GitCheckoutOption) (err error) { + // worktree + wt, err := c.r.Worktree() + if err != nil { + return trace.TraceError(err) + } + + // apply options + o := &git.CheckoutOptions{} + for _, opt := range opts { + opt(o) + } + + // checkout to the branch + if err := wt.Checkout(o); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) Commit(msg string, opts ...GitCommitOption) (err error) { + // worktree + wt, err := c.r.Worktree() + if err != nil { + return trace.TraceError(err) + } + + // apply options + o := &git.CommitOptions{} + for _, opt := range opts { + opt(o) + } + + // commit + if _, err := wt.Commit(msg, o); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) Pull(opts ...GitPullOption) (err error) { + // worktree + wt, err := c.r.Worktree() + if err != nil { + return trace.TraceError(err) + } + + // auth + auth, err := c.getGitAuth() + if err != nil { + return err + } + if auth != nil { + opts = append(opts, WithAuthPull(auth)) + } + + // apply options + o := &git.PullOptions{} + for _, opt := range opts { + opt(o) + } + + // pull + if err := wt.Pull(o); err != nil { + if err == transport.ErrEmptyRemoteRepository { + return nil + } + if err == transport.ErrEmptyUploadPackRequest { + return nil + } + if err == git.NoErrAlreadyUpToDate { + return nil + } + if err == git.ErrNonFastForwardUpdate { + return nil + } + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) Push(opts ...GitPushOption) (err error) { + // auth + auth, err := c.getGitAuth() + if err != nil { + return err + } + if auth != nil { + opts = append(opts, WithAuthPush(auth)) + } + + // apply options + o := &git.PushOptions{} + for _, opt := range opts { + opt(o) + } + + // push + if err := c.r.Push(o); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) Reset(opts ...GitResetOption) (err error) { + // apply options + o := &git.ResetOptions{ + Mode: git.HardReset, + } + for _, opt := range opts { + opt(o) + } + + // worktree + wt, err := c.r.Worktree() + if err != nil { + return err + } + + // reset + if err := wt.Reset(o); err != nil { + return err + } + + // clean + if err := wt.Clean(&git.CleanOptions{Dir: true}); err != nil { + return err + } + + return nil +} + +func (c *GitClient) CreateBranch(branch, remote string, ref *plumbing.Reference) (err error) { + return c.createBranch(branch, remote, ref) +} + +func (c *GitClient) CheckoutBranchFromRef(branch string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) { + return c.CheckoutBranchWithRemote(branch, "", ref, opts...) +} + +func (c *GitClient) CheckoutBranchWithRemoteFromRef(branch, remote string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) { + return c.CheckoutBranchWithRemote(branch, remote, ref, opts...) +} + +func (c *GitClient) CheckoutBranch(branch string, opts ...GitCheckoutOption) (err error) { + return c.CheckoutBranchWithRemote(branch, "", nil, opts...) +} + +func (c *GitClient) CheckoutBranchWithRemote(branch, remote string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) { + if remote == "" { + remote = GitRemoteNameOrigin + } + + // remote + if _, err := c.r.Remote(remote); err != nil { + return trace.TraceError(err) + } + + // check if the branch exists + b, err := c.r.Branch(branch) + if err != nil { + if err == git.ErrBranchNotFound { + // create a new branch if it does not exist + if err := c.createBranch(branch, remote, ref); err != nil { + return err + } + b, err = c.r.Branch(branch) + if err != nil { + return trace.TraceError(err) + } + } else { + return trace.TraceError(err) + } + } + + // set branch remote + if remote != "" { + b.Remote = remote + } + + // add to options + opts = append(opts, WithBranch(branch)) + + return c.Checkout(opts...) +} + +func (c *GitClient) CheckoutHash(hash string, opts ...GitCheckoutOption) (err error) { + // add to options + opts = append(opts, WithHash(hash)) + + return c.Checkout(opts...) +} + +func (c *GitClient) MoveBranch(from, to string) (err error) { + wt, err := c.r.Worktree() + if err != nil { + return trace.TraceError(err) + } + if err := wt.Checkout(&git.CheckoutOptions{ + Create: true, + Branch: plumbing.NewBranchReferenceName(to), + }); err != nil { + return trace.TraceError(err) + } + fromRef, err := c.r.Reference(plumbing.NewBranchReferenceName(from), false) + if err != nil { + return trace.TraceError(err) + } + if err := c.r.Storer.RemoveReference(fromRef.Name()); err != nil { + return trace.TraceError(err) + } + return nil +} + +func (c *GitClient) CommitAll(msg string, opts ...GitCommitOption) (err error) { + // worktree + wt, err := c.r.Worktree() + if err != nil { + return trace.TraceError(err) + } + + // add all files + if _, err := wt.Add("."); err != nil { + return trace.TraceError(err) + } + + return c.Commit(msg, opts...) +} + +func (c *GitClient) GetLogs() (logs []GitLog, err error) { + iter, err := c.r.Log(&git.LogOptions{ + All: true, + }) + if err != nil { + return nil, trace.TraceError(err) + } + if err := iter.ForEach(func(commit *object.Commit) error { + log := GitLog{ + Hash: commit.Hash.String(), + Msg: commit.Message, + AuthorName: commit.Author.Name, + AuthorEmail: commit.Author.Email, + Timestamp: commit.Author.When, + } + logs = append(logs, log) + return nil + }); err != nil { + return nil, trace.TraceError(err) + } + return +} + +func (c *GitClient) GetLogsWithRefs() (logs []GitLog, err error) { + // logs without tags + logs, err = c.GetLogs() + if err != nil { + return nil, err + } + + // branches + branches, err := c.GetBranches() + if err != nil { + return nil, err + } + + // tags + tags, err := c.GetTags() + if err != nil { + return nil, err + } + + // refs + refs := append(branches, tags...) + + // refs map + refsMap := map[string][]GitRef{} + for _, ref := range refs { + _, ok := refsMap[ref.Hash] + if !ok { + refsMap[ref.Hash] = []GitRef{} + } + refsMap[ref.Hash] = append(refsMap[ref.Hash], ref) + } + + // iterate logs + for i, l := range logs { + refs, ok := refsMap[l.Hash] + if ok { + logs[i].Refs = refs + } + } + + return logs, nil +} + +func (c *GitClient) GetRepository() (r *git.Repository) { + return c.r +} + +func (c *GitClient) GetPath() (path string) { + return c.path +} + +func (c *GitClient) SetPath(path string) { + c.path = path +} + +func (c *GitClient) GetRemoteUrl() (path string) { + return c.remoteUrl +} + +func (c *GitClient) SetRemoteUrl(url string) { + c.remoteUrl = url +} + +func (c *GitClient) GetIsMem() (isMem bool) { + return c.isMem +} + +func (c *GitClient) SetIsMem(isMem bool) { + c.isMem = isMem +} + +func (c *GitClient) GetAuthType() (authType GitAuthType) { + return c.authType +} + +func (c *GitClient) SetAuthType(authType GitAuthType) { + c.authType = authType +} + +func (c *GitClient) GetUsername() (username string) { + return c.username +} + +func (c *GitClient) SetUsername(username string) { + c.username = username +} + +func (c *GitClient) GetPassword() (password string) { + return c.password +} + +func (c *GitClient) SetPassword(password string) { + c.password = password +} + +func (c *GitClient) GetPrivateKey() (key string) { + return c.privateKey +} + +func (c *GitClient) SetPrivateKey(key string) { + c.privateKey = key +} + +func (c *GitClient) GetPrivateKeyPath() (path string) { + return c.privateKeyPath +} + +func (c *GitClient) SetPrivateKeyPath(path string) { + c.privateKeyPath = path +} + +func (c *GitClient) GetCurrentBranch() (branch string, err error) { + // attempt to get branch from .git/HEAD + headRefStr, err := c.getHeadRef() + if err != nil { + return "", err + } + + // if .git/HEAD points to refs/heads/master, return branch as master + if headRefStr == plumbing.Master.String() { + return GitBranchNameMaster, nil + } + + // attempt to get head ref + headRef, err := c.r.Head() + if err != nil { + return "", trace.TraceError(err) + } + if !headRef.Name().IsBranch() { + return "", trace.TraceError(ErrUnableToGetCurrentBranch) + } + + return headRef.Name().Short(), nil +} + +func (c *GitClient) GetCurrentBranchRef() (ref *GitRef, err error) { + currentBranch, err := c.GetCurrentBranch() + if err != nil { + return nil, err + } + branches, err := c.GetBranches() + if err != nil { + return nil, err + } + for _, branch := range branches { + if branch.Name == currentBranch { + return &branch, nil + } + } + return nil, trace.TraceError(ErrUnableToGetCurrentBranch) +} + +func (c *GitClient) GetBranches() (branches []GitRef, err error) { + iter, err := c.r.Branches() + if err != nil { + return nil, trace.TraceError(err) + } + + _ = iter.ForEach(func(r *plumbing.Reference) error { + branches = append(branches, GitRef{ + Type: GitRefTypeBranch, + Name: r.Name().Short(), + Hash: r.Hash().String(), + }) + return nil + }) + + return branches, nil +} + +func (c *GitClient) GetRemoteRefs(remoteName string) (gitRefs []GitRef, err error) { + // remote + r, err := c.r.Remote(remoteName) + if err != nil { + if err == git.ErrRemoteNotFound { + return nil, nil + } + return nil, trace.TraceError(err) + } + + // auth + auth, err := c.getGitAuth() + if err != nil { + return nil, err + } + + // refs + refs, err := r.List(&git.ListOptions{Auth: auth}) + if err != nil { + if err != transport.ErrEmptyRemoteRepository { + return nil, trace.TraceError(err) + } + return nil, nil + } + + // iterate refs + for _, ref := range refs { + // ref type + var refType string + if strings.HasPrefix(ref.Name().String(), "refs/heads") { + refType = GitRefTypeBranch + } else if strings.HasPrefix(ref.Name().String(), "refs/tags") { + refType = GitRefTypeTag + } else { + continue + } + + // add to branches + gitRefs = append(gitRefs, GitRef{ + Type: refType, + Name: ref.Name().Short(), + FullName: ref.Name().String(), + Hash: ref.Hash().String(), + }) + } + + // logs without tags + logs, err := c.GetLogs() + if err != nil { + return nil, err + } + + // logs map + logsMap := map[string]GitLog{} + for _, l := range logs { + logsMap[l.Hash] = l + } + + // iterate git refs + for i, gitRef := range gitRefs { + l, ok := logsMap[gitRef.Hash] + if !ok { + continue + } + gitRefs[i].Timestamp = l.Timestamp + } + + // sort git refs + sort.Slice(gitRefs, func(i, j int) bool { + return gitRefs[i].Timestamp.Unix() > gitRefs[j].Timestamp.Unix() + }) + + return gitRefs, nil +} + +func (c *GitClient) GetTags() (tags []GitRef, err error) { + iter, err := c.r.Tags() + if err != nil { + return nil, trace.TraceError(err) + } + + _ = iter.ForEach(func(r *plumbing.Reference) error { + tags = append(tags, GitRef{ + Type: GitRefTypeTag, + Name: r.Name().Short(), + Hash: r.Hash().String(), + }) + return nil + }) + + return tags, nil +} + +func (c *GitClient) GetStatus() (statusList []GitFileStatus, err error) { + // worktree + wt, err := c.r.Worktree() + if err != nil { + return nil, trace.TraceError(err) + } + + // status + status, err := wt.Status() + if err != nil { + log.Warnf("failed to get worktree status: %v", err) + } + + // file status list + var list []GitFileStatus + for filePath, fileStatus := range status { + // file name + fileName := path.Base(filePath) + + // file status + s := GitFileStatus{ + Path: filePath, + Name: fileName, + IsDir: false, + Staging: c.getStatusString(fileStatus.Staging), + Worktree: c.getStatusString(fileStatus.Worktree), + Extra: fileStatus.Extra, + } + + // add to list + list = append(list, s) + } + + // sort list ascending + sort.Slice(list, func(i, j int) bool { + return list[i].Path < list[j].Path + }) + + return list, nil +} + +func (c *GitClient) Add(filePath string) (err error) { + // worktree + wt, err := c.r.Worktree() + if err != nil { + return trace.TraceError(err) + } + + if _, err := wt.Add(filePath); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) GetRemote(name string) (r *git.Remote, err error) { + return c.r.Remote(name) +} + +func (c *GitClient) CreateRemote(cfg *config.RemoteConfig) (r *git.Remote, err error) { + return c.r.CreateRemote(cfg) +} + +func (c *GitClient) DeleteRemote(name string) (err error) { + return c.r.DeleteRemote(name) +} + +func (c *GitClient) IsRemoteChanged() (ok bool, err error) { + return c.isRemoteChanged() +} + +func (c *GitClient) initMem() (err error) { + // validate options + if !c.isMem || c.path == "" { + return trace.TraceError(ErrInvalidOptions) + } + + // get storage and worktree + storage, wt := c.getMemStorageAndMemFs(c.path) + + // attempt to init + c.r, err = git.Init(storage, wt) + if err != nil { + if err == git.ErrRepositoryAlreadyExists { + // if already exists, attempt to open + c.r, err = git.Open(storage, wt) + if err != nil { + return trace.TraceError(err) + } + } else { + return trace.TraceError(err) + } + } + + return nil +} + +func (c *GitClient) initFs() (err error) { + // validate options + if c.path == "" { + return trace.TraceError(ErrInvalidOptions) + } + + // create directory if not exists + _, err = os.Stat(c.path) + if err != nil { + if err := os.MkdirAll(c.path, os.ModePerm); err != nil { + return trace.TraceError(err) + } + err = nil + } + + // try to open repo + c.r, err = git.PlainOpen(c.path) + if err == git.ErrRepositoryNotExists { + // repo not exists, init + c.r, err = git.PlainInit(c.path, false) + if err != nil { + return trace.TraceError(err) + } + } else if err != nil { + // error + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) clone() (err error) { + // validate + if c.remoteUrl == "" { + return trace.TraceError(ErrUnableToCloneWithEmptyRemoteUrl) + } + + // auth + auth, err := c.getGitAuth() + if err != nil { + return err + } + + // options + o := &git.CloneOptions{ + URL: c.remoteUrl, + Auth: auth, + } + + // clone + if _, err := git.PlainClone(c.path, false, o); err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (c *GitClient) getInitType() (res GitInitType) { + if c.isMem { + return GitInitTypeMem + } else { + return GitInitTypeFs + } +} + +func (c *GitClient) createRemote(remoteName string, url string) (err error) { + _, err = c.r.CreateRemote(&config.RemoteConfig{ + Name: remoteName, + URLs: []string{url}, + }) + if err != nil { + return trace.TraceError(err) + } + return +} + +func (c *GitClient) getMemStorageAndMemFs(key string) (storage *memory.Storage, fs billy.Filesystem) { + // storage + storageItem, ok := GitMemStorages.Load(key) + if !ok { + storage = memory.NewStorage() + GitMemStorages.Store(key, storage) + } else { + switch storageItem.(type) { + case *memory.Storage: + storage = storageItem.(*memory.Storage) + default: + storage = memory.NewStorage() + GitMemStorages.Store(key, storage) + } + } + + // file system + fsItem, ok := GitMemFileSystem.Load(key) + if !ok { + fs = memfs.New() + GitMemFileSystem.Store(key, fs) + } else { + switch fsItem.(type) { + case billy.Filesystem: + fs = fsItem.(billy.Filesystem) + default: + fs = memfs.New() + GitMemFileSystem.Store(key, fs) + } + } + + return storage, fs +} + +func (c *GitClient) getGitAuth() (auth transport.AuthMethod, err error) { + switch c.authType { + case GitAuthTypeNone: + return nil, nil + case GitAuthTypeHTTP: + if c.username == "" && c.password == "" { + return nil, nil + } + auth = &http.BasicAuth{ + Username: c.username, + Password: c.password, + } + return auth, nil + case GitAuthTypeSSH: + var privateKeyData []byte + if c.privateKey != "" { + // private key content + privateKeyData = []byte(c.privateKey) + } else if c.privateKeyPath != "" { + // read from private key file + privateKeyData, err = ioutil.ReadFile(c.privateKeyPath) + if err != nil { + return nil, trace.TraceError(err) + } + } else { + // no private key + return nil, nil + } + signer, err := ssh.ParsePrivateKey(privateKeyData) + if err != nil { + return nil, trace.TraceError(err) + } + auth = &gitssh.PublicKeys{ + User: c.username, + Signer: signer, + HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }, + } + return auth, nil + default: + return nil, trace.TraceError(ErrInvalidAuthType) + } +} + +func (c *GitClient) getHeadRef() (ref string, err error) { + wt, err := c.r.Worktree() + if err != nil { + return "", trace.TraceError(err) + } + fh, err := wt.Filesystem.Open(path.Join(".git", "HEAD")) + if err != nil { + return "", trace.TraceError(err) + } + data, err := ioutil.ReadAll(fh) + if err != nil { + return "", trace.TraceError(err) + } + m := headRefRegexp.FindStringSubmatch(string(data)) + if len(m) < 2 { + return "", trace.TraceError(ErrInvalidHeadRef) + } + return m[1], nil +} + +func (c *GitClient) getStatusString(statusCode git.StatusCode) (code string) { + return string(statusCode) + //switch statusCode { + //} + //Unmodified StatusCode = ' ' + //Untracked StatusCode = '?' + //Modified StatusCode = 'M' + //Added StatusCode = 'A' + //Deleted StatusCode = 'D' + //Renamed StatusCode = 'R' + //Copied StatusCode = 'C' + //UpdatedButUnmerged StatusCode = 'U' +} + +func (c *GitClient) getDirPaths(filePath string) (paths []string) { + pathItems := strings.Split(filePath, "/") + + var items []string + for i, pathItem := range pathItems { + if i == len(pathItems)-1 { + continue + } + items = append(items, pathItem) + dirPath := strings.Join(items, "/") + paths = append(paths, dirPath) + } + + return paths +} + +func (c *GitClient) createBranch(branch, remote string, ref *plumbing.Reference) (err error) { + // create a new branch if it does not exist + cfg := config.Branch{ + Name: branch, + Remote: remote, + } + if err := c.r.CreateBranch(&cfg); err != nil { + return err + } + // if ref is nil + if ref == nil { + // try to set to remote ref of branch first + ref, err = c.getBranchHashRef(branch, remote) + + // if no matched remote branch, set to HEAD + if err == ErrNoMatchedRemoteBranch { + ref, err = c.r.Head() + if err != nil { + return trace.TraceError(err) + } + } + + // error + if err != nil { + return trace.TraceError(err) + } + } + + // branch reference name + branchRefName := plumbing.NewBranchReferenceName(branch) + + // branch reference + branchRef := plumbing.NewHashReference(branchRefName, ref.Hash()) + + // set HEAD to branch reference + if err := c.r.Storer.SetReference(branchRef); err != nil { + return err + } + + return nil +} + +func (c *GitClient) getBranchHashRef(branch, remote string) (hashRef *plumbing.Reference, err error) { + refs, err := c.GetRemoteRefs(remote) + if err != nil { + return nil, err + } + var branchRef *GitRef + for _, r := range refs { + if r.Name == branch { + branchRef = &r + break + } + } + if branchRef == nil { + return nil, ErrNoMatchedRemoteBranch + } + branchHashRef := plumbing.NewHashReference(plumbing.NewBranchReferenceName(branch), plumbing.NewHash(branchRef.Hash)) + return branchHashRef, nil +} + +func (c *GitClient) isRemoteChanged() (ok bool, err error) { + b, err := c.GetCurrentBranchRef() + if err != nil { + return false, err + } + refs, err := c.GetRemoteRefs(GitRemoteNameOrigin) + if err != nil { + return false, err + } + for _, r := range refs { + if r.Name == b.Name { + return r.Hash != b.Hash, nil + } + } + return false, nil +} + +func NewGitClient(opts ...GitOption) (c *GitClient, err error) { + // client + c = &GitClient{ + isMem: false, + authType: GitAuthTypeNone, + username: "git", + privateKeyPath: getDefaultPublicKeyPath(), + } + + // apply options + for _, opt := range opts { + opt(c) + } + + // init + if err := c.Init(); err != nil { + return c, err + } + + return +} diff --git a/vcs/git_options.go b/vcs/git_options.go new file mode 100644 index 000000000..a990c1e1f --- /dev/null +++ b/vcs/git_options.go @@ -0,0 +1,250 @@ +package vcs + +import ( + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + "strings" +) + +type GitOption func(c *GitClient) + +func WithPath(path string) GitOption { + return func(c *GitClient) { + c.path = path + } +} + +func WithRemoteUrl(url string) GitOption { + return func(c *GitClient) { + c.remoteUrl = url + } +} + +func WithIsMem() GitOption { + return func(c *GitClient) { + c.isMem = true + } +} + +func WithAuthType(authType GitAuthType) GitOption { + return func(c *GitClient) { + c.authType = authType + } +} + +func WithUsername(username string) GitOption { + return func(c *GitClient) { + c.username = username + } +} + +func WithPassword(password string) GitOption { + return func(c *GitClient) { + c.password = password + } +} + +func WithPrivateKey(key string) GitOption { + return func(c *GitClient) { + c.privateKey = key + } +} + +func WithDefaultBranch(branch string) GitOption { + return func(c *GitClient) { + c.defaultBranch = branch + } +} + +func WithPrivateKeyPath(path string) GitOption { + return func(c *GitClient) { + c.privateKeyPath = path + } +} + +type GitCloneOption func(o *git.CloneOptions) + +func WithURL(url string) GitCloneOption { + return func(o *git.CloneOptions) { + o.URL = url + } +} + +func WithAuthClone(auth transport.AuthMethod) GitCloneOption { + return func(o *git.CloneOptions) { + o.Auth = auth + } +} + +func WithRemoteName(name string) GitCloneOption { + return func(o *git.CloneOptions) { + o.RemoteName = name + } +} + +func WithSingleBranch(singleBranch bool) GitCloneOption { + return func(o *git.CloneOptions) { + o.SingleBranch = singleBranch + } +} + +func WithNoCheckout(noCheckout bool) GitCloneOption { + return func(o *git.CloneOptions) { + o.NoCheckout = noCheckout + } +} + +func WithDepthClone(depth int) GitCloneOption { + return func(o *git.CloneOptions) { + o.Depth = depth + } +} + +func WithRecurseSubmodules(recurseSubmodules git.SubmoduleRescursivity) GitCloneOption { + return func(o *git.CloneOptions) { + o.RecurseSubmodules = recurseSubmodules + } +} + +func WithTags(tags git.TagMode) GitCloneOption { + return func(o *git.CloneOptions) { + o.Tags = tags + } +} + +type GitCheckoutOption func(o *git.CheckoutOptions) + +func WithBranch(branch string) GitCheckoutOption { + return func(o *git.CheckoutOptions) { + if strings.HasPrefix(branch, "refs/heads") { + o.Branch = plumbing.ReferenceName(branch) + } else { + o.Branch = plumbing.NewBranchReferenceName(branch) + } + } +} + +func WithHash(hash string) GitCheckoutOption { + return func(o *git.CheckoutOptions) { + h := plumbing.NewHash(hash) + if h.IsZero() { + return + } + o.Hash = h + } +} + +type GitCommitOption func(o *git.CommitOptions) + +func WithAll(all bool) GitCommitOption { + return func(o *git.CommitOptions) { + o.All = all + } +} + +func WithAuthor(author *object.Signature) GitCommitOption { + return func(o *git.CommitOptions) { + o.Author = author + } +} + +func WithCommitter(committer *object.Signature) GitCommitOption { + return func(o *git.CommitOptions) { + o.Committer = committer + } +} + +func WithParents(parents []plumbing.Hash) GitCommitOption { + return func(o *git.CommitOptions) { + o.Parents = parents + } +} + +type GitPullOption func(o *git.PullOptions) + +func WithRemoteNamePull(name string) GitPullOption { + return func(o *git.PullOptions) { + o.RemoteName = name + } +} + +func WithBranchNamePull(branch string) GitPullOption { + return func(o *git.PullOptions) { + o.ReferenceName = plumbing.NewBranchReferenceName(branch) + } +} + +func WithDepthPull(depth int) GitPullOption { + return func(o *git.PullOptions) { + o.Depth = depth + } +} + +func WithAuthPull(auth transport.AuthMethod) GitPullOption { + return func(o *git.PullOptions) { + if auth != nil { + o.Auth = auth + } + } +} + +func WithRecurseSubmodulesPull(recurseSubmodules git.SubmoduleRescursivity) GitPullOption { + return func(o *git.PullOptions) { + o.RecurseSubmodules = recurseSubmodules + } +} + +func WithForcePull(force bool) GitPullOption { + return func(o *git.PullOptions) { + o.Force = force + } +} + +type GitPushOption func(o *git.PushOptions) + +func WithRemoteNamePush(name string) GitPushOption { + return func(o *git.PushOptions) { + o.RemoteName = name + } +} + +func WithRefSpecs(specs []config.RefSpec) GitPushOption { + return func(o *git.PushOptions) { + o.RefSpecs = specs + } +} + +func WithAuthPush(auth transport.AuthMethod) GitPushOption { + return func(o *git.PushOptions) { + o.Auth = auth + } +} + +func WithPrune(prune bool) GitPushOption { + return func(o *git.PushOptions) { + o.Prune = prune + } +} + +func WithForcePush(force bool) GitPushOption { + return func(o *git.PushOptions) { + o.Force = force + } +} + +type GitResetOption func(o *git.ResetOptions) + +func WithCommit(commit plumbing.Hash) GitResetOption { + return func(o *git.ResetOptions) { + o.Commit = commit + } +} + +func WithMode(mode git.ResetMode) GitResetOption { + return func(o *git.ResetOptions) { + o.Mode = mode + } +} diff --git a/vcs/git_store.go b/vcs/git_store.go new file mode 100644 index 000000000..5795c3903 --- /dev/null +++ b/vcs/git_store.go @@ -0,0 +1,6 @@ +package vcs + +import "sync" + +var GitMemStorages = sync.Map{} +var GitMemFileSystem = sync.Map{} diff --git a/vcs/git_utils.go b/vcs/git_utils.go new file mode 100644 index 000000000..cc674e321 --- /dev/null +++ b/vcs/git_utils.go @@ -0,0 +1,67 @@ +package vcs + +import ( + "github.com/go-git/go-git/v5" + "os" + "path" +) + +func CreateBareGitRepo(path string) (err error) { + // validate options + if path == "" { + return ErrInvalidRepoPath + } + + // validate if exists + if IsGitRepoExists(path) { + return ErrRepoAlreadyExists + } + + // create directory if not exists + _, err = os.Stat(path) + if err != nil { + if err := os.MkdirAll(path, os.FileMode(0766)); err != nil { + return err + } + err = nil + } + + // init + if _, err := git.PlainInit(path, true); err != nil { + return err + } + + return nil +} + +func CloneGitRepo(path, url string, opts ...GitCloneOption) (c *GitClient, err error) { + // url + opts = append(opts, WithURL(url)) + + // apply options + o := &git.CloneOptions{} + for _, opt := range opts { + opt(o) + } + + // clone + if _, err := git.PlainClone(path, false, o); err != nil { + return nil, err + } + + return NewGitClient(WithPath(path)) +} + +func IsGitRepoExists(repoPath string) (ok bool) { + dotGitPath := path.Join(repoPath, git.GitDirName) + if _, err := os.Stat(dotGitPath); err == nil { + return true + } + + headPath := path.Join(repoPath, "HEAD") + if _, err := os.Stat(headPath); err == nil { + return true + } + + return false +} diff --git a/vcs/go.mod b/vcs/go.mod new file mode 100644 index 000000000..56d2c2513 --- /dev/null +++ b/vcs/go.mod @@ -0,0 +1,38 @@ +module github.com/crawlab-team/crawlab/vcs + +go 1.22 + +require ( + github.com/apex/log v1.9.0 + github.com/go-git/go-billy/v5 v5.5.0 + github.com/go-git/go-git/v5 v5.12.0 + github.com/stretchr/testify v1.9.0 + golang.org/x/crypto v0.21.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/tools v0.14.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/vcs/go.sum b/vcs/go.sum new file mode 100644 index 000000000..07feb5ec3 --- /dev/null +++ b/vcs/go.sum @@ -0,0 +1,164 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/vcs/interface.go b/vcs/interface.go new file mode 100644 index 000000000..a86fbf98c --- /dev/null +++ b/vcs/interface.go @@ -0,0 +1,12 @@ +package vcs + +type Client interface { + Init() (err error) + Dispose() (err error) + Clone(opts ...GitCloneOption) (err error) + Checkout(opts ...GitCheckoutOption) (err error) + Commit(msg string, opts ...GitCommitOption) (err error) + Pull(opts ...GitPullOption) (err error) + Push(opts ...GitPushOption) (err error) + Reset(opts ...GitResetOption) (err error) +} diff --git a/vcs/test/base.go b/vcs/test/base.go new file mode 100644 index 000000000..22a812470 --- /dev/null +++ b/vcs/test/base.go @@ -0,0 +1,138 @@ +package test + +import ( + vcs "github.com/crawlab-team/crawlab/vcs" + "io/ioutil" + "os" + "path" + "sync" + "testing" + "time" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + RemoteRepoPath string + LocalRepoPath string + LocalRepo *vcs.GitClient + FsRepoPath string + MemRepoPath string + AuthRepoPath string + AuthRepoPath1 string + AuthRepoPath2 string + AuthRepoPath3 string + TestFileName string + TestFileContent string + TestBranchName string + TestCommitMessage string + InitialCommitMessage string + InitialReadmeFileName string + InitialReadmeFileContent string +} + +func (t *Test) Setup(t2 *testing.T) { + var err error + t2.Cleanup(t.Cleanup) + + // remote repo + if err := vcs.CreateBareGitRepo(t.RemoteRepoPath); err != nil { + panic(err) + } + + // local repo (fs) + t.LocalRepo, err = vcs.NewGitClient( + vcs.WithPath(t.LocalRepoPath), + vcs.WithRemoteUrl(t.RemoteRepoPath), + ) + if err != nil { + panic(err) + } + + // initial commit + filePath := path.Join(t.LocalRepoPath, t.InitialReadmeFileContent) + if err := ioutil.WriteFile(filePath, []byte(t.InitialReadmeFileContent), os.FileMode(0766)); err != nil { + panic(err) + } + if err := t.LocalRepo.CommitAll(t.InitialCommitMessage); err != nil { + panic(err) + } +} + +func (t *Test) Cleanup() { + if err := T.LocalRepo.Dispose(); err != nil { + panic(err) + } + if err := os.RemoveAll(T.RemoteRepoPath); err != nil { + panic(err) + } + + vcs.GitMemStorages = sync.Map{} + vcs.GitMemFileSystem = sync.Map{} + + // wait to avoid caching + time.Sleep(500 * time.Millisecond) +} + +func NewTest() (t *Test, err error) { + t = &Test{} + + // clear tmp directory + _ = os.RemoveAll("./tmp") + _ = os.MkdirAll("./tmp", os.FileMode(0766)) + + // remote repo path + t.RemoteRepoPath = "./tmp/test_remote_repo" + + // local repo path + t.LocalRepoPath = "./tmp/test_local_repo" + + // fs repo path + t.FsRepoPath = "./tmp/test_fs_repo" + + // mem repo path + t.MemRepoPath = "./tmp/test_mem_repo" + + // auth repo path + t.AuthRepoPath = "./tmp/test_auth_repo" + + // auth repo path 1 + t.AuthRepoPath1 = "./tmp/test_auth_repo1" + + // auth repo path 2 + t.AuthRepoPath2 = "./tmp/test_auth_repo2" + + // auth repo path 3 + t.AuthRepoPath3 = "./tmp/test_auth_repo3" + + // test file name + t.TestFileName = "test_file.txt" + + // test file content + t.TestFileContent = "it works" + + // test branch name + t.TestBranchName = "develop" + + // test commit message + t.InitialCommitMessage = "test commit" + + // initial commit message + t.InitialCommitMessage = "initial commit" + + // initial readme file name + t.InitialReadmeFileName = "README.md" + + // initial readme file content + t.InitialReadmeFileContent = "README" + + return t, nil +} diff --git a/vcs/test/credential.go b/vcs/test/credential.go new file mode 100644 index 000000000..7af7b537f --- /dev/null +++ b/vcs/test/credential.go @@ -0,0 +1,13 @@ +package test + +type Credential struct { + Username string `json:"username"` + Password string `json:"password"` + TestRepoHttpUrl string `json:"test_repo_http_url"` + TestRepoMultiBranchUrl string `json:"test_repo_multi_branch_url"` + SshUsername string `json:"ssh_username"` + SshPassword string `json:"ssh_password"` + TestRepoSshUrl string `json:"test_repo_ssh_url"` + PrivateKey string `json:"private_key"` + PrivateKeyPath string `json:"private_key_path"` +} diff --git a/vcs/test/credentials.example.json b/vcs/test/credentials.example.json new file mode 100644 index 000000000..b031c55a7 --- /dev/null +++ b/vcs/test/credentials.example.json @@ -0,0 +1,10 @@ +{ + "username": "changeit", + "password": "changeit", + "test_repo_http_url": "https://gitee.com/tikazyq/test-repo", + "ssh_username": "git", + "ssh_password": "", + "test_repo_ssh_url": "git@gitee.com:tikazyq/test-repo.git", + "private_key": "changeit", + "private_key_path": "/root/.ssh/id_rsa" +} \ No newline at end of file diff --git a/vcs/test/git_test.go b/vcs/test/git_test.go new file mode 100644 index 000000000..e97c87397 --- /dev/null +++ b/vcs/test/git_test.go @@ -0,0 +1,510 @@ +package test + +import ( + "encoding/json" + "fmt" + "github.com/crawlab-team/crawlab/vcs" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "path" + "testing" + "time" +) + +func TestNewGitClient_Existing(t *testing.T) { + var err error + T.Setup(t) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.LocalRepo.GetPath()), + ) + require.Nil(t, err) + require.NotEmpty(t, c.GetRepository()) +} + +func TestNewGitClient_Fs(t *testing.T) { + var err error + T.Setup(t) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.FsRepoPath), + vcs.WithRemoteUrl(T.RemoteRepoPath), + ) + require.Nil(t, err) + require.NotEmpty(t, c.GetRepository()) + require.Equal(t, T.RemoteRepoPath, c.GetRemoteUrl()) +} + +func TestNewGitClient_Mem(t *testing.T) { + var err error + T.Setup(t) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.MemRepoPath), + vcs.WithRemoteUrl(T.RemoteRepoPath), + vcs.WithIsMem(), + ) + require.Nil(t, err) + require.NotEmpty(t, c.GetRepository()) +} + +func TestGitClient_CommitAllAndCheckoutBranch(t *testing.T) { + var err error + T.Setup(t) + + // commit + filePath := path.Join(T.LocalRepoPath, T.TestFileName) + err = ioutil.WriteFile(filePath, []byte(T.TestFileContent), os.FileMode(0766)) + require.Nil(t, err) + err = T.LocalRepo.CommitAll(T.TestCommitMessage) + require.Nil(t, err) + + // checkout branch + err = T.LocalRepo.CheckoutBranch(T.TestBranchName) + require.Nil(t, err) + + // validate + branch, err := T.LocalRepo.GetCurrentBranch() + require.Nil(t, err) + require.Equal(t, T.TestBranchName, branch) + + // dispose + err = T.LocalRepo.Dispose() + require.Nil(t, err) +} + +func TestGitClient_Push(t *testing.T) { + var err error + T.Setup(t) + + // commit + filePath := path.Join(T.LocalRepoPath, T.TestFileName) + err = ioutil.WriteFile(filePath, []byte(T.TestFileContent), os.FileMode(0766)) + require.Nil(t, err) + err = T.LocalRepo.CommitAll(T.TestCommitMessage) + require.Nil(t, err) + + // push + err = T.LocalRepo.Push() + require.Nil(t, err) +} + +func TestGitClient_Reset(t *testing.T) { + var err error + T.Setup(t) + + // file + filePath := path.Join(T.LocalRepoPath, T.TestFileName) + err = ioutil.WriteFile(filePath, []byte(T.TestFileContent), os.FileMode(0766)) + require.Nil(t, err) + + // reset + err = T.LocalRepo.Reset(vcs.WithMode(git.HardReset)) // git reset --hard + require.Nil(t, err) + _, err = os.Stat(filePath) + require.IsType(t, &os.PathError{}, err) +} + +func TestGitClient_GetLogs(t *testing.T) { + var err error + T.Setup(t) + + // file + filePath := path.Join(T.LocalRepoPath, T.TestFileName) + err = ioutil.WriteFile(filePath, []byte(T.TestFileContent), os.FileMode(0766)) + require.Nil(t, err) + err = T.LocalRepo.CommitAll(T.TestCommitMessage) + require.Nil(t, err) + + // get logs + logs, err := T.LocalRepo.GetLogs() + require.Nil(t, err) + require.Greater(t, len(logs), 0) + require.Equal(t, T.TestCommitMessage, logs[0].Msg) +} + +func TestGitClient_InitWithHttpAuth(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + + // create new git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath), + vcs.WithRemoteUrl(cred.TestRepoHttpUrl), + vcs.WithAuthType(vcs.GitAuthTypeHTTP), + vcs.WithUsername(cred.Username), + vcs.WithPassword(cred.Password), + ) + require.Nil(t, err) + require.Equal(t, cred.TestRepoHttpUrl, c.GetRemoteUrl()) + require.Equal(t, vcs.GitAuthTypeHTTP, c.GetAuthType()) + require.Equal(t, cred.Username, c.GetUsername()) + + // pull + err = c.Pull() + require.Nil(t, err) + + // validate + files, err := ioutil.ReadDir(T.AuthRepoPath) + require.Greater(t, len(files), 0) + data, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "README.md")) + require.Nil(t, err) + + // dispose + err = c.Dispose() + require.Nil(t, err) +} + +func TestGitClient_MoveBranch(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + + // create new git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath), + vcs.WithRemoteUrl(cred.TestRepoHttpUrl), + vcs.WithAuthType(vcs.GitAuthTypeHTTP), + vcs.WithUsername(cred.Username), + vcs.WithPassword(cred.Password), + ) + + // pull + err = c.Pull(vcs.WithBranchNamePull(vcs.GitBranchNameMain)) + require.Nil(t, err) + + // move branch + err = c.MoveBranch(vcs.GitBranchNameMaster, vcs.GitBranchNameMain) + require.Nil(t, err) + + // validate + var branchNames []string + branches, err := c.GetBranches() + require.Nil(t, err) + for _, b := range branches { + branchNames = append(branchNames, b.Name) + } + require.Contains(t, branchNames, vcs.GitBranchNameMain) + require.NotContains(t, branchNames, vcs.GitBranchNameMaster) +} + +func TestGitClient_PullWithHttpAuth(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + + // create new git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath), + vcs.WithRemoteUrl(cred.TestRepoHttpUrl), + vcs.WithAuthType(vcs.GitAuthTypeHTTP), + vcs.WithUsername(cred.Username), + vcs.WithPassword(cred.Password), + ) + require.Nil(t, err) + + // create remote + r, err := c.CreateRemote(&config.RemoteConfig{ + Name: vcs.GitRemoteNameUpstream, + URLs: []string{cred.TestRepoHttpUrl}, + }) + require.Nil(t, err) + require.NotNil(t, r) + + // pull + err = c.Pull( + vcs.WithRemoteNamePull(vcs.GitRemoteNameUpstream), + vcs.WithBranchNamePull(vcs.GitBranchNameMain), + ) + require.Nil(t, err) + + // validate + files, err := ioutil.ReadDir(T.AuthRepoPath) + require.Greater(t, len(files), 0) + data, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "README.md")) + require.Nil(t, err) + + // dispose + err = c.Dispose() + require.Nil(t, err) +} + +func TestGitClient_CheckoutRemoteBranchWithHttpAuth(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + + // create new git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath), + vcs.WithRemoteUrl(cred.TestRepoMultiBranchUrl), + vcs.WithAuthType(vcs.GitAuthTypeHTTP), + vcs.WithUsername(cred.Username), + vcs.WithPassword(cred.Password), + ) + require.Nil(t, err) + + // pull + err = c.Pull( + vcs.WithRemoteNamePull(vcs.GitRemoteNameOrigin), + vcs.WithBranchNamePull(vcs.GitBranchNameMain), + ) + require.Nil(t, err) + + // validate + _, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "MAIN")) + require.Nil(t, err) + + // checkout remote branch + err = c.CheckoutBranchWithRemote(vcs.GitBranchNameRelease, vcs.GitRemoteNameOrigin, nil) + require.Nil(t, err) + + // validate + _, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "RELEASE")) + require.Nil(t, err) + + // checkout remote branch + err = c.CheckoutBranchWithRemote(vcs.GitBranchNameDevelop, vcs.GitRemoteNameOrigin, nil) + require.Nil(t, err) + + // validate + _, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "DEVELOP")) + require.Nil(t, err) + + // dispose + err = c.Dispose() + require.Nil(t, err) +} + +func TestGitClient_InitWithSshAuth_PrivateKey(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + fmt.Println(cred.SshUsername) + fmt.Println(cred.SshPassword) + fmt.Println(cred.TestRepoSshUrl) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath), + vcs.WithRemoteUrl(cred.TestRepoSshUrl), + vcs.WithAuthType(vcs.GitAuthTypeSSH), + vcs.WithUsername(cred.SshUsername), + vcs.WithPassword(cred.SshPassword), + vcs.WithPrivateKey(cred.PrivateKey), + ) + require.Nil(t, err) + require.Equal(t, cred.TestRepoSshUrl, c.GetRemoteUrl()) + require.Equal(t, vcs.GitAuthTypeSSH, c.GetAuthType()) + require.Equal(t, cred.SshUsername, c.GetUsername()) + fmt.Println(c.GetAuthType()) + + // pull + err = c.Pull() + require.Nil(t, err) + + // validate + files, err := ioutil.ReadDir(T.AuthRepoPath) + require.Greater(t, len(files), 0) + data, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "README.md")) + require.Nil(t, err) + + // dispose + err = c.Dispose() + require.Nil(t, err) +} + +func TestGitClient_InitWithSshAuth_PrivateKeyPath(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath), + vcs.WithRemoteUrl(cred.TestRepoSshUrl), + vcs.WithAuthType(vcs.GitAuthTypeSSH), + vcs.WithUsername(cred.SshUsername), + vcs.WithPassword(cred.SshPassword), + vcs.WithPrivateKeyPath(cred.PrivateKeyPath), + ) + require.Nil(t, err) + require.Equal(t, cred.TestRepoSshUrl, c.GetRemoteUrl()) + require.Equal(t, vcs.GitAuthTypeSSH, c.GetAuthType()) + require.Equal(t, cred.SshUsername, c.GetUsername()) + require.Equal(t, cred.PrivateKeyPath, c.GetPrivateKeyPath()) + + // pull + err = c.Pull() + require.Nil(t, err) + + // validate + files, err := ioutil.ReadDir(T.AuthRepoPath) + require.Greater(t, len(files), 0) + data, err = ioutil.ReadFile(path.Join(T.AuthRepoPath, "README.md")) + require.Nil(t, err) +} + +func TestGitClient_Dispose_Fs(t *testing.T) { + var err error + T.Setup(t) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.FsRepoPath), + vcs.WithRemoteUrl(T.RemoteRepoPath), + ) + require.Nil(t, err) + + // path exists + require.DirExists(t, T.FsRepoPath) + + // dispose + err = c.Dispose() + require.Nil(t, err) + + // validate + _, err = os.Stat(T.FsRepoPath) + require.IsType(t, &os.PathError{}, err) +} + +func TestGitClient_Dispose_Mem(t *testing.T) { + var err error + T.Setup(t) + + // git client + c, err := vcs.NewGitClient( + vcs.WithPath(T.MemRepoPath), + vcs.WithRemoteUrl(T.RemoteRepoPath), + vcs.WithIsMem(), + ) + require.Nil(t, err) + + // mem map exists + stItem, ok := vcs.GitMemStorages.Load(T.MemRepoPath) + require.True(t, ok) + require.IsType(t, &memory.Storage{}, stItem) + fsItem, ok := vcs.GitMemFileSystem.Load(T.MemRepoPath) + require.True(t, ok) + require.IsType(t, memfs.New(), fsItem) + + // dispose + err = c.Dispose() + require.Nil(t, err) + + // validate + _, ok = vcs.GitMemStorages.Load("./tmp/test_repo") + require.False(t, ok) + _, ok = vcs.GitMemFileSystem.Load("./tmp/test_repo") + require.False(t, ok) +} + +func TestGitClient_IsRemoteChanged(t *testing.T) { + var err error + T.Setup(t) + + // get credentials + var cred Credential + data, err := ioutil.ReadFile("credentials.json") + require.Nil(t, err) + err = json.Unmarshal(data, &cred) + require.Nil(t, err) + + // git client + c1, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath1), + vcs.WithRemoteUrl(cred.TestRepoSshUrl), + vcs.WithAuthType(vcs.GitAuthTypeSSH), + vcs.WithUsername(cred.SshUsername), + vcs.WithPassword(cred.SshPassword), + vcs.WithPrivateKeyPath(cred.PrivateKeyPath), + ) + require.Nil(t, err) + err = c1.Pull() + require.Nil(t, err) + err = c1.CheckoutBranch("main") + require.Nil(t, err) + + // git client (for validation) + c2, err := vcs.NewGitClient( + vcs.WithPath(T.AuthRepoPath2), + vcs.WithRemoteUrl(cred.TestRepoSshUrl), + vcs.WithAuthType(vcs.GitAuthTypeSSH), + vcs.WithUsername(cred.SshUsername), + vcs.WithPassword(cred.SshPassword), + vcs.WithPrivateKeyPath(cred.PrivateKeyPath), + ) + require.Nil(t, err) + err = c2.Pull() + require.Nil(t, err) + err = c2.CheckoutBranch("main") + require.Nil(t, err) + + // commit and push + testFileName := fmt.Sprintf("test-%d.txt", time.Now().Unix()) + filePath := path.Join(c1.GetPath(), testFileName) + err = ioutil.WriteFile(filePath, []byte(T.TestFileContent), os.FileMode(0766)) + require.Nil(t, err) + err = c1.Add(testFileName) + require.Nil(t, err) + err = c1.CommitAll(fmt.Sprintf("added %s", testFileName)) + require.Nil(t, err) + err = c1.Push() + require.Nil(t, err) + + // validate + ok, err := c2.IsRemoteChanged() + require.Nil(t, err) + require.True(t, ok) + + // pull and validate + err = c2.Pull() + require.Nil(t, err) + ok, err = c2.IsRemoteChanged() + require.Nil(t, err) + require.False(t, ok) +} diff --git a/vcs/utils.go b/vcs/utils.go new file mode 100644 index 000000000..b486bf1e5 --- /dev/null +++ b/vcs/utils.go @@ -0,0 +1,15 @@ +package vcs + +import ( + "os/user" + "path/filepath" +) + +func getDefaultPublicKeyPath() (path string) { + u, err := user.Current() + if err != nil { + return path + } + path = filepath.Join(u.HomeDir, ".ssh", "id_rsa") + return +}