From 27f2e421160a924e91f9400f8a06695eba7d534d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 16:51:09 +0800 Subject: [PATCH 001/106] fix: dependency issue --- backend/bin/update-deps.sh | 2 +- backend/go.mod | 94 ++++++----- backend/go.mod.dev | 4 +- backend/go.mod.local | 4 +- backend/go.sum | 318 ++++++++++++++++--------------------- backend/main.go | 2 +- core/go.mod | 15 +- core/go.sum | 6 + go.work | 1 + go.work.sum | 20 ++- 10 files changed, 223 insertions(+), 243 deletions(-) diff --git a/backend/bin/update-deps.sh b/backend/bin/update-deps.sh index a8ed28fd0..38d6eab94 100755 --- a/backend/bin/update-deps.sh +++ b/backend/bin/update-deps.sh @@ -1,4 +1,4 @@ #!/bin/bash -go get -u github.com/crawlab-team/crawlab-core@main +go get -u github.com/crawlab-team/crawlab/core@main go mod tidy diff --git a/backend/go.mod b/backend/go.mod index 861f29191..abd2ff574 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,34 +1,50 @@ module crawlab -go 1.18 +go 1.22 -require github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203 +replace ( + github.com/crawlab-team/crawlab/core => ../core + github.com/crawlab-team/crawlab/db => ../db + github.com/crawlab-team/crawlab/fs => ../fs + github.com/crawlab-team/crawlab/grpc => ../grpc + github.com/crawlab-team/crawlab/template-parser => ../template-parser + github.com/crawlab-team/crawlab/trace => ../trace + github.com/crawlab-team/crawlab/vcs => ../vcs +) + +require github.com/crawlab-team/crawlab/core v0.6.3-0.20231031044528-37e6d73eb203 + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/crawlab-team/crawlab/trace v0.1.0 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/sirupsen/logrus v1.9.0 // indirect +) require ( + cloud.google.com/go v0.99.0 // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/ReneKroon/ttlcache v1.7.0 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aokoli/goutils v1.0.1 // indirect github.com/apex/log v1.9.0 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.1.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/cloudflare/circl v1.3.3 // indirect - github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 // indirect - github.com/crawlab-team/crawlab/fs v0.6.3 // indirect - github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c // indirect - github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 // indirect - github.com/crawlab-team/crawlab/vcs v0.1.1 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/crawlab-team/crawlab/db v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614083850-ff2830bd7994 github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect - github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.7.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -37,20 +53,19 @@ require ( github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.4.1 // indirect - github.com/go-git/go-git/v5 v5.7.0 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/go-stack/stack v1.8.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect github.com/hashicorp/go-uuid v1.0.1 // indirect @@ -69,17 +84,14 @@ require ( github.com/jackc/pgx/v4 v4.15.0 // indirect github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect github.com/linxGnu/gumble v1.0.0 // indirect github.com/magiconair/properties v1.8.5 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/matcornic/hermes/v2 v2.1.0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect @@ -89,22 +101,20 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/olekukonko/tablewriter v0.0.1 // indirect - github.com/olivere/elastic/v7 v7.0.15 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/satori/go.uuid v1.2.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/segmentio/kafka-go v0.4.39 // indirect - github.com/sergi/go-diff v1.3.1 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/skeema/knownhosts v1.1.1 // indirect - github.com/spf13/afero v1.6.0 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/cobra v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -122,24 +132,24 @@ require ( github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.0.2 // indirect - github.com/xdg-go/stringprep v1.0.2 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect - go.mongodb.org/mongo-driver v1.8.0 // indirect + go.mongodb.org/mongo-driver v1.15.0 // indirect go.uber.org/dig v1.10.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect - golang.org/x/tools v0.10.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect - google.golang.org/grpc v1.42.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/backend/go.mod.dev b/backend/go.mod.dev index f45ffe255..89afabb12 100644 --- a/backend/go.mod.dev +++ b/backend/go.mod.dev @@ -3,7 +3,7 @@ module crawlab go 1.16 replace ( - github.com/crawlab-team/crawlab-core => ../../crawlab-core + github.com/crawlab-team/crawlab/core => ../../crawlab-core github.com/crawlab-team/crawlab-vcs => ../../crawlab-vcs github.com/crawlab-team/crawlab/fs => ../../crawlab-fs github.com/crawlab-team/crawlab/db => ../../crawlab-db @@ -11,7 +11,7 @@ replace ( require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 + github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200 github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.mod.local b/backend/go.mod.local index ff0ab04c5..fd8c268e7 100644 --- a/backend/go.mod.local +++ b/backend/go.mod.local @@ -3,7 +3,7 @@ module crawlab go 1.16 replace ( - github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core + github.com/crawlab-team/crawlab/core => /libs/crawlab-team/crawlab-core github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs github.com/crawlab-team/crawlab/fs => /libs/crawlab-team/crawlab-fs github.com/crawlab-team/crawlab/db => /libs/crawlab-team/crawlab-db @@ -11,7 +11,7 @@ replace ( require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 + github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200 github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.sum b/backend/go.sum index 4e7bd4b37..c6e582ba6 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -3,6 +3,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,6 +16,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -36,7 +38,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -47,6 +48,9 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -61,24 +65,28 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec h1:vV3RryLxt42+ZIVOFbYJCH1jsZNTNmj2NYru5zfx+4E= -github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= @@ -93,16 +101,13 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= @@ -123,8 +128,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -137,58 +143,28 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230708092022-c40c0fdc2c28 h1:x2mzygXuU6mjlFk8zr8C45VvYEJn/wF1AgrGj5wfW8k= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230708092022-c40c0fdc2c28/go.mod h1:rrcJUzh/PHR2WKBaahsR8baRstCdnEEFLx+pOA8IenU= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230726071808-25ed70356f13 h1:fP4OGh4q4hRLj9xOxdxHxJ3kMx8cxxcHMp/DxtJukJ4= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230726071808-25ed70356f13/go.mod h1:rrcJUzh/PHR2WKBaahsR8baRstCdnEEFLx+pOA8IenU= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230804030437-664646cf1665 h1:LAX9MN0NVIZHkyFDNH+3LvsD5AgoV7/FsbdDjVQr3OQ= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230804030437-664646cf1665/go.mod h1:rrcJUzh/PHR2WKBaahsR8baRstCdnEEFLx+pOA8IenU= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231021045242-07956209f653 h1:uOCkVHVqr2KGk/lv68LbBM3uQoJT6p4ywXInPitM3wo= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231021045242-07956209f653/go.mod h1:HNuAjSVVZpHhpyUP4k1F2YKxEiarZESFolSyx7YjgZ8= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203 h1:nyANfzoPgTSYJxuTye1uj44An8Cjou9QmcKRES7Gdwg= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203/go.mod h1:HNuAjSVVZpHhpyUP4k1F2YKxEiarZESFolSyx7YjgZ8= -github.com/crawlab-team/crawlab/db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= -github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab/fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73 h1:xIgfVPa3ZJWC72Y57oHS41n4jRtGZPn1YDEYBgMj2EU= -github.com/crawlab-team/crawlab/fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= -github.com/crawlab-team/crawlab/fs v0.6.3 h1:mS91sYu+tOPavjYvt4CZ8YwY5okEiwCAuyx/5RIbXJY= -github.com/crawlab-team/crawlab/fs v0.6.3/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= -github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c h1:jX0iax3WHwomWGQVWrCTy8a4zYDsKKyuspP3+04XCcU= -github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c/go.mod h1:Bq2Pm967EYWbjhP5Ghc4DV2LZgbOLMzLftJXDJYz/gs= -github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 h1:A/XSUuGgGMn+z+lFd2ye2ClgIKhDZYUerhOL5jePQhU= -github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I= -github.com/crawlab-team/crawlab/vcs v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= -github.com/crawlab-team/crawlab/vcs v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= -github.com/crawlab-team/crawlab/vcs v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= -github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -205,28 +181,32 @@ github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gavv/httpexpect/v2 v2.2.0 h1:0VwaEBmQaNFHX9x591A8Up+8shCwdF/nF0qlRd/nI48= +github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= +github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= -github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= -github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -240,30 +220,30 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -294,12 +274,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -314,8 +294,10 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -330,6 +312,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -339,27 +322,23 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -373,27 +352,22 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -405,7 +379,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E= +github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -464,13 +439,6 @@ github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -484,23 +452,22 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -510,25 +477,17 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -536,6 +495,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -548,7 +509,6 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -556,15 +516,13 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= @@ -576,27 +534,24 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= -github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -604,37 +559,34 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -643,7 +595,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -652,8 +605,8 @@ github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KR github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -666,33 +619,28 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= -github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= @@ -702,6 +650,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -712,14 +662,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -731,7 +680,6 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -740,7 +688,9 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= +github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= @@ -750,23 +700,28 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -776,15 +731,13 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.mongodb.org/mongo-driver v1.8.0 h1:R/P/JJzu8LJvJ1lDfph9GLNIKQxEtIHFfnUUUve35zY= -go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -828,17 +781,18 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -879,16 +833,14 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -920,6 +872,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -934,8 +887,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -952,8 +905,9 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -967,14 +921,13 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1020,6 +973,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,6 +981,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1044,22 +999,22 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1069,27 +1024,25 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1101,7 +1054,6 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1135,6 +1087,7 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1143,8 +1096,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1189,8 +1142,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1226,7 +1180,9 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1281,8 +1237,9 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1297,9 +1254,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1313,18 +1269,15 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1363,7 +1316,8 @@ modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= -moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= +moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= +moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/backend/main.go b/backend/main.go index 013590cc1..fe3048328 100644 --- a/backend/main.go +++ b/backend/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/crawlab-team/crawlab-core/cmd" + "github.com/crawlab-team/crawlab/core/cmd" ) func main() { diff --git a/core/go.mod b/core/go.mod index 35a77a516..218008536 100644 --- a/core/go.mod +++ b/core/go.mod @@ -4,23 +4,22 @@ go 1.22 replace ( github.com/crawlab-team/crawlab/db => ../db + github.com/crawlab-team/crawlab/fs => ../fs github.com/crawlab-team/crawlab/grpc => ../grpc github.com/crawlab-team/crawlab/template-parser => ../template-parser github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs - github.com/crawlab-team/crawlab/fs => ../fs ) require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/fs v0.6.3-2 - github.com/crawlab-team/crawlab/db v0.6.0-1 - github.com/crawlab-team/crawlab/grpc v0.0.0 - github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e - github.com/crawlab-team/crawlab/trace v0.1.0 - github.com/crawlab-team/crawlab/vcs v0.1.1 + github.com/crawlab-team/crawlab/db v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614083850-ff2830bd7994 github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.5.1 @@ -71,6 +70,7 @@ require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect @@ -117,6 +117,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect + github.com/linxGnu/gumble v1.0.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect diff --git a/core/go.sum b/core/go.sum index a6e708a6d..766155485 100644 --- a/core/go.sum +++ b/core/go.sum @@ -156,6 +156,8 @@ github.com/crawlab-team/crawlab/fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/ github.com/crawlab-team/crawlab/fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0= github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= +github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -494,6 +496,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= +github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= @@ -615,6 +619,7 @@ github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43 github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= @@ -711,6 +716,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= +github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= diff --git a/go.work b/go.work index 24bc5dbc4..bb89f2f12 100644 --- a/go.work +++ b/go.work @@ -1,6 +1,7 @@ go 1.22 use ( + backend core db grpc diff --git a/go.work.sum b/go.work.sum index 7e136fd5a..6973a18ac 100644 --- a/go.work.sum +++ b/go.work.sum @@ -125,6 +125,7 @@ cloud.google.com/go/grafeas v0.3.4 h1:D4x32R/cHX3MTofKwirz015uEdVk4uAxvZkZCZkOrF cloud.google.com/go/grafeas v0.3.4/go.mod h1:A5m316hcG+AulafjAbPKXBO/+I5itU4LOdKO2R/uDIc= cloud.google.com/go/gsuiteaddons v1.6.5 h1:CZEbaBwmbYdhFw21Fwbo+C35HMe36fTE0FBSR4KSfWg= cloud.google.com/go/gsuiteaddons v1.6.5/go.mod h1:Lo4P2IvO8uZ9W+RaC6s1JVxo42vgy+TX5a6hfBZ0ubs= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/iap v1.9.4 h1:94zirc2r4t6KzhAMW0R6Dme005eTP6yf7g6vN4IhRrA= @@ -219,6 +220,7 @@ cloud.google.com/go/speech v1.21.1 h1:nuFc+Kj5B8de75nN4FdPyUbI2SiBoHZG6BLurXL56Q cloud.google.com/go/speech v1.21.1/go.mod h1:E5GHZXYQlkqWQwY5xRSLHw2ci5NMQNG52FfMU1aZrIA= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= +cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= cloud.google.com/go/storagetransfer v1.10.4 h1:dy4fL3wO0VABvzM05ycMUPFHxTPbJz9Em8ikAJVqSbI= @@ -301,7 +303,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= @@ -315,7 +316,6 @@ github.com/fasthttp/websocket v1.4.3-rc.6 h1:omHqsl8j+KXpmzRjF8bmzOSYJ8GnS0E3efi github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CACNqNvCqQCoty/Lc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= @@ -326,6 +326,7 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -356,6 +357,7 @@ github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= @@ -369,6 +371,7 @@ github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= @@ -430,7 +433,6 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -490,18 +492,22 @@ go.etcd.io/etcd/client/v2 v2.305.1 h1:vtxYCKWA9x31w0WJj7DdqsHFNjhkigdAnziDtkZb/l go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0/go.mod h1:tIKj3DbO8N9Y2xo52og3irLsPI4GW02DSMtrVgNMgxg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0/go.mod h1:rdENBZMT2OE6Ne/KLwpiXudnAsbdrdBaqBvTN8M8BgA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= @@ -509,6 +515,7 @@ go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8p go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= @@ -543,7 +550,6 @@ golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -568,13 +574,13 @@ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6f google.golang.org/api v0.62.0 h1:PhGymJMXfGBzc4lBRmrx9+1w4w2wEzURHNGF/sD/xGc= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/api v0.160.0/go.mod h1:0mu0TpK33qnydLvWqbImq2b1eQ5FHRSDCBzAxX9ZHyw= google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= google.golang.org/api v0.164.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeqUB5o= google.golang.org/api v0.166.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA= google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= @@ -587,6 +593,7 @@ google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp4 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= @@ -599,6 +606,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014/go.mod h1:SaPjaZGWb0lPqs6Ittu0spdfrOArqji4ZdeP5IC/9N4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= @@ -611,13 +619,13 @@ google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpX google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= From 0834d6771b2772f2806f2edc437062221c754e2e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 17:32:56 +0800 Subject: [PATCH 002/106] fix: dependency issue --- backend/go.mod | 30 ++-- backend/go.sum | 36 ++-- core/go.mod | 58 +++--- core/go.sum | 132 +++++++------- db/go.mod | 37 ++-- fs/go.mod | 20 ++- fs/go.sum | 27 +-- go.work | 2 +- go.work.sum | 392 ++++++++++++++++++++++++++++++++++++++++- grpc/go.mod | 8 +- grpc/go.sum | 16 +- template-parser/go.mod | 8 +- template-parser/go.sum | 8 +- vcs/go.mod | 12 +- vcs/go.sum | 16 +- 15 files changed, 589 insertions(+), 213 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index abd2ff574..27fa3b36f 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -24,7 +24,7 @@ require ( ) require ( - cloud.google.com/go v0.99.0 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -38,11 +38,11 @@ require ( github.com/cenkalti/backoff/v4 v4.1.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/crawlab-team/crawlab/db v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/db v0.6.0-1 // indirect + github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 // indirect + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 // indirect + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614083850-ff2830bd7994 // indirect + github.com/crawlab-team/crawlab/vcs v0.1.1 // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect @@ -140,16 +140,16 @@ require ( go.mongodb.org/mongo-driver v1.15.0 // indirect go.uber.org/dig v1.10.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/backend/go.sum b/backend/go.sum index c6e582ba6..b192a3caf 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -28,7 +28,6 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -36,6 +35,7 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -791,8 +791,7 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -833,8 +832,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -887,8 +885,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -906,8 +903,7 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -921,8 +917,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1005,16 +1000,14 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1028,8 +1021,7 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1096,8 +1088,7 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1143,8 +1134,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1209,8 +1198,8 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1254,8 +1243,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= diff --git a/core/go.mod b/core/go.mod index 218008536..4034912d3 100644 --- a/core/go.mod +++ b/core/go.mod @@ -15,14 +15,15 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/db v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/db v0.6.0-1 github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/template-parser v0.0.0-00010101000000-000000000000 github.com/crawlab-team/crawlab/trace v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614083850-ff2830bd7994 + github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 - github.com/fsnotify/fsnotify v1.5.1 + github.com/fsnotify/fsnotify v1.7.0 github.com/gavv/httpexpect/v2 v2.16.0 github.com/gin-gonic/gin v1.9.1 github.com/go-git/go-git/v5 v5.12.0 @@ -40,7 +41,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v1.3.0 - github.com/spf13/viper v1.10.0 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.9.1 github.com/upper/db/v4 v4.6.0 @@ -56,7 +57,6 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -71,7 +71,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect github.com/fatih/color v1.15.0 // indirect @@ -113,41 +113,42 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect github.com/linxGnu/gumble v1.0.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/olekukonko/tablewriter v0.0.1 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sanity-io/litter v1.5.5 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/smartystreets/assertions v1.0.0 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -169,21 +170,24 @@ require ( github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect - gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect moul.io/http2curl/v2 v2.3.0 // indirect ) diff --git a/core/go.sum b/core/go.sum index 766155485..8828b1167 100644 --- a/core/go.sum +++ b/core/go.sum @@ -3,7 +3,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -16,7 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -28,8 +26,8 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -37,9 +35,8 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= -cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -52,7 +49,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -152,21 +148,16 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/crawlab/fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/jK8p0L9HhA= -github.com/crawlab-team/crawlab/fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0= -github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= -github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= -github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -196,9 +187,12 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= @@ -323,7 +317,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -339,7 +332,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -470,8 +462,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -498,10 +490,10 @@ github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -542,8 +534,9 @@ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTS github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -564,10 +557,9 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -579,10 +571,10 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -616,6 +608,10 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -650,21 +646,24 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -685,12 +684,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= @@ -760,7 +759,6 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -781,6 +779,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= @@ -789,6 +789,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -811,7 +813,6 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -819,11 +820,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -835,6 +835,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -864,8 +866,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -903,7 +905,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -919,8 +920,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -938,8 +939,8 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -953,8 +954,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1005,7 +1006,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1013,7 +1013,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1040,16 +1039,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1063,8 +1062,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1124,7 +1123,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1133,8 +1131,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1180,8 +1178,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1217,9 +1213,7 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1247,8 +1241,8 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1292,8 +1286,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1308,8 +1302,9 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= @@ -1324,7 +1319,6 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/db/go.mod b/db/go.mod index 861550fc0..987d9c4d6 100644 --- a/db/go.mod +++ b/db/go.mod @@ -2,51 +2,56 @@ module github.com/crawlab-team/crawlab/db go 1.22 +replace github.com/crawlab-team/crawlab/trace => ../trace + require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/vcs v0.1.1 + github.com/crawlab-team/crawlab/trace v0.0.0-00010101000000-000000000000 github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 - github.com/spf13/viper v1.10.0 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.15.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/lib/pq v1.10.4 // indirect - github.com/magiconair/properties v1.8.5 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/ztrue/tracerr v0.4.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/fs/go.mod b/fs/go.mod index bc989c3de..db1decd27 100644 --- a/fs/go.mod +++ b/fs/go.mod @@ -2,16 +2,24 @@ module github.com/crawlab-team/crawlab/fs go 1.22 -replace ( - github.com/crawlab-team/crawlab/trace => ../trace -) +replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 - github.com/emirpasic/gods v1.18.1 // indirect - github.com/google/uuid v1.1.1 + github.com/emirpasic/gods v1.18.1 + github.com/google/uuid v1.6.0 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/linxGnu/gumble v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/stretchr/testify v1.6.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/fs/go.sum b/fs/go.sum index dff61a29c..957e00efe 100644 --- a/fs/go.sum +++ b/fs/go.sum @@ -7,13 +7,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= -github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936 h1:c4SgTj2baDqD2UYa1eCpj3ukOF3mXOjvOCP4cWwgfyw= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -24,21 +19,19 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -46,13 +39,13 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -60,10 +53,9 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -72,8 +64,6 @@ github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eN github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y= -github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -87,16 +77,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go.work b/go.work index bb89f2f12..e45174e92 100644 --- a/go.work +++ b/go.work @@ -4,9 +4,9 @@ use ( backend core db + fs grpc template-parser trace vcs - fs ) diff --git a/go.work.sum b/go.work.sum index 6973a18ac..d8bd2dc88 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,277 +1,534 @@ +cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= +cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= cloud.google.com/go/accessapproval v1.7.5 h1:uzmAMSgYcnlHa9X9YSQZ4Q1wlfl4NNkZyQgho1Z6p04= cloud.google.com/go/accessapproval v1.7.5/go.mod h1:g88i1ok5dvQ9XJsxpUInWWvUBrIZhyPDPbk4T01OoJ0= +cloud.google.com/go/accessapproval v1.7.7 h1:vO95gvBi7qUgfA9SflexQs9hB4U4tnri/GwADIrLQy8= +cloud.google.com/go/accessapproval v1.7.7/go.mod h1:10ZDPYiTm8tgxuMPid8s2DL93BfCt6xBh/Vg0Xd8pU0= cloud.google.com/go/accesscontextmanager v1.8.5 h1:2GLNaNu9KRJhJBFTIVRoPwk6xE5mUDgD47abBq4Zp/I= cloud.google.com/go/accesscontextmanager v1.8.5/go.mod h1:TInEhcZ7V9jptGNqN3EzZ5XMhT6ijWxTGjzyETwmL0Q= +cloud.google.com/go/accesscontextmanager v1.8.7 h1:GgdNoDwZR5RIO3j8XwXqa6Gc6q5mP3KYMdFC7FEVyG4= +cloud.google.com/go/accesscontextmanager v1.8.7/go.mod h1:jSvChL1NBQ+uLY9zUBdPy9VIlozPoHptdBnRYeWuQoM= cloud.google.com/go/aiplatform v1.60.0 h1:0cSrii1ZeLr16MbBoocyy5KVnrSdiQ3KN/vtrTe7RqE= cloud.google.com/go/aiplatform v1.60.0/go.mod h1:eTlGuHOahHprZw3Hio5VKmtThIOak5/qy6pzdsqcQnM= +cloud.google.com/go/aiplatform v1.67.0 h1:YWeqD4BjYwrmY4fa+isGcw0P81lJ3dKVxbWxdBchoiU= +cloud.google.com/go/aiplatform v1.67.0/go.mod h1:s/sJ6btBEr6bKnrNWdK9ZgHCvwbZNdP90b3DDtxxw+Y= cloud.google.com/go/analytics v0.23.0 h1:Q+y94XH84jM8SK8O7qiY/PJRexb6n7dRbQ6PiUa4YGM= cloud.google.com/go/analytics v0.23.0/go.mod h1:YPd7Bvik3WS95KBok2gPXDqQPHy08TsCQG6CdUCb+u0= +cloud.google.com/go/analytics v0.23.2 h1:O0fj88npvQFxg8LfXo7fArcSrC/wtAstGuWQ7dCHWjg= +cloud.google.com/go/analytics v0.23.2/go.mod h1:vtE3olAXZ6edJYk1UOndEs6EfaEc9T2B28Y4G5/a7Fo= cloud.google.com/go/apigateway v1.6.5 h1:sPXnpk+6TneKIrjCjcpX5YGsAKy3PTdpIchoj8/74OE= cloud.google.com/go/apigateway v1.6.5/go.mod h1:6wCwvYRckRQogyDDltpANi3zsCDl6kWi0b4Je+w2UiI= +cloud.google.com/go/apigateway v1.6.7 h1:DO5Vn3zmY1aDyfoqni8e8+x+lwrfLCoAAbEui9NB0y8= +cloud.google.com/go/apigateway v1.6.7/go.mod h1:7wAMb/33Rzln+PrGK16GbGOfA1zAO5Pq6wp19jtIt7c= cloud.google.com/go/apigeeconnect v1.6.5 h1:CrfIKv9Go3fh/QfQgisU3MeP90Ww7l/sVGmr3TpECo8= cloud.google.com/go/apigeeconnect v1.6.5/go.mod h1:MEKm3AiT7s11PqTfKE3KZluZA9O91FNysvd3E6SJ6Ow= +cloud.google.com/go/apigeeconnect v1.6.7 h1:z08Xuv7ZtaB2d4jsJi9/WhbnnI5s19wlLDZpssn3Fus= +cloud.google.com/go/apigeeconnect v1.6.7/go.mod h1:hZxCKvAvDdKX8+eT0g5eEAbRSS9Gkzi+MPWbgAMAy5U= cloud.google.com/go/apigeeregistry v0.8.3 h1:C+QU2K+DzDjk4g074ouwHQGkoff1h5OMQp6sblCVreQ= cloud.google.com/go/apigeeregistry v0.8.3/go.mod h1:aInOWnqF4yMQx8kTjDqHNXjZGh/mxeNlAf52YqtASUs= +cloud.google.com/go/apigeeregistry v0.8.5 h1:o1C/+IvzwYeV1doum61XmJQ/Bwpk/4+2DT1JyVu2x64= +cloud.google.com/go/apigeeregistry v0.8.5/go.mod h1:ZMg60hq2K35tlqZ1VVywb9yjFzk9AJ7zqxrysOxLi3o= cloud.google.com/go/appengine v1.8.5 h1:l2SviT44zWQiOv8bPoMBzW0vOcMO22iO0s+nVtVhdts= cloud.google.com/go/appengine v1.8.5/go.mod h1:uHBgNoGLTS5di7BvU25NFDuKa82v0qQLjyMJLuPQrVo= +cloud.google.com/go/appengine v1.8.7 h1:qYrjEHEFY7+CL4QlHIHuwTgrTnZbSKzdPFqgjZDsQNo= +cloud.google.com/go/appengine v1.8.7/go.mod h1:1Fwg2+QTgkmN6Y+ALGwV8INLbdkI7+vIvhcKPZCML0g= cloud.google.com/go/area120 v0.8.5 h1:vTs08KPLN/iMzTbxpu5ciL06KcsrVPMjz4IwcQyZ4uY= cloud.google.com/go/area120 v0.8.5/go.mod h1:BcoFCbDLZjsfe4EkCnEq1LKvHSK0Ew/zk5UFu6GMyA0= +cloud.google.com/go/area120 v0.8.7 h1:sUrR96yokdL6tTTXK0X13V1TLMta8/1u328bRG5lWZc= +cloud.google.com/go/area120 v0.8.7/go.mod h1:L/xTq4NLP9mmxiGdcsVz7y1JLc9DI8pfaXRXbnjkR6w= cloud.google.com/go/artifactregistry v1.14.7 h1:W9sVlyb1VRcUf83w7aM3yMsnp4HS4PoyGqYQNG0O5lI= cloud.google.com/go/artifactregistry v1.14.7/go.mod h1:0AUKhzWQzfmeTvT4SjfI4zjot72EMfrkvL9g9aRjnnM= +cloud.google.com/go/artifactregistry v1.14.9 h1:SSvoD0ofOydm5gA1++15pW9VPgQbk0OmNlcb7JczoO4= +cloud.google.com/go/artifactregistry v1.14.9/go.mod h1:n2OsUqbYoUI2KxpzQZumm6TtBgtRf++QulEohdnlsvI= cloud.google.com/go/asset v1.17.2 h1:xgFnBP3luSbUcC9RWJvb3Zkt+y/wW6PKwPHr3ssnIP8= cloud.google.com/go/asset v1.17.2/go.mod h1:SVbzde67ehddSoKf5uebOD1sYw8Ab/jD/9EIeWg99q4= +cloud.google.com/go/asset v1.19.1 h1:mCqyoaDjDzaW1RqmmQtCJuawb9nca5bEu7HvVcpZDwg= +cloud.google.com/go/asset v1.19.1/go.mod h1:kGOS8DiCXv6wU/JWmHWCgaErtSZ6uN5noCy0YwVaGfs= cloud.google.com/go/assuredworkloads v1.11.5 h1:gCrN3IyvqY3cP0wh2h43d99CgH3G+WYs9CeuFVKChR8= cloud.google.com/go/assuredworkloads v1.11.5/go.mod h1:FKJ3g3ZvkL2D7qtqIGnDufFkHxwIpNM9vtmhvt+6wqk= +cloud.google.com/go/assuredworkloads v1.11.7 h1:xieyFA+JKyTDkO/Z9UyVEpkHW8pDYykU51O4G0pvXEg= +cloud.google.com/go/assuredworkloads v1.11.7/go.mod h1:CqXcRH9N0KCDtHhFisv7kk+cl//lyV+pYXGi1h8rCEU= +cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= +cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= +cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg= +cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/automl v1.13.5 h1:ijiJy9sYWh75WrqImXsfWc1e3HR3iO+ef9fvW03Ig/4= cloud.google.com/go/automl v1.13.5/go.mod h1:MDw3vLem3yh+SvmSgeYUmUKqyls6NzSumDm9OJ3xJ1Y= +cloud.google.com/go/automl v1.13.7 h1:w9AyogtMLXbcy5kzXPvk/Q3MGQkgJH7ZDB8fAUUxTt8= +cloud.google.com/go/automl v1.13.7/go.mod h1:E+s0VOsYXUdXpq0y4gNZpi0A/s6y9+lAarmV5Eqlg40= cloud.google.com/go/baremetalsolution v1.2.4 h1:LFydisRmS7hQk9P/YhekwuZGqb45TW4QavcrMToWo5A= cloud.google.com/go/baremetalsolution v1.2.4/go.mod h1:BHCmxgpevw9IEryE99HbYEfxXkAEA3hkMJbYYsHtIuY= +cloud.google.com/go/baremetalsolution v1.2.6 h1:W4oSMS6vRCo9DLr1RPyDP8oeLverbvhJRzaZSsipft8= +cloud.google.com/go/baremetalsolution v1.2.6/go.mod h1:KkS2BtYXC7YGbr42067nzFr+ABFMs6cxEcA1F+cedIw= cloud.google.com/go/batch v1.8.0 h1:2HK4JerwVaIcCh/lJiHwh6+uswPthiMMWhiSWLELayk= cloud.google.com/go/batch v1.8.0/go.mod h1:k8V7f6VE2Suc0zUM4WtoibNrA6D3dqBpB+++e3vSGYc= +cloud.google.com/go/batch v1.8.7 h1:zaQwOAd7TlE84pwPHavNMsnv5zRyRV8ym2DJ4iQ2cV0= +cloud.google.com/go/batch v1.8.7/go.mod h1:O5/u2z8Wc7E90Bh4yQVLQIr800/0PM5Qzvjac3Jxt4k= cloud.google.com/go/beyondcorp v1.0.4 h1:qs0J0O9Ol2h1yA0AU+r7l3hOCPzs2MjE1d6d/kaHIKo= cloud.google.com/go/beyondcorp v1.0.4/go.mod h1:Gx8/Rk2MxrvWfn4WIhHIG1NV7IBfg14pTKv1+EArVcc= +cloud.google.com/go/beyondcorp v1.0.6 h1:KBcujO3QRvBIwzZLtvQEPB9SXdovHnMBx0V/uhucH9o= +cloud.google.com/go/beyondcorp v1.0.6/go.mod h1:wRkenqrVRtnGFfnyvIg0zBFUdN2jIfeojFF9JJDwVIA= cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.59.1 h1:CpT+/njKuKT3CEmswm6IbhNu9u35zt5dO4yPDLW+nG4= cloud.google.com/go/bigquery v1.59.1/go.mod h1:VP1UJYgevyTwsV7desjzNzDND5p6hZB+Z8gZJN1GQUc= +cloud.google.com/go/bigquery v1.61.0 h1:w2Goy9n6gh91LVi6B2Sc+HpBl8WbWhIyzdvVvrAuEIw= +cloud.google.com/go/bigquery v1.61.0/go.mod h1:PjZUje0IocbuTOdq4DBOJLNYB0WF3pAKBHzAYyxCwFo= cloud.google.com/go/billing v1.18.2 h1:oWUEQvuC4JvtnqLZ35zgzdbuHt4Itbftvzbe6aEyFdE= cloud.google.com/go/billing v1.18.2/go.mod h1:PPIwVsOOQ7xzbADCwNe8nvK776QpfrOAUkvKjCUcpSE= +cloud.google.com/go/billing v1.18.5 h1:GbOg1uGvoV8FXxMStFoNcq5z9AEUwCpKt/6GNcuDSZM= +cloud.google.com/go/billing v1.18.5/go.mod h1:lHw7fxS6p7hLWEPzdIolMtOd0ahLwlokW06BzbleKP8= cloud.google.com/go/binaryauthorization v1.8.1 h1:1jcyh2uIUwSZkJ/JmL8kd5SUkL/Krbv8zmYLEbAz6kY= cloud.google.com/go/binaryauthorization v1.8.1/go.mod h1:1HVRyBerREA/nhI7yLang4Zn7vfNVA3okoAR9qYQJAQ= +cloud.google.com/go/binaryauthorization v1.8.3 h1:RHnEM4HXbWShlGhPA0Jzj2YYETCHxmisNMU0OE2fXQM= +cloud.google.com/go/binaryauthorization v1.8.3/go.mod h1:Cul4SsGlbzEsWPOz2sH8m+g2Xergb6ikspUyQ7iOThE= cloud.google.com/go/certificatemanager v1.7.5 h1:UMBr/twXvH3jcT5J5/YjRxf2tvwTYIfrpemTebe0txc= cloud.google.com/go/certificatemanager v1.7.5/go.mod h1:uX+v7kWqy0Y3NG/ZhNvffh0kuqkKZIXdvlZRO7z0VtM= +cloud.google.com/go/certificatemanager v1.8.1 h1:XURrQhj5COWAEvICivbGID/Hu67AvMYHAhMRIyc3Ux8= +cloud.google.com/go/certificatemanager v1.8.1/go.mod h1:hDQzr50Vx2gDB+dOfmDSsQzJy/UPrYRdzBdJ5gAVFIc= cloud.google.com/go/channel v1.17.5 h1:/omiBnyFjm4S1ETHoOmJbL7LH7Ljcei4rYG6Sj3hc80= cloud.google.com/go/channel v1.17.5/go.mod h1:FlpaOSINDAXgEext0KMaBq/vwpLMkkPAw9b2mApQeHc= +cloud.google.com/go/channel v1.17.7 h1:PrplNaAS6Dn187e+OcGzyEKETX8iL3tCaDqcPPW7Zoo= +cloud.google.com/go/channel v1.17.7/go.mod h1:b+FkgBrhMKM3GOqKUvqHFY/vwgp+rwsAuaMd54wCdN4= cloud.google.com/go/cloudbuild v1.15.1 h1:ZB6oOmJo+MTov9n629fiCrO9YZPOg25FZvQ7gIHu5ng= cloud.google.com/go/cloudbuild v1.15.1/go.mod h1:gIofXZSu+XD2Uy+qkOrGKEx45zd7s28u/k8f99qKals= +cloud.google.com/go/cloudbuild v1.16.1 h1:zkCG1dBezxRM3dtgQ9h1Y+IJ7V+lARWgp0l9k/SZsfU= +cloud.google.com/go/cloudbuild v1.16.1/go.mod h1:c2KUANTtCBD8AsRavpPout6Vx8W+fsn5zTsWxCpWgq4= cloud.google.com/go/clouddms v1.7.4 h1:Sr0Zo5EAcPQiCBgHWICg3VGkcdS/LLP1d9SR7qQBM/s= cloud.google.com/go/clouddms v1.7.4/go.mod h1:RdrVqoFG9RWI5AvZ81SxJ/xvxPdtcRhFotwdE79DieY= +cloud.google.com/go/clouddms v1.7.6 h1:Q47KKoA0zsNcC9U5aCmop5TPPItVq4cx7Wwqgra+5PU= +cloud.google.com/go/clouddms v1.7.6/go.mod h1:8HWZ2tznZ0mNAtTpfnRNT0QOThqn9MBUqTj0Lx8npIs= cloud.google.com/go/cloudtasks v1.12.6 h1:EUt1hIZ9bLv8Iz9yWaCrqgMnIU+Tdh0yXM1MMVGhjfE= cloud.google.com/go/cloudtasks v1.12.6/go.mod h1:b7c7fe4+TJsFZfDyzO51F7cjq7HLUlRi/KZQLQjDsaY= +cloud.google.com/go/cloudtasks v1.12.8 h1:Y0HUuiCAVk9BojLItOycBl91tY25NXH8oFsyi1IC/U4= +cloud.google.com/go/cloudtasks v1.12.8/go.mod h1:aX8qWCtmVf4H4SDYUbeZth9C0n9dBj4dwiTYi4Or/P4= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute v1.27.0 h1:EGawh2RUnfHT5g8f/FX3Ds6KZuIBC77hZoDrBvEZw94= +cloud.google.com/go/compute v1.27.0/go.mod h1:LG5HwRmWFKM2C5XxHRiNzkLLXW48WwvyVC0mfWsYPOM= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.13.0 h1:6Vs/YnDG5STGjlWMEjN/xtmft7MrOTOnOZYUZtGTx0w= cloud.google.com/go/contactcenterinsights v1.13.0/go.mod h1:ieq5d5EtHsu8vhe2y3amtZ+BE+AQwX5qAy7cpo0POsI= +cloud.google.com/go/contactcenterinsights v1.13.2 h1:46ertIh+cGkTg/lN7fN+TOx09SoM65dpdUp96vXBcMY= +cloud.google.com/go/contactcenterinsights v1.13.2/go.mod h1:AfkSB8t7mt2sIY6WpfO61nD9J9fcidIchtxm9FqJVXk= cloud.google.com/go/container v1.31.0 h1:MAaNH7VRNPWEhvqOypq2j+7ONJKrKzon4v9nS3nLZe0= cloud.google.com/go/container v1.31.0/go.mod h1:7yABn5s3Iv3lmw7oMmyGbeV6tQj86njcTijkkGuvdZA= +cloud.google.com/go/container v1.35.1 h1:Vbu/3PZNrgV1Z5DGcRubQdUccX/uMUDNc+NgHNIfbEk= +cloud.google.com/go/container v1.35.1/go.mod h1:udm8fgLm3TtpnjFN4QLLjZezAIIp/VnMo316yIRVRQU= cloud.google.com/go/containeranalysis v0.11.4 h1:doJ0M1ljS4hS0D2UbHywlHGwB7sQLNrt9vFk9Zyi7vY= cloud.google.com/go/containeranalysis v0.11.4/go.mod h1:cVZT7rXYBS9NG1rhQbWL9pWbXCKHWJPYraE8/FTSYPE= +cloud.google.com/go/containeranalysis v0.11.6 h1:mSrneOVadcpnDZHJebg+ts/10azGTUKOCSQET7KdT7g= +cloud.google.com/go/containeranalysis v0.11.6/go.mod h1:YRf7nxcTcN63/Kz9f86efzvrV33g/UV8JDdudRbYEUI= cloud.google.com/go/datacatalog v1.19.3 h1:A0vKYCQdxQuV4Pi0LL9p39Vwvg4jH5yYveMv50gU5Tw= cloud.google.com/go/datacatalog v1.19.3/go.mod h1:ra8V3UAsciBpJKQ+z9Whkxzxv7jmQg1hfODr3N3YPJ4= +cloud.google.com/go/datacatalog v1.20.1 h1:czcba5mxwRM5V//jSadyig0y+8aOHmN7gUl9GbHu59E= +cloud.google.com/go/datacatalog v1.20.1/go.mod h1:Jzc2CoHudhuZhpv78UBAjMEg3w7I9jHA11SbRshWUjk= cloud.google.com/go/dataflow v0.9.5 h1:RYHtcPhmE664+F0Je46p+NvFbG8z//KCXp+uEqB4jZU= cloud.google.com/go/dataflow v0.9.5/go.mod h1:udl6oi8pfUHnL0z6UN9Lf9chGqzDMVqcYTcZ1aPnCZQ= +cloud.google.com/go/dataflow v0.9.7 h1:wKEakCbRevlwsWqTn34pWJUFmdbx0HKwpRH6HhU7NIs= +cloud.google.com/go/dataflow v0.9.7/go.mod h1:3BjkOxANrm1G3+/EBnEsTEEgJu1f79mFqoOOZfz3v+E= cloud.google.com/go/dataform v0.9.2 h1:5e4eqGrd0iDTCg4Q+VlAao5j2naKAA7xRurNtwmUknU= cloud.google.com/go/dataform v0.9.2/go.mod h1:S8cQUwPNWXo7m/g3DhWHsLBoufRNn9EgFrMgne2j7cI= +cloud.google.com/go/dataform v0.9.4 h1:MiK1Us7YP9+sdNViUE4X2B2vLScrKcjOPw5b6uamZvE= +cloud.google.com/go/dataform v0.9.4/go.mod h1:jjo4XY+56UrNE0wsEQsfAw4caUs4DLJVSyFBDelRDtQ= cloud.google.com/go/datafusion v1.7.5 h1:HQ/BUOP8OIGJxuztpYvNvlb+/U+/Bfs9SO8tQbh61fk= cloud.google.com/go/datafusion v1.7.5/go.mod h1:bYH53Oa5UiqahfbNK9YuYKteeD4RbQSNMx7JF7peGHc= +cloud.google.com/go/datafusion v1.7.7 h1:ViFnMnUK7LNcWvisZgihxXit76JxSHFeijYI5U/gjOE= +cloud.google.com/go/datafusion v1.7.7/go.mod h1:qGTtQcUs8l51lFA9ywuxmZJhS4ozxsBSus6ItqCUWMU= cloud.google.com/go/datalabeling v0.8.5 h1:GpIFRdm0qIZNsxqURFJwHt0ZBJZ0nF/mUVEigR7PH/8= cloud.google.com/go/datalabeling v0.8.5/go.mod h1:IABB2lxQnkdUbMnQaOl2prCOfms20mcPxDBm36lps+s= +cloud.google.com/go/datalabeling v0.8.7 h1:M6irSHns6VxMro+IbvDxDJLD6tkfjlW+mo2MPaM23KA= +cloud.google.com/go/datalabeling v0.8.7/go.mod h1:/PPncW5gxrU15UzJEGQoOT3IobeudHGvoExrtZ8ZBwo= cloud.google.com/go/dataplex v1.14.2 h1:fxIfdU8fxzR3clhOoNI7XFppvAmndxDu1AMH+qX9WKQ= cloud.google.com/go/dataplex v1.14.2/go.mod h1:0oGOSFlEKef1cQeAHXy4GZPB/Ife0fz/PxBf+ZymA2U= +cloud.google.com/go/dataplex v1.16.0 h1:e8SV0yKuSjgHEZaQcZwjKXe0ta1jZrvLxX/2i/IAG+8= +cloud.google.com/go/dataplex v1.16.0/go.mod h1:OlBoytuQ56+7aUCC03D34CtoF/4TJ5SiIrLsBdDu87Q= cloud.google.com/go/dataproc v1.12.0 h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3YgGU= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataproc/v2 v2.4.0 h1:/u81Fd+BvCLp+xjctI1DiWVJn6cn9/s3Akc8xPH02yk= cloud.google.com/go/dataproc/v2 v2.4.0/go.mod h1:3B1Ht2aRB8VZIteGxQS/iNSJGzt9+CA0WGnDVMEm7Z4= +cloud.google.com/go/dataproc/v2 v2.4.2 h1:RNMG5ffWKdbWOkwvjC4GqxLaxEaWFpm2hQCF2WFW/vo= +cloud.google.com/go/dataproc/v2 v2.4.2/go.mod h1:smGSj1LZP3wtnsM9eyRuDYftNAroAl6gvKp/Wk64XDE= cloud.google.com/go/dataqna v0.8.5 h1:9ybXs3nr9BzxSGC04SsvtuXaHY0qmJSLIpIAbZo9GqQ= cloud.google.com/go/dataqna v0.8.5/go.mod h1:vgihg1mz6n7pb5q2YJF7KlXve6tCglInd6XO0JGOlWM= +cloud.google.com/go/dataqna v0.8.7 h1:qM60MGNTGsSJuzAziVJjtRA7pGby2dA8OuqdVRe/lYo= +cloud.google.com/go/dataqna v0.8.7/go.mod h1:hvxGaSvINAVH5EJJsONIwT1y+B7OQogjHPjizOFoWOo= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.15.0 h1:0P9WcsQeTWjuD1H14JIY7XQscIPQ4Laje8ti96IC5vg= cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= +cloud.google.com/go/datastore v1.17.0 h1:UEmzuUdyDE58HV2jcb0BoqwCAwsJS2mtHapCsMmhVh0= +cloud.google.com/go/datastore v1.17.0/go.mod h1:RiRZU0G6VVlIVlv1HRo3vSAPFHULV0ddBNsXO+Sony4= cloud.google.com/go/datastream v1.10.4 h1:o1QDKMo/hk0FN7vhoUQURREuA0rgKmnYapB+1M+7Qz4= cloud.google.com/go/datastream v1.10.4/go.mod h1:7kRxPdxZxhPg3MFeCSulmAJnil8NJGGvSNdn4p1sRZo= +cloud.google.com/go/datastream v1.10.6 h1:FfNUy9j3aRQ99L4a5Rdm82RMuiw0BIe3lpPn2ykom8k= +cloud.google.com/go/datastream v1.10.6/go.mod h1:lPeXWNbQ1rfRPjBFBLUdi+5r7XrniabdIiEaCaAU55o= cloud.google.com/go/deploy v1.17.1 h1:m27Ojwj03gvpJqCbodLYiVmE9x4/LrHGGMjzc0LBfM4= cloud.google.com/go/deploy v1.17.1/go.mod h1:SXQyfsXrk0fBmgBHRzBjQbZhMfKZ3hMQBw5ym7MN/50= +cloud.google.com/go/deploy v1.19.0 h1:fzbObuGgoViO0ArFuOQIJ2yr5bH5YzbORVvMDBrDC5I= +cloud.google.com/go/deploy v1.19.0/go.mod h1:BW9vAujmxi4b/+S7ViEuYR65GiEsqL6Mhf5S/9TeDRU= cloud.google.com/go/dialogflow v1.49.0 h1:KqG0oxGE71qo0lRVyAoeBozefCvsMfcDzDjoLYSY0F4= cloud.google.com/go/dialogflow v1.49.0/go.mod h1:dhVrXKETtdPlpPhE7+2/k4Z8FRNUp6kMV3EW3oz/fe0= +cloud.google.com/go/dialogflow v1.53.0 h1:C9wQ0odRYQsar0XqwCQb0c13BkRBsoSjOaejOg5ntgQ= +cloud.google.com/go/dialogflow v1.53.0/go.mod h1:LqAvxq7bXiiGC3/DWIz9XXCxth2z2qpSnBAAmlNOj6U= cloud.google.com/go/dlp v1.11.2 h1:lTipOuJaSjlYnnotPMbEhKURLC6GzCMDDzVbJAEbmYM= cloud.google.com/go/dlp v1.11.2/go.mod h1:9Czi+8Y/FegpWzgSfkRlyz+jwW6Te9Rv26P3UfU/h/w= +cloud.google.com/go/dlp v1.14.0 h1:/GQVl5gOPR2dUemrR2YJxZG5D9MCE3AYgmDxjzP54jI= +cloud.google.com/go/dlp v1.14.0/go.mod h1:4fvEu3EbLsHrgH3QFdFlTNIiCP5mHwdYhS/8KChDIC4= cloud.google.com/go/documentai v1.25.0 h1:lI62GMEEPO6vXJI9hj+G9WjOvnR0hEjvjokrnex4cxA= cloud.google.com/go/documentai v1.25.0/go.mod h1:ftLnzw5VcXkLItp6pw1mFic91tMRyfv6hHEY5br4KzY= +cloud.google.com/go/documentai v1.30.0 h1:6KI6P04WExzrfbciW5RTEQScBEY98Fc4VtS04ufT3Js= +cloud.google.com/go/documentai v1.30.0/go.mod h1:3Qt8PMt3S8W6w3VeoYFraaMS2GJRrXFnvkyn+GpB1n0= cloud.google.com/go/domains v0.9.5 h1:Mml/R6s3vQQvFPpi/9oX3O5dRirgjyJ8cksK8N19Y7g= cloud.google.com/go/domains v0.9.5/go.mod h1:dBzlxgepazdFhvG7u23XMhmMKBjrkoUNaw0A8AQB55Y= +cloud.google.com/go/domains v0.9.7 h1:IixFIMRzUJWZUAOe8s/K2X4Bvtp0A3xjHLljfNC4aSo= +cloud.google.com/go/domains v0.9.7/go.mod h1:u/yVf3BgfPJW3QDZl51qTJcDXo9PLqnEIxfGmGgbHEc= cloud.google.com/go/edgecontainer v1.1.5 h1:tBY32km78ScpK2aOP84JoW/+wtpx5WluyPUSEE3270U= cloud.google.com/go/edgecontainer v1.1.5/go.mod h1:rgcjrba3DEDEQAidT4yuzaKWTbkTI5zAMu3yy6ZWS0M= +cloud.google.com/go/edgecontainer v1.2.1 h1:xa6MIQhGylE24QdWaxhfIfAJE3Pupcr+i77WEx3NJrg= +cloud.google.com/go/edgecontainer v1.2.1/go.mod h1:OE2D0lbkmGDVYLCvpj8Y0M4a4K076QB7E2JupqOR/qU= cloud.google.com/go/errorreporting v0.3.0 h1:kj1XEWMu8P0qlLhm3FwcaFsUvXChV/OraZwA70trRR0= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.6.6 h1:13eHn5qBnsawxI7mIrv4jRIEmQ1xg0Ztqw5ZGqtUNfA= cloud.google.com/go/essentialcontacts v1.6.6/go.mod h1:XbqHJGaiH0v2UvtuucfOzFXN+rpL/aU5BCZLn4DYl1Q= +cloud.google.com/go/essentialcontacts v1.6.8 h1:p5Y7ZNVPiV9pEAHzvWiPcSiQRMQqcuHxOP0ZOP0vVww= +cloud.google.com/go/essentialcontacts v1.6.8/go.mod h1:EHONVDSum2xxG2p+myyVda/FwwvGbY58ZYC4XqI/lDQ= cloud.google.com/go/eventarc v1.13.4 h1:ORkd6/UV5FIdA8KZQDLNZYKS7BBOrj0p01DXPmT4tE4= cloud.google.com/go/eventarc v1.13.4/go.mod h1:zV5sFVoAa9orc/52Q+OuYUG9xL2IIZTbbuTHC6JSY8s= +cloud.google.com/go/eventarc v1.13.6 h1:we+qx5uCZ88aQzQS3MJXRvAh/ik+EmqVyjcW1oYFW44= +cloud.google.com/go/eventarc v1.13.6/go.mod h1:QReOaYnDNdjwAQQWNC7nfr63WnaKFUw7MSdQ9PXJYj0= cloud.google.com/go/filestore v1.8.1 h1:X5G4y/vrUo1B8Nsz93qSWTMAcM8LXbGUldq33OdcdCw= cloud.google.com/go/filestore v1.8.1/go.mod h1:MbN9KcaM47DRTIuLfQhJEsjaocVebNtNQhSLhKCF5GM= +cloud.google.com/go/filestore v1.8.3 h1:CpRnsUpMU5gxUKyfh7TD0SM+E+7E4ORaDea2JctKfpY= +cloud.google.com/go/filestore v1.8.3/go.mod h1:QTpkYpKBF6jlPRmJwhLqXfJQjVrQisplyb4e2CwfJWc= cloud.google.com/go/firestore v1.6.1 h1:8rBq3zRjnHx8UtBvaOWqBB1xq9jH6/wltfQLlTMh2Fw= cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= cloud.google.com/go/firestore v1.14.0 h1:8aLcKnMPoldYU3YHgu4t2exrKhLQkqaXAGqT0ljrFVw= cloud.google.com/go/firestore v1.14.0/go.mod h1:96MVaHLsEhbvkBEdZgfN+AS/GIkco1LRpH9Xp9YZfzQ= +cloud.google.com/go/firestore v1.15.0 h1:/k8ppuWOtNuDHt2tsRV42yI21uaGnKDEQnRFeBpbFF8= +cloud.google.com/go/firestore v1.15.0/go.mod h1:GWOxFXcv8GZUtYpWHw/w6IuYNux/BtmeVTMmjrm4yhk= cloud.google.com/go/functions v1.16.0 h1:IWVylmK5F6hJ3R5zaRW7jI5PrWhCvtBVU4axQLmXSo4= cloud.google.com/go/functions v1.16.0/go.mod h1:nbNpfAG7SG7Duw/o1iZ6ohvL7mc6MapWQVpqtM29n8k= +cloud.google.com/go/functions v1.16.2 h1:83bd2lCgtu2nLbX2jrqsrQhIs7VuVA1N6Op5syeRVIg= +cloud.google.com/go/functions v1.16.2/go.mod h1:+gMvV5E3nMb9EPqX6XwRb646jTyVz8q4yk3DD6xxHpg= cloud.google.com/go/gaming v1.9.0 h1:7vEhFnZmd931Mo7sZ6pJy7uQPDxF7m7v8xtBheG08tc= cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= cloud.google.com/go/gkebackup v1.3.5 h1:iuE8KNtTsPOc79qeWoNS8zOWoXPD9SAdOmwgxtlCmh8= cloud.google.com/go/gkebackup v1.3.5/go.mod h1:KJ77KkNN7Wm1LdMopOelV6OodM01pMuK2/5Zt1t4Tvc= +cloud.google.com/go/gkebackup v1.5.0 h1:wysUXEkggPwENZY3BXroOyWoyVfPypzaqNHgOZD9Kck= +cloud.google.com/go/gkebackup v1.5.0/go.mod h1:eLaf/+n8jEmIvOvDriGjo99SN7wRvVadoqzbZu0WzEw= cloud.google.com/go/gkeconnect v0.8.5 h1:17d+ZSSXKqG/RwZCq3oFMIWLPI8Zw3b8+a9/BEVlwH0= cloud.google.com/go/gkeconnect v0.8.5/go.mod h1:LC/rS7+CuJ5fgIbXv8tCD/mdfnlAadTaUufgOkmijuk= +cloud.google.com/go/gkeconnect v0.8.7 h1:BfXsTXYs5xlicAlgbtlo8Cw+YdzU3PrlBg7dATJUwrk= +cloud.google.com/go/gkeconnect v0.8.7/go.mod h1:iUH1jgQpTyNFMK5LgXEq2o0beIJ2p7KKUUFerkf/eGc= cloud.google.com/go/gkehub v0.14.5 h1:RboLNFzf9wEMSo7DrKVBlf+YhK/A/jrLN454L5Tz99Q= cloud.google.com/go/gkehub v0.14.5/go.mod h1:6bzqxM+a+vEH/h8W8ec4OJl4r36laxTs3A/fMNHJ0wA= +cloud.google.com/go/gkehub v0.14.7 h1:bHwcvgh8AmcYm6p6/ZrWW3a7J7sKBDtqtsyVXKssnPs= +cloud.google.com/go/gkehub v0.14.7/go.mod h1:NLORJVTQeCdxyAjDgUwUp0A6BLEaNLq84mCiulsM4OE= cloud.google.com/go/gkemulticloud v1.1.1 h1:rsSZAGLhyjyE/bE2ToT5fqo1qSW7S+Ubsc9jFOcbhSI= cloud.google.com/go/gkemulticloud v1.1.1/go.mod h1:C+a4vcHlWeEIf45IB5FFR5XGjTeYhF83+AYIpTy4i2Q= +cloud.google.com/go/gkemulticloud v1.2.0 h1:zaWBakKPT6mPHVn5iefuRqttjpbNsb8LlMw9KgfyfyU= +cloud.google.com/go/gkemulticloud v1.2.0/go.mod h1:iN5wBxTLPR6VTBWpkUsOP2zuPOLqZ/KbgG1bZir1Cng= cloud.google.com/go/grafeas v0.3.4 h1:D4x32R/cHX3MTofKwirz015uEdVk4uAxvZkZCZkOrF4= cloud.google.com/go/grafeas v0.3.4/go.mod h1:A5m316hcG+AulafjAbPKXBO/+I5itU4LOdKO2R/uDIc= +cloud.google.com/go/grafeas v0.3.5 h1:Z87HxC4vnjR1kWWtzP6BuQXa6xBmndRK/kaz4iu6oMA= +cloud.google.com/go/grafeas v0.3.5/go.mod h1:y54iTBcI+lgUdI+kAPKb8jtPqeTkA2dsYzWSrQtpc5s= cloud.google.com/go/gsuiteaddons v1.6.5 h1:CZEbaBwmbYdhFw21Fwbo+C35HMe36fTE0FBSR4KSfWg= cloud.google.com/go/gsuiteaddons v1.6.5/go.mod h1:Lo4P2IvO8uZ9W+RaC6s1JVxo42vgy+TX5a6hfBZ0ubs= +cloud.google.com/go/gsuiteaddons v1.6.7 h1:06Jg3JeLslEfBYX1sDqOPLnF7a3wmhNcDUXF/fVOb50= +cloud.google.com/go/gsuiteaddons v1.6.7/go.mod h1:u+sGBvr07OKNnOnQiB/Co1q4U2cjo50ERQwvnlcpNis= cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= cloud.google.com/go/iap v1.9.4 h1:94zirc2r4t6KzhAMW0R6Dme005eTP6yf7g6vN4IhRrA= cloud.google.com/go/iap v1.9.4/go.mod h1:vO4mSq0xNf/Pu6E5paORLASBwEmphXEjgCFg7aeNu1w= +cloud.google.com/go/iap v1.9.6 h1:rcuRS9XfOgr1v6TAoihVeSXntOnpVhFlVHtPfgOkLAo= +cloud.google.com/go/iap v1.9.6/go.mod h1:YiK+tbhDszhaVifvzt2zTEF2ch9duHtp6xzxj9a0sQk= cloud.google.com/go/ids v1.4.5 h1:xd4U7pgl3GHV+MABnv1BF4/Vy/zBF7CYC8XngkOLzag= cloud.google.com/go/ids v1.4.5/go.mod h1:p0ZnyzjMWxww6d2DvMGnFwCsSxDJM666Iir1bK1UuBo= +cloud.google.com/go/ids v1.4.7 h1:wtd+r415yrfZ8LsB6yH6WrOZ26tYt7w6wy3i5a4HQZ8= +cloud.google.com/go/ids v1.4.7/go.mod h1:yUkDC71u73lJoTaoONy0dsA0T7foekvg6ZRg9IJL0AA= cloud.google.com/go/iot v1.7.5 h1:munTeBlbqI33iuTYgXy7S8lW2TCgi5l1hA4roSIY+EE= cloud.google.com/go/iot v1.7.5/go.mod h1:nq3/sqTz3HGaWJi1xNiX7F41ThOzpud67vwk0YsSsqs= +cloud.google.com/go/iot v1.7.7 h1:M9SKIj9eoxoXCzytkLZVAuf5wmoui1OeDqEjC97wRbY= +cloud.google.com/go/iot v1.7.7/go.mod h1:tr0bCOSPXtsg64TwwZ/1x+ReTWKlQRVXbM+DnrE54yM= cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= +cloud.google.com/go/kms v1.17.1 h1:5k0wXqkxL+YcXd4viQzTqCgzzVKKxzgrK+rCZJytEQs= +cloud.google.com/go/kms v1.17.1/go.mod h1:DCMnCF/apA6fZk5Cj4XsD979OyHAqFasPuA5Sd0kGlQ= cloud.google.com/go/language v1.12.3 h1:iaJZg6K4j/2PvZZVcjeO/btcWWIllVRBhuTFjGO4LXs= cloud.google.com/go/language v1.12.3/go.mod h1:evFX9wECX6mksEva8RbRnr/4wi/vKGYnAJrTRXU8+f8= +cloud.google.com/go/language v1.12.5 h1:kOYJEcuZgyUX/i/4DFrfXPcrddm1XCQD2lDI5hIFmZQ= +cloud.google.com/go/language v1.12.5/go.mod h1:w/6a7+Rhg6Bc2Uzw6thRdKKNjnOzfKTJuxzD0JZZ0nM= cloud.google.com/go/lifesciences v0.9.5 h1:gXvN70m2p+4zgJFzaz6gMKaxTuF9WJ0USYoMLWAOm8g= cloud.google.com/go/lifesciences v0.9.5/go.mod h1:OdBm0n7C0Osh5yZB7j9BXyrMnTRGBJIZonUMxo5CzPw= +cloud.google.com/go/lifesciences v0.9.7 h1:qqEmApr5YFOQjkrU8Jy6o6QpkESqfGbfrE6bnUZZbV8= +cloud.google.com/go/lifesciences v0.9.7/go.mod h1:FQ713PhjAOHqUVnuwsCe1KPi9oAdaTfh58h1xPiW13g= cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= +cloud.google.com/go/logging v1.10.0 h1:f+ZXMqyrSJ5vZ5pE/zr0xC8y/M9BLNzQeLBwfeZ+wY4= +cloud.google.com/go/logging v1.10.0/go.mod h1:EHOwcxlltJrYGqMGfghSet736KR3hX1MAj614mrMk9I= cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/managedidentities v1.6.5 h1:+bpih1piZVLxla/XBqeSUzJBp8gv9plGHIMAI7DLpDM= cloud.google.com/go/managedidentities v1.6.5/go.mod h1:fkFI2PwwyRQbjLxlm5bQ8SjtObFMW3ChBGNqaMcgZjI= +cloud.google.com/go/managedidentities v1.6.7 h1:uWA9WQyfA0JdkeAFymWUsa3qE9tC33LUElla790Ou1A= +cloud.google.com/go/managedidentities v1.6.7/go.mod h1:UzslJgHnc6luoyx2JV19cTCi2Fni/7UtlcLeSYRzTV8= cloud.google.com/go/maps v1.6.4 h1:EVCZAiDvog9So46460BGbCasPhi613exoaQbpilMVlk= cloud.google.com/go/maps v1.6.4/go.mod h1:rhjqRy8NWmDJ53saCfsXQ0LKwBHfi6OSh5wkq6BaMhI= +cloud.google.com/go/maps v1.11.0 h1:+//LeUr5ARVau1wVsxLkLnFtYviq9YFS+fB/mhfAOGQ= +cloud.google.com/go/maps v1.11.0/go.mod h1:XcSsd8lg4ZhLPCtJ2YHcu/xLVePBzZOlI7GmR2cRCws= cloud.google.com/go/mediatranslation v0.8.5 h1:c76KdIXljQHSCb/Cy47S8H4s05A4zbK3pAFGzwcczZo= cloud.google.com/go/mediatranslation v0.8.5/go.mod h1:y7kTHYIPCIfgyLbKncgqouXJtLsU+26hZhHEEy80fSs= +cloud.google.com/go/mediatranslation v0.8.7 h1:izgww3TlyvWyDWdFKnrASpbh12IkAuw8o2ION8sAjX0= +cloud.google.com/go/mediatranslation v0.8.7/go.mod h1:6eJbPj1QJwiCP8R4K413qMx6ZHZJUi9QFpApqY88xWU= cloud.google.com/go/memcache v1.10.5 h1:yeDv5qxRedFosvpMSEswrqUsJM5OdWvssPHFliNFTc4= cloud.google.com/go/memcache v1.10.5/go.mod h1:/FcblbNd0FdMsx4natdj+2GWzTq+cjZvMa1I+9QsuMA= +cloud.google.com/go/memcache v1.10.7 h1:hE7f3ze3+eWh/EbYXEz7oXkm0LXcr7UCoLklwi7gsLU= +cloud.google.com/go/memcache v1.10.7/go.mod h1:SrU6+QBhvXJV0TA59+B3oCHtLkPx37eqdKmRUlmSE1k= cloud.google.com/go/metastore v1.13.4 h1:dR7vqWXlK6IYR8Wbu9mdFfwlVjodIBhd1JRrpZftTEg= cloud.google.com/go/metastore v1.13.4/go.mod h1:FMv9bvPInEfX9Ac1cVcRXp8EBBQnBcqH6gz3KvJ9BAE= +cloud.google.com/go/metastore v1.13.6 h1:otHcJkci5f/sNRedrSM7eM81QRnu0yZ3HvkvWGphABA= +cloud.google.com/go/metastore v1.13.6/go.mod h1:OBCVMCP7X9vA4KKD+5J4Q3d+tiyKxalQZnksQMq5MKY= cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg= +cloud.google.com/go/monitoring v1.19.0 h1:NCXf8hfQi+Kmr56QJezXRZ6GPb80ZI7El1XztyUuLQI= +cloud.google.com/go/monitoring v1.19.0/go.mod h1:25IeMR5cQ5BoZ8j1eogHE5VPJLlReQ7zFp5OiLgiGZw= cloud.google.com/go/networkconnectivity v1.14.4 h1:GBfXFhLyPspnaBE3nI/BRjdhW8vcbpT9QjE/4kDCDdc= cloud.google.com/go/networkconnectivity v1.14.4/go.mod h1:PU12q++/IMnDJAB+3r+tJtuCXCfwfN+C6Niyj6ji1Po= +cloud.google.com/go/networkconnectivity v1.14.6 h1:jYpQ86mZ7OYZc7WadvCIlIaPXmXhr5nD7wgE/ekMVpM= +cloud.google.com/go/networkconnectivity v1.14.6/go.mod h1:/azB7+oCSmyBs74Z26EogZ2N3UcXxdCHkCPcz8G32bU= cloud.google.com/go/networkmanagement v1.9.4 h1:aLV5GcosBNmd6M8+a0ekB0XlLRexv4fvnJJrYnqeBcg= cloud.google.com/go/networkmanagement v1.9.4/go.mod h1:daWJAl0KTFytFL7ar33I6R/oNBH8eEOX/rBNHrC/8TA= +cloud.google.com/go/networkmanagement v1.13.2 h1:Ex1/aYkA0areleSmOGXHvEFBGohteIYJr2SGPrjOUe0= +cloud.google.com/go/networkmanagement v1.13.2/go.mod h1:24VrV/5HFIOXMEtVQEUoB4m/w8UWvUPAYjfnYZcBc4c= cloud.google.com/go/networksecurity v0.9.5 h1:+caSxBTj0E8OYVh/5wElFdjEMO1S/rZtE1152Cepchc= cloud.google.com/go/networksecurity v0.9.5/go.mod h1:KNkjH/RsylSGyyZ8wXpue8xpCEK+bTtvof8SBfIhMG8= +cloud.google.com/go/networksecurity v0.9.7 h1:aepEkfiwOvUL9eu3ginVZhTaXDRHncQKi9lTT1BycH0= +cloud.google.com/go/networksecurity v0.9.7/go.mod h1:aB6UiPnh/l32+TRvgTeOxVRVAHAFFqvK+ll3idU5BoY= cloud.google.com/go/notebooks v1.11.3 h1:FH48boYmrWVQ6k0Mx/WrnNafXncT5iSYxA8CNyWTgy0= cloud.google.com/go/notebooks v1.11.3/go.mod h1:0wQyI2dQC3AZyQqWnRsp+yA+kY4gC7ZIVP4Qg3AQcgo= +cloud.google.com/go/notebooks v1.11.5 h1:sFU1ETg1HfIN/Tev8gD0dleAITLv7cHp0JClwFmJ6bo= +cloud.google.com/go/notebooks v1.11.5/go.mod h1:pz6P8l2TvhWqAW3sysIsS0g2IUJKOzEklsjWJfi8sd4= cloud.google.com/go/optimization v1.6.3 h1:63NZaWyN+5rZEKHPX4ACpw3BjgyeuY8+rCehiCMaGPY= cloud.google.com/go/optimization v1.6.3/go.mod h1:8ve3svp3W6NFcAEFr4SfJxrldzhUl4VMUJmhrqVKtYA= +cloud.google.com/go/optimization v1.6.5 h1:FPfowA/LEckKTQT0A4NJMI2bSou999c2ZyFX1zGiYxY= +cloud.google.com/go/optimization v1.6.5/go.mod h1:eiJjNge1NqqLYyY75AtIGeQWKO0cvzD1ct/moCFaP2Q= cloud.google.com/go/orchestration v1.8.5 h1:YHgWMlrPttIVGItgGfuvO2KM7x+y9ivN/Yk92pMm1a4= cloud.google.com/go/orchestration v1.8.5/go.mod h1:C1J7HesE96Ba8/hZ71ISTV2UAat0bwN+pi85ky38Yq8= +cloud.google.com/go/orchestration v1.9.2 h1:C2WL4ZnclXsh4XickGhKYKlPjqVZj35y1sbRjdsZ3g4= +cloud.google.com/go/orchestration v1.9.2/go.mod h1:8bGNigqCQb/O1kK7PeStSNlyi58rQvZqDiuXT9KAcbg= cloud.google.com/go/orgpolicy v1.12.1 h1:2JbXigqBJVp8Dx5dONUttFqewu4fP0p3pgOdIZAhpYU= cloud.google.com/go/orgpolicy v1.12.1/go.mod h1:aibX78RDl5pcK3jA8ysDQCFkVxLj3aOQqrbBaUL2V5I= +cloud.google.com/go/orgpolicy v1.12.3 h1:fGftW2bPi8vTjQm57xlwtLBZQcrgC+c3HMFBzJ+KWPc= +cloud.google.com/go/orgpolicy v1.12.3/go.mod h1:6BOgIgFjWfJzTsVcib/4QNHOAeOjCdaBj69aJVs//MA= cloud.google.com/go/osconfig v1.12.5 h1:Mo5jGAxOMKH/PmDY7fgY19yFcVbvwREb5D5zMPQjFfo= cloud.google.com/go/osconfig v1.12.5/go.mod h1:D9QFdxzfjgw3h/+ZaAb5NypM8bhOMqBzgmbhzWViiW8= +cloud.google.com/go/osconfig v1.12.7 h1:HXsXGFaFaLTklwKgSob/GSE+c3verYDQDgreFaosxyc= +cloud.google.com/go/osconfig v1.12.7/go.mod h1:ID7Lbqr0fiihKMwAOoPomWRqsZYKWxfiuafNZ9j1Y1M= cloud.google.com/go/oslogin v1.13.1 h1:1K4nOT5VEZNt7XkhaTXupBYos5HjzvJMfhvyD2wWdFs= cloud.google.com/go/oslogin v1.13.1/go.mod h1:vS8Sr/jR7QvPWpCjNqy6LYZr5Zs1e8ZGW/KPn9gmhws= +cloud.google.com/go/oslogin v1.13.3 h1:7AgOWH1oMPrB1AVU0/f47ADdOt+XfdBY7QRb8tcMUp8= +cloud.google.com/go/oslogin v1.13.3/go.mod h1:WW7Rs1OJQ1iSUckZDilvNBSNPE8on740zF+4ZDR4o8U= cloud.google.com/go/phishingprotection v0.8.5 h1:DH3WFLzEoJdW/6xgsmoDqOwT1xddFi7gKu0QGZQhpGU= cloud.google.com/go/phishingprotection v0.8.5/go.mod h1:g1smd68F7mF1hgQPuYn3z8HDbNre8L6Z0b7XMYFmX7I= +cloud.google.com/go/phishingprotection v0.8.7 h1:CbCjfR/pgDHyRMu94o9nuGwaONEcarWnUfSGGw+I2ZI= +cloud.google.com/go/phishingprotection v0.8.7/go.mod h1:FtYaOyGc/HQQU7wY4sfwYZBFDKAL+YtVBjUj8E3A3/I= cloud.google.com/go/policytroubleshooter v1.10.3 h1:c0WOzC6hz964QWNBkyKfna8A2jOIx1zzZa43Gx/P09o= cloud.google.com/go/policytroubleshooter v1.10.3/go.mod h1:+ZqG3agHT7WPb4EBIRqUv4OyIwRTZvsVDHZ8GlZaoxk= +cloud.google.com/go/policytroubleshooter v1.10.5 h1:LGt85MZUKlq9oqsbBL9+M6jAyeuR1TtCx6k5HfAQxTY= +cloud.google.com/go/policytroubleshooter v1.10.5/go.mod h1:bpOf94YxjWUqsVKokzPBibMSAx937Jp2UNGVoMAtGYI= cloud.google.com/go/privatecatalog v0.9.5 h1:UZ0assTnATXSggoxUIh61RjTQ4P9zCMk/kEMbn0nMYA= cloud.google.com/go/privatecatalog v0.9.5/go.mod h1:fVWeBOVe7uj2n3kWRGlUQqR/pOd450J9yZoOECcQqJk= +cloud.google.com/go/privatecatalog v0.9.7 h1:wGZKKJhYyuf4gcAEywQqQ6F19yxhBJGnzgyxOTbJjBw= +cloud.google.com/go/privatecatalog v0.9.7/go.mod h1:NWLa8MCL6NkRSt8jhL8Goy2A/oHkvkeAxiA0gv0rIXI= cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= cloud.google.com/go/pubsub v1.36.1 h1:dfEPuGCHGbWUhaMCTHUFjfroILEkx55iUmKBZTP5f+Y= cloud.google.com/go/pubsub v1.36.1/go.mod h1:iYjCa9EzWOoBiTdd4ps7QoMtMln5NwaZQpK1hbRfBDE= +cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc= +cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA= cloud.google.com/go/pubsublite v1.8.1 h1:pX+idpWMIH30/K7c0epN6V703xpIcMXWRjKJsz0tYGY= cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= +cloud.google.com/go/pubsublite v1.8.2 h1:jLQozsEVr+c6tOU13vDugtnaBSUy/PD5zK6mhm+uF1Y= +cloud.google.com/go/pubsublite v1.8.2/go.mod h1:4r8GSa9NznExjuLPEJlF1VjOPOpgf3IT6k8x/YgaOPI= cloud.google.com/go/recaptchaenterprise/v2 v2.9.2 h1:U3Wfq12X9cVMuTpsWDSURnXF0Z9hSPTHj+xsnXDRLsw= cloud.google.com/go/recaptchaenterprise/v2 v2.9.2/go.mod h1:trwwGkfhCmp05Ll5MSJPXY7yvnO0p4v3orGANAFHAuU= +cloud.google.com/go/recaptchaenterprise/v2 v2.13.0 h1:+QG02kE63W13vXI+rwAxFF3EhGX6K7gXwFz9OKwKcHw= +cloud.google.com/go/recaptchaenterprise/v2 v2.13.0/go.mod h1:jNYyn2ScR4DTg+VNhjhv/vJQdaU8qz+NpmpIzEE7HFQ= cloud.google.com/go/recommendationengine v0.8.5 h1:ineqLswaCSBY0csYv5/wuXJMBlxATK6Xc5jJkpiTEdM= cloud.google.com/go/recommendationengine v0.8.5/go.mod h1:A38rIXHGFvoPvmy6pZLozr0g59NRNREz4cx7F58HAsQ= +cloud.google.com/go/recommendationengine v0.8.7 h1:N6n/TEr0FQzeP4ZtvF5daMszOhdZI94uMiPiAi9kFMo= +cloud.google.com/go/recommendationengine v0.8.7/go.mod h1:YsUIbweUcpm46OzpVEsV5/z+kjuV6GzMxl7OAKIGgKE= cloud.google.com/go/recommender v1.12.1 h1:LVLYS3r3u0MSCxQSDUtLSkporEGi9OAE6hGvayrZNPs= cloud.google.com/go/recommender v1.12.1/go.mod h1:gf95SInWNND5aPas3yjwl0I572dtudMhMIG4ni8nr+0= +cloud.google.com/go/recommender v1.12.3 h1:v9x75vXP5wMXw3QG3xmgjVHLlqYufuLn/ht3oNWCA3w= +cloud.google.com/go/recommender v1.12.3/go.mod h1:OgN0MjV7/6FZUUPgF2QPQtYErtZdZc4u+5onvurcGEI= cloud.google.com/go/redis v1.14.2 h1:QF0maEdVv0Fj/2roU8sX3NpiDBzP9ICYTO+5F32gQNo= cloud.google.com/go/redis v1.14.2/go.mod h1:g0Lu7RRRz46ENdFKQ2EcQZBAJ2PtJHJLuiiRuEXwyQw= +cloud.google.com/go/redis v1.16.0 h1:1veL/h/x5bgzG2CLK2cdG3plWdgO0p1qoHgwFBqG7+c= +cloud.google.com/go/redis v1.16.0/go.mod h1:NLzG3Ur8ykVIZk+i5ienRnycsvWzQ0uCLcil6Htc544= cloud.google.com/go/resourcemanager v1.9.5 h1:AZWr1vWVDKGwfLsVhcN+vcwOz3xqqYxtmMa0aABCMms= cloud.google.com/go/resourcemanager v1.9.5/go.mod h1:hep6KjelHA+ToEjOfO3garMKi/CLYwTqeAw7YiEI9x8= +cloud.google.com/go/resourcemanager v1.9.7 h1:SdvD0PaPX60+yeKoSe16mawFpM0EPuiPPihTIVlhRsY= +cloud.google.com/go/resourcemanager v1.9.7/go.mod h1:cQH6lJwESufxEu6KepsoNAsjrUtYYNXRwxm4QFE5g8A= cloud.google.com/go/resourcesettings v1.6.5 h1:BTr5MVykJwClASci/7Og4Qfx70aQ4n3epsNLj94ZYgw= cloud.google.com/go/resourcesettings v1.6.5/go.mod h1:WBOIWZraXZOGAgoR4ukNj0o0HiSMO62H9RpFi9WjP9I= +cloud.google.com/go/resourcesettings v1.7.0 h1:yEuByg5XBHhTG9wPEU7GiEtC9Orp1wSEyiiX4IPqoSY= +cloud.google.com/go/resourcesettings v1.7.0/go.mod h1:pFzZYOQMyf1hco9pbNWGEms6N/2E7nwh0oVU1Tz+4qA= cloud.google.com/go/retail v1.16.0 h1:Fn1GuAua1c6crCGqfJ1qMxG1Xh10Tg/x5EUODEHMqkw= cloud.google.com/go/retail v1.16.0/go.mod h1:LW7tllVveZo4ReWt68VnldZFWJRzsh9np+01J9dYWzE= +cloud.google.com/go/retail v1.16.2 h1:msP5a8BOxVym2DvoubeWAxAeV6VhYkKnYHc2XOkd/+U= +cloud.google.com/go/retail v1.16.2/go.mod h1:T7UcBh4/eoxRBpP3vwZCoa+PYA9/qWRTmOCsV8DRdZ0= cloud.google.com/go/run v1.3.4 h1:m9WDA7DzTpczhZggwYlZcBWgCRb+kgSIisWn1sbw2rQ= cloud.google.com/go/run v1.3.4/go.mod h1:FGieuZvQ3tj1e9GnzXqrMABSuir38AJg5xhiYq+SF3o= +cloud.google.com/go/run v1.3.7 h1:E4Z5e681Qh7UJrJRMCgYhp+3tkcoXiaKGh3UZmUPaAQ= +cloud.google.com/go/run v1.3.7/go.mod h1:iEUflDx4Js+wK0NzF5o7hE9Dj7QqJKnRj0/b6rhVq20= cloud.google.com/go/scheduler v1.10.6 h1:5U8iXLoQ03qOB+ZXlAecU7fiE33+u3QiM9nh4cd0eTE= cloud.google.com/go/scheduler v1.10.6/go.mod h1:pe2pNCtJ+R01E06XCDOJs1XvAMbv28ZsQEbqknxGOuE= +cloud.google.com/go/scheduler v1.10.8 h1:Jn/unfNUgRiNJRc1nrApzimKiVj91UYlLT8mMfpUu48= +cloud.google.com/go/scheduler v1.10.8/go.mod h1:0YXHjROF1f5qTMvGTm4o7GH1PGAcmu/H/7J7cHOiHl0= cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= +cloud.google.com/go/secretmanager v1.13.1 h1:TTGo2Vz7ZxYn2QbmuFP7Zo4lDm5VsbzBjDReo3SA5h4= +cloud.google.com/go/secretmanager v1.13.1/go.mod h1:y9Ioh7EHp1aqEKGYXk3BOC+vkhlHm9ujL7bURT4oI/4= cloud.google.com/go/security v1.15.5 h1:wTKJQ10j8EYgvE8Y+KhovxDRVDk2iv/OsxZ6GrLP3kE= cloud.google.com/go/security v1.15.5/go.mod h1:KS6X2eG3ynWjqcIX976fuToN5juVkF6Ra6c7MPnldtc= +cloud.google.com/go/security v1.17.0 h1:u4RCnEQPvlrrnFRFinU0T3WsjtrsQErkWBfqTM5oUQI= +cloud.google.com/go/security v1.17.0/go.mod h1:eSuFs0SlBv1gWg7gHIoF0hYOvcSwJCek/GFXtgO6aA0= cloud.google.com/go/securitycenter v1.24.4 h1:/5jjkZ+uGe8hZ7pvd7pO30VW/a+pT2MrrdgOqjyucKQ= cloud.google.com/go/securitycenter v1.24.4/go.mod h1:PSccin+o1EMYKcFQzz9HMMnZ2r9+7jbc+LvPjXhpwcU= +cloud.google.com/go/securitycenter v1.30.0 h1:Y8C0I/mzLbaxAl5cw3EaLox0Rvpy+VUwEuCGWIQDMU8= +cloud.google.com/go/securitycenter v1.30.0/go.mod h1:/tmosjS/dfTnzJxOzZhTXdX3MXWsCmPWfcYOgkJmaJk= cloud.google.com/go/servicedirectory v1.11.4 h1:da7HFI1229kyzIyuVEzHXip0cw0d+E0s8mjQby0WN+k= cloud.google.com/go/servicedirectory v1.11.4/go.mod h1:Bz2T9t+/Ehg6x+Y7Ycq5xiShYLD96NfEsWNHyitj1qM= +cloud.google.com/go/servicedirectory v1.11.7 h1:c3OAhTcZ8LbIiKps5T3p6i0QcPI8/aWYwOfoZobICKo= +cloud.google.com/go/servicedirectory v1.11.7/go.mod h1:fiO/tM0jBpVhpCAe7Yp5HmEsmxSUcOoc4vPrO02v68I= cloud.google.com/go/shell v1.7.5 h1:3Fq2hzO0ZSyaqBboJrFkwwf/qMufDtqwwA6ep8EZxEI= cloud.google.com/go/shell v1.7.5/go.mod h1:hL2++7F47/IfpfTO53KYf1EC+F56k3ThfNEXd4zcuiE= +cloud.google.com/go/shell v1.7.7 h1:HxCzcUxSsCh6FJWkmbOUrGI1sKe4E1Yy4vaykn4RhJ4= +cloud.google.com/go/shell v1.7.7/go.mod h1:7OYaMm3TFMSZBh8+QYw6Qef+fdklp7CjjpxYAoJpZbQ= cloud.google.com/go/spanner v1.56.0 h1:o/Cv7/zZ1WgRXVCd5g3Nc23ZI39p/1pWFqFwvg6Wcu8= cloud.google.com/go/spanner v1.56.0/go.mod h1:DndqtUKQAt3VLuV2Le+9Y3WTnq5cNKrnLb/Piqcj+h0= +cloud.google.com/go/spanner v1.63.0 h1:P6+BY70Wtol4MtryBgnXZVTZfsdySEvWfz0EpyLwHi4= +cloud.google.com/go/spanner v1.63.0/go.mod h1:iqDx7urZpgD7RekZ+CFvBRH6kVTW1ZSEb2HMDKOp5Cc= cloud.google.com/go/speech v1.21.1 h1:nuFc+Kj5B8de75nN4FdPyUbI2SiBoHZG6BLurXL56Q0= cloud.google.com/go/speech v1.21.1/go.mod h1:E5GHZXYQlkqWQwY5xRSLHw2ci5NMQNG52FfMU1aZrIA= +cloud.google.com/go/speech v1.23.1 h1:TcWEAOLQH1Lb2fhHS6/GjvAh+ue0dt4xUDHXHG6vF04= +cloud.google.com/go/speech v1.23.1/go.mod h1:UNgzNxhNBuo/OxpF1rMhA/U2rdai7ILL6PBXFs70wq0= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= +cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= +cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= +cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= cloud.google.com/go/storagetransfer v1.10.4 h1:dy4fL3wO0VABvzM05ycMUPFHxTPbJz9Em8ikAJVqSbI= cloud.google.com/go/storagetransfer v1.10.4/go.mod h1:vef30rZKu5HSEf/x1tK3WfWrL0XVoUQN/EPDRGPzjZs= +cloud.google.com/go/storagetransfer v1.10.6 h1:CXmoNEvz7y2NtHFZuH3Z8ASN43rxRINWa2Q/IlBzM2k= +cloud.google.com/go/storagetransfer v1.10.6/go.mod h1:3sAgY1bx1TpIzfSzdvNGHrGYldeCTyGI/Rzk6Lc6A7w= cloud.google.com/go/talent v1.6.6 h1:JssV0CE3FNujuSWn7SkosOzg7qrMxVnt6txOfGcMSa4= cloud.google.com/go/talent v1.6.6/go.mod h1:y/WQDKrhVz12WagoarpAIyKKMeKGKHWPoReZ0g8tseQ= +cloud.google.com/go/talent v1.6.8 h1:RoyEtftfJrbwJcu63zuWE4IjC76xMyVsJBhmleIp3bE= +cloud.google.com/go/talent v1.6.8/go.mod h1:kqPAJvhxmhoUTuqxjjk2KqA8zUEeTDmH+qKztVubGlQ= cloud.google.com/go/texttospeech v1.7.5 h1:dxY2Q5mHCbrGa3oPR2O3PCicdnvKa1JmwGQK36EFLOw= cloud.google.com/go/texttospeech v1.7.5/go.mod h1:tzpCuNWPwrNJnEa4Pu5taALuZL4QRRLcb+K9pbhXT6M= +cloud.google.com/go/texttospeech v1.7.7 h1:qR6Mu+EM2OfaZR1/Rl8BDBTVfi2X5OtwKKvJRSQyG+o= +cloud.google.com/go/texttospeech v1.7.7/go.mod h1:XO4Wr2VzWHjzQpMe3gS58Oj68nmtXMyuuH+4t0wy9eA= cloud.google.com/go/tpu v1.6.5 h1:C8YyYda8WtNdBoCgFwwBzZd+S6+EScHOxM/z1h0NNp8= cloud.google.com/go/tpu v1.6.5/go.mod h1:P9DFOEBIBhuEcZhXi+wPoVy/cji+0ICFi4TtTkMHSSs= +cloud.google.com/go/tpu v1.6.7 h1:ngQokxUB1z2gvHn3vAf04m7SFnNYMiQIIpny81fCGAs= +cloud.google.com/go/tpu v1.6.7/go.mod h1:o8qxg7/Jgt7TCgZc3jNkd4kTsDwuYD3c4JTMqXZ36hU= cloud.google.com/go/trace v1.10.5 h1:0pr4lIKJ5XZFYD9GtxXEWr0KkVeigc3wlGpZco0X1oA= cloud.google.com/go/trace v1.10.5/go.mod h1:9hjCV1nGBCtXbAE4YK7OqJ8pmPYSxPA0I67JwRd5s3M= +cloud.google.com/go/trace v1.10.7 h1:gK8z2BIJQ3KIYGddw9RJLne5Fx0FEXkrEQzPaeEYVvk= +cloud.google.com/go/trace v1.10.7/go.mod h1:qk3eiKmZX0ar2dzIJN/3QhY2PIFh1eqcIdaN5uEjQPM= cloud.google.com/go/translate v1.10.1 h1:upovZ0wRMdzZvXnu+RPam41B0mRJ+coRXFP2cYFJ7ew= cloud.google.com/go/translate v1.10.1/go.mod h1:adGZcQNom/3ogU65N9UXHOnnSvjPwA/jKQUMnsYXOyk= +cloud.google.com/go/translate v1.10.3 h1:g+B29z4gtRGsiKDoTF+bNeH25bLRokAaElygX2FcZkE= +cloud.google.com/go/translate v1.10.3/go.mod h1:GW0vC1qvPtd3pgtypCv4k4U8B7EdgK9/QEF2aJEUovs= cloud.google.com/go/video v1.20.4 h1:TXwotxkShP1OqgKsbd+b8N5hrIHavSyLGvYnLGCZ7xc= cloud.google.com/go/video v1.20.4/go.mod h1:LyUVjyW+Bwj7dh3UJnUGZfyqjEto9DnrvTe1f/+QrW0= +cloud.google.com/go/video v1.21.0 h1:ue/1C8TF8H2TMzKMBdNnFxT7QaeWMtqfDr9TSQGgUhA= +cloud.google.com/go/video v1.21.0/go.mod h1:Kqh97xHXZ/bIClgDHf5zkKvU3cvYnLyRefmC8yCBqKI= cloud.google.com/go/videointelligence v1.11.5 h1:mYaWH8uhUCXLJCN3gdXswKzRa2+lK0zN6/KsIubm6pE= cloud.google.com/go/videointelligence v1.11.5/go.mod h1:/PkeQjpRponmOerPeJxNPuxvi12HlW7Em0lJO14FC3I= +cloud.google.com/go/videointelligence v1.11.7 h1:SKBkFTuOclESLjQL1LwraqVFm2fL5oL9tbzKITU+FOY= +cloud.google.com/go/videointelligence v1.11.7/go.mod h1:iMCXbfjurmBVgKuyLedTzv90kcnppOJ6ttb0+rLDID0= cloud.google.com/go/vision/v2 v2.8.0 h1:W52z1b6LdGI66MVhE70g/NFty9zCYYcjdKuycqmlhtg= cloud.google.com/go/vision/v2 v2.8.0/go.mod h1:ocqDiA2j97pvgogdyhoxiQp2ZkDCyr0HWpicywGGRhU= +cloud.google.com/go/vision/v2 v2.8.2 h1:j9RxG8DcyJO/D7/ps2pOey8VZys+TMqF79bWAhuM7QU= +cloud.google.com/go/vision/v2 v2.8.2/go.mod h1:BHZA1LC7dcHjSr9U9OVhxMtLKd5l2jKPzLRALEJvuaw= cloud.google.com/go/vmmigration v1.7.5 h1:5v9RT2vWyuw3pK2ox0HQpkoftO7Q7/8591dTxxQc79g= cloud.google.com/go/vmmigration v1.7.5/go.mod h1:pkvO6huVnVWzkFioxSghZxIGcsstDvYiVCxQ9ZH3eYI= +cloud.google.com/go/vmmigration v1.7.7 h1:bf2qKqEN7iqT62IptQ/FDadoDLJI9sthyrW3PVaH8bY= +cloud.google.com/go/vmmigration v1.7.7/go.mod h1:qYIK5caZY3IDMXQK+A09dy81QU8qBW0/JDTc39OaKRw= cloud.google.com/go/vmwareengine v1.1.1 h1:EGdDi9QbqThfZq3ILcDK5g+m9jTevc34AY5tACx5v7k= cloud.google.com/go/vmwareengine v1.1.1/go.mod h1:nMpdsIVkUrSaX8UvmnBhzVzG7PPvNYc5BszcvIVudYs= +cloud.google.com/go/vmwareengine v1.1.3 h1:x4KwHB4JlBEzMaITVhrbbpHrU+2I5LrlvHGEEluT0vc= +cloud.google.com/go/vmwareengine v1.1.3/go.mod h1:UoyF6LTdrIJRvDN8uUB8d0yimP5A5Ehkr1SRzL1APZw= cloud.google.com/go/vpcaccess v1.7.5 h1:XyL6hTLtEM/eE4F1GEge8xUN9ZCkiVWn44K/YA7z1rQ= cloud.google.com/go/vpcaccess v1.7.5/go.mod h1:slc5ZRvvjP78c2dnL7m4l4R9GwL3wDLcpIWz6P/ziig= +cloud.google.com/go/vpcaccess v1.7.7 h1:F5woMLufKnshmDvPVxCzoC+Di12RYXQ1W8kNmpBT8z0= +cloud.google.com/go/vpcaccess v1.7.7/go.mod h1:EzfSlgkoAnFWEMznZW0dVNvdjFjEW97vFlKk4VNBhwY= cloud.google.com/go/webrisk v1.9.5 h1:251MvGuC8wisNN7+jqu9DDDZAi38KiMXxOpA/EWy4dE= cloud.google.com/go/webrisk v1.9.5/go.mod h1:aako0Fzep1Q714cPEM5E+mtYX8/jsfegAuS8aivxy3U= +cloud.google.com/go/webrisk v1.9.7 h1:EWTSVagWWeQjVAsebiF/wJMwC5bq6Zz3LqOmD9Uid4s= +cloud.google.com/go/webrisk v1.9.7/go.mod h1:7FkQtqcKLeNwXCdhthdXHIQNcFWPF/OubrlyRcLHNuQ= cloud.google.com/go/websecurityscanner v1.6.5 h1:YqWZrZYabG88TZt7364XWRJGhxmxhony2ZUyZEYMF2k= cloud.google.com/go/websecurityscanner v1.6.5/go.mod h1:QR+DWaxAz2pWooylsBF854/Ijvuoa3FCyS1zBa1rAVQ= +cloud.google.com/go/websecurityscanner v1.6.7 h1:R5OW5SNRqD0DSEmyWLUMNYAXWYnz/NLSXBawVFrc9a0= +cloud.google.com/go/websecurityscanner v1.6.7/go.mod h1:EpiW84G5KXxsjtFKK7fSMQNt8JcuLA8tQp7j0cyV458= cloud.google.com/go/workflows v1.12.4 h1:uHNmUiatTbPQ4H1pabwfzpfEYD4BBnqDHqMm2IesOh4= cloud.google.com/go/workflows v1.12.4/go.mod h1:yQ7HUqOkdJK4duVtMeBCAOPiN1ZF1E9pAMX51vpwB/w= +cloud.google.com/go/workflows v1.12.6 h1:2bE69mh68law1UZWPjgmvOQsjsGSppRudABAXwNAy58= +cloud.google.com/go/workflows v1.12.6/go.mod h1:oDbEHKa4otYg4abwdw2Z094jB0TLLiFGAPA78EDAKag= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= +github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apex/logs v1.0.0 h1:adOwhOTeXzZTnVuEK13wuJNBFutP0sOfutRS8NY+G6A= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a h1:2KLQMJ8msqoPHIPDufkxVcoTtcmE5+1sL9950m4R9Pk= github.com/aphistic/sweet v0.2.0 h1:I4z+fAUqvKfvZV/CHi5dV0QuwbmIvYYFDjG0Ss5QpAs= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4= github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= @@ -304,6 +561,8 @@ github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= @@ -314,6 +573,7 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fasthttp/websocket v1.4.3-rc.6 h1:omHqsl8j+KXpmzRjF8bmzOSYJ8GnS0E3efi1wYT+niY= github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CACNqNvCqQCoty/Lc= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -342,6 +602,8 @@ github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -352,11 +614,14 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= @@ -368,31 +633,60 @@ github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qK github.com/googleapis/gax-go/v2 v2.12.1/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 h1:zC34cGQu69FG7qzJ3WiKW244WfhDC3xxYMeNOX2gtUQ= +github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/consul/api v1.11.0 h1:Hw/G8TtRvOElqxVIhBzXciiSTbapq8hZ2XKZsXk5ZCE= +github.com/hashicorp/consul/api v1.28.2 h1:mXfkRHrpHN4YY3RqL09nXU1eHKLNiuAN4kHvDQ16k/8= +github.com/hashicorp/consul/api v1.28.2/go.mod h1:KyzqzgMEya+IZPcD65YFoOVAgPpbfERu4I/tzG6/ueE= github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= +github.com/hashicorp/consul/sdk v0.16.0 h1:SE9m0W6DEfgIVCJX7xU+iv/hUl4m/nxqMTnCdMxDpJ8= +github.com/hashicorp/consul/sdk v0.16.0/go.mod h1:7pxqqhqoaPqnBnzXD1StKed62LqJeClzVsUEy85Zr0A= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ= github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj34fMA= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -431,6 +725,12 @@ github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWe github.com/mmcloughlin/avo v0.5.0 h1:nAco9/aI9Lg2kiuROBY6BhCI/z0t5jEvJfjWbL8qXLU= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= +github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= +github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= @@ -442,13 +742,21 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgc github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= +github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/prometheus/client_golang v1.4.0 h1:YVIb/fVcOTMSqtqZWSKnHpSLBxu8DKgxq8z6RuBZwqI= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= @@ -459,6 +767,8 @@ github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/sagikazarmark/crypt v0.3.0 h1:TV5DVog+pihN4Rr0rN1IClv4ePpkzdg9sPrw7WDofZ8= +github.com/sagikazarmark/crypt v0.19.0 h1:WMyLTjHBo64UvNcWqpzY3pbZTYgnemZU8FBZigKc42E= +github.com/sagikazarmark/crypt v0.19.0/go.mod h1:c6vimRziqqERhtSe0MhIvzE1w54FrCHtrXb5NH/ja78= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873 h1:N3Af8f13ooDKcIhsmFT7Z05CStZWu4C7Md0uDEy4q6o= github.com/savsgio/gotils v0.0.0-20210617111740-97865ed5a873/go.mod h1:dmPawKuiAeG/aFYVs2i+Dyosoo7FNcm+Pi8iK6ZUrX8= @@ -469,6 +779,8 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CY github.com/smartystreets/gunit v1.0.0 h1:RyPDUFcJbvtXlhJPk7v+wnxZRY2EUokhEYl2EJOPToI= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE= github.com/tj/go-buffer v1.1.0 h1:Lo2OsPHlIxXF24zApe15AbK3bJLAOvkkxEA6Ux4c47M= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2 h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc= @@ -485,10 +797,20 @@ github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaD github.com/zenazn/goji v0.9.0 h1:RSQQAbXGArQ0dIDEq+PI6WqN6if+5KHu6x2Cx/GXLTQ= go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M= +go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI= +go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/etcd/api/v3 v3.5.1 h1:v28cktvBq+7vGyJXF8G+rWJmj+1XUmMtqcLnH8hDocM= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= go.etcd.io/etcd/client/pkg/v3 v3.5.1 h1:XIQcHCFSG53bJETYeRJtIxdLv2EWRGxcfzR8lSnTH4E= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= go.etcd.io/etcd/client/v2 v2.305.1 h1:vtxYCKWA9x31w0WJj7DdqsHFNjhkigdAnziDtkZb/l4= +go.etcd.io/etcd/client/v2 v2.305.12 h1:0m4ovXYo1CHaA/Mp3X/Fak5sRNIWf01wk/X1/G3sGKI= +go.etcd.io/etcd/client/v2 v2.305.12/go.mod h1:aQ/yhsxMu+Oht1FOupSr60oBvcS9cKXHrzBpDsPTf9E= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -515,6 +837,8 @@ go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8p go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= @@ -524,41 +848,68 @@ go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzc go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -574,44 +925,78 @@ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6f google.golang.org/api v0.62.0 h1:PhGymJMXfGBzc4lBRmrx9+1w4w2wEzURHNGF/sD/xGc= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.152.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/api v0.160.0/go.mod h1:0mu0TpK33qnydLvWqbImq2b1eQ5FHRSDCBzAxX9ZHyw= google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= google.golang.org/api v0.164.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeqUB5o= google.golang.org/api v0.166.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA= +google.golang.org/api v0.167.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA= google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/api v0.171.0 h1:w174hnBPqut76FzW5Qaupt7zY8Kql6fiVjgys4f58sU= +google.golang.org/api v0.171.0/go.mod h1:Hnq5AHm4OTMt2BUVjael2CWZFD6vksJdWCWiUAmjC9o= +google.golang.org/api v0.176.1/go.mod h1:j2MaSDYcvYV1lkZ1+SMW4IeF90SrEyFA+tluDYWRrFg= +google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOnFQDw= +google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= +google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= +google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4stdSSsxPDK8Azkrk/ECl5HvdPf6nbZrTS5M= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= +google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78 h1:YqFWYZXim8bG9v68xU8WjTZmYKb5M5dMeSOWIp6jogI= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:vh/N7795ftP0AkN1w8XKqN4w1OdUKXW5Eummda+ofv8= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240314234333-6e1732d8331c h1:4z0DVWmDWWZ4OeQHLrb6lLBE3uCgSLs9DDA5Zb36DFg= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240314234333-6e1732d8331c/go.mod h1:IN9OQUXZ0xT+26MDwZL8fJcYw+y99b0eYPA2U15Jt8o= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240521202816-d264139d666e h1:Px+x8PNp8izq1ORW6jI007V/fRZ3bWrgcWHImtBduXc= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240521202816-d264139d666e/go.mod h1:0J6mmn3XAEjfNbPvpH63c0RXCjGNFcCzlEfWSN4In+k= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014/go.mod h1:SaPjaZGWb0lPqs6Ittu0spdfrOArqji4ZdeP5IC/9N4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= @@ -624,11 +1009,14 @@ google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFL google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= @@ -652,3 +1040,5 @@ rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/grpc/go.mod b/grpc/go.mod index bbc0968c8..0fecf6223 100644 --- a/grpc/go.mod +++ b/grpc/go.mod @@ -5,12 +5,12 @@ go 1.22 require ( github.com/golang/protobuf v1.5.4 google.golang.org/grpc v1.64.0 - google.golang.org/protobuf v1.34.1 + google.golang.org/protobuf v1.34.2 ) require ( - golang.org/x/net v0.22.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect ) diff --git a/grpc/go.sum b/grpc/go.sum index d41d2dc22..4f6247a6b 100644 --- a/grpc/go.sum +++ b/grpc/go.sum @@ -2,15 +2,15 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/template-parser/go.mod b/template-parser/go.mod index 7af02e620..f80482bfb 100644 --- a/template-parser/go.mod +++ b/template-parser/go.mod @@ -36,10 +36,10 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/template-parser/go.sum b/template-parser/go.sum index e7103492b..261a3d6d8 100644 --- a/template-parser/go.sum +++ b/template-parser/go.sum @@ -131,7 +131,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -171,7 +171,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -182,11 +182,11 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/vcs/go.mod b/vcs/go.mod index 56d2c2513..3a09c2ec8 100644 --- a/vcs/go.mod +++ b/vcs/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 github.com/stretchr/testify v1.9.0 - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.23.0 ) require ( @@ -28,11 +28,11 @@ require ( github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/tools v0.20.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/vcs/go.sum b/vcs/go.sum index 07feb5ec3..4dacc7235 100644 --- a/vcs/go.sum +++ b/vcs/go.sum @@ -95,10 +95,10 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -107,12 +107,12 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -128,12 +128,12 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -141,12 +141,12 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From de987f7f12c9c0e10162cbcd614124c68485e2b4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 17:47:30 +0800 Subject: [PATCH 003/106] fix: dependency issue --- backend/go.mod | 34 +++++---- backend/go.sum | 90 +++++++++++++--------- core/go.mod | 15 ++-- core/go.sum | 4 +- db/go.mod | 10 +-- fs/go.mod | 5 +- fs/go.sum | 4 +- go.work.sum | 27 +++++-- grpc/go.mod | 2 +- grpc/go.sum | 3 +- template-parser/go.mod | 33 ++++---- template-parser/go.sum | 167 +++++------------------------------------ vcs/go.mod | 4 +- vcs/go.sum | 4 +- 14 files changed, 159 insertions(+), 243 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 27fa3b36f..a5b47a32c 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -12,7 +12,7 @@ replace ( github.com/crawlab-team/crawlab/vcs => ../vcs ) -require github.com/crawlab-team/crawlab/core v0.6.3-0.20231031044528-37e6d73eb203 +require github.com/crawlab-team/crawlab/core v0.0.0-20240614093256-0834d6771b27 require ( dario.cat/mergo v1.0.0 // indirect @@ -48,7 +48,7 @@ require ( github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.7.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect @@ -68,7 +68,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect - github.com/hashicorp/go-uuid v1.0.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -86,42 +86,43 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect github.com/linxGnu/gumble v1.0.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/matcornic/hermes/v2 v2.1.0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/olekukonko/tablewriter v0.0.1 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/segmentio/kafka-go v0.4.39 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/skeema/knownhosts v1.2.2 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.10.0 // indirect + github.com/spf13/viper v1.19.0 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/thoas/go-funk v0.9.1 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect @@ -138,9 +139,13 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.mongodb.org/mongo-driver v1.15.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.10.0 // indirect + go.uber.org/goleak v1.1.11 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sync v0.7.0 // indirect @@ -152,9 +157,8 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect - gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index b192a3caf..63fb69cd8 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -3,7 +3,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -16,7 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -36,6 +34,7 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -48,7 +47,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -154,8 +152,9 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -185,9 +184,12 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= @@ -312,7 +314,6 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -328,7 +329,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -356,8 +356,9 @@ github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR3 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -455,8 +456,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -484,8 +485,9 @@ github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -525,8 +527,9 @@ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTS github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -545,10 +548,9 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -559,9 +561,9 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -595,6 +597,10 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -628,21 +634,24 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -662,11 +671,12 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -751,14 +761,19 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -781,17 +796,16 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -803,6 +817,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -833,6 +849,7 @@ golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -870,7 +887,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -886,6 +902,7 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -904,6 +921,7 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -918,6 +936,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -968,7 +987,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -976,7 +994,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1001,6 +1018,7 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1008,6 +1026,7 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1022,6 +1041,7 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1079,7 +1099,6 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1089,6 +1108,7 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1169,9 +1189,7 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1200,6 +1218,7 @@ google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1244,6 +1263,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1257,8 +1277,9 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= @@ -1272,7 +1293,6 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/core/go.mod b/core/go.mod index 4034912d3..0474d4951 100644 --- a/core/go.mod +++ b/core/go.mod @@ -15,12 +15,12 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/db v0.6.0-1 - github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/template-parser v0.0.0-00010101000000-000000000000 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614083850-ff2830bd7994 - github.com/crawlab-team/crawlab/vcs v0.1.1 + github.com/crawlab-team/crawlab/db v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/fs v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614093256-0834d6771b27 github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 @@ -30,7 +30,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 - github.com/hashicorp/go-uuid v1.0.1 + github.com/hashicorp/go-uuid v1.0.3 github.com/imroc/req v0.3.0 github.com/matcornic/hermes/v2 v2.1.0 github.com/mitchellh/go-homedir v1.1.0 @@ -171,6 +171,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.uber.org/atomic v1.9.0 // indirect + go.uber.org/goleak v1.1.11 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.23.0 // indirect diff --git a/core/go.sum b/core/go.sum index 8828b1167..6d1aae6d0 100644 --- a/core/go.sum +++ b/core/go.sum @@ -359,8 +359,8 @@ github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR3 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -783,8 +783,8 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= diff --git a/db/go.mod b/db/go.mod index 987d9c4d6..a25cc3911 100644 --- a/db/go.mod +++ b/db/go.mod @@ -7,7 +7,7 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/trace v0.0.0-00010101000000-000000000000 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614093256-0834d6771b27 github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 github.com/spf13/viper v1.19.0 @@ -46,11 +46,11 @@ require ( github.com/ztrue/tracerr v0.4.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/fs/go.mod b/fs/go.mod index db1decd27..e6ea7ce2f 100644 --- a/fs/go.mod +++ b/fs/go.mod @@ -8,17 +8,18 @@ require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614093256-0834d6771b27 github.com/emirpasic/gods v1.18.1 github.com/google/uuid v1.6.0 github.com/stretchr/testify v1.9.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/kr/pretty v0.3.1 // indirect github.com/linxGnu/gumble v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/fs/go.sum b/fs/go.sum index 957e00efe..4a03f0778 100644 --- a/fs/go.sum +++ b/fs/go.sum @@ -10,8 +10,8 @@ github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -42,8 +42,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= diff --git a/go.work.sum b/go.work.sum index d8bd2dc88..9c9482bac 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,3 +1,5 @@ +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= @@ -248,6 +250,7 @@ cloud.google.com/go/gsuiteaddons v1.6.5 h1:CZEbaBwmbYdhFw21Fwbo+C35HMe36fTE0FBSR cloud.google.com/go/gsuiteaddons v1.6.5/go.mod h1:Lo4P2IvO8uZ9W+RaC6s1JVxo42vgy+TX5a6hfBZ0ubs= cloud.google.com/go/gsuiteaddons v1.6.7 h1:06Jg3JeLslEfBYX1sDqOPLnF7a3wmhNcDUXF/fVOb50= cloud.google.com/go/gsuiteaddons v1.6.7/go.mod h1:u+sGBvr07OKNnOnQiB/Co1q4U2cjo50ERQwvnlcpNis= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= @@ -436,6 +439,8 @@ cloud.google.com/go/speech v1.23.1 h1:TcWEAOLQH1Lb2fhHS6/GjvAh+ue0dt4xUDHXHG6vF0 cloud.google.com/go/speech v1.23.1/go.mod h1:UNgzNxhNBuo/OxpF1rMhA/U2rdai7ILL6PBXFs70wq0= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= @@ -616,6 +621,7 @@ github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdf github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= @@ -638,6 +644,7 @@ github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBH github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 h1:zC34cGQu69FG7qzJ3WiKW244WfhDC3xxYMeNOX2gtUQ= github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -673,7 +680,6 @@ github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sL github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -704,10 +710,10 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/lyft/protoc-gen-star v0.5.3 h1:zSGLzsUew8RT+ZKPHc3jnf8XLaVyHzTcAFBzHtCNR20= github.com/lyft/protoc-gen-star/v2 v2.0.3 h1:/3+/2sWyXeMLzKd1bX+ixWKgEMsULrIivpDsuaF441o= @@ -737,11 +743,13 @@ github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= @@ -779,6 +787,8 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CY github.com/smartystreets/gunit v1.0.0 h1:RyPDUFcJbvtXlhJPk7v+wnxZRY2EUokhEYl2EJOPToI= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE= @@ -848,14 +858,15 @@ go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzc go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= @@ -872,6 +883,7 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDA golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= @@ -894,7 +906,8 @@ golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -917,8 +930,10 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqG golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= @@ -943,6 +958,8 @@ google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= @@ -990,7 +1007,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= @@ -1021,6 +1037,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQ gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= modernc.org/b v1.0.2 h1:iPC2u39ebzq12GOC2yXT4mve0HrWcH85cz+midWjzeo= modernc.org/db v1.0.3 h1:apxOlWU69je04bY22OT6J0RL23mzvUy22EgTAVyw+Yg= diff --git a/grpc/go.mod b/grpc/go.mod index 0fecf6223..c9c005c9a 100644 --- a/grpc/go.mod +++ b/grpc/go.mod @@ -12,5 +12,5 @@ require ( golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect ) diff --git a/grpc/go.sum b/grpc/go.sum index 4f6247a6b..664480a81 100644 --- a/grpc/go.sum +++ b/grpc/go.sum @@ -8,8 +8,7 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= diff --git a/template-parser/go.mod b/template-parser/go.mod index f80482bfb..259e07f35 100644 --- a/template-parser/go.mod +++ b/template-parser/go.mod @@ -14,34 +14,37 @@ require ( require ( github.com/apex/log v1.9.0 // indirect github.com/cenkalti/backoff/v4 v4.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.10.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/spf13/viper v1.19.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/template-parser/go.sum b/template-parser/go.sum index 261a3d6d8..537225e29 100644 --- a/template-parser/go.sum +++ b/template-parser/go.sum @@ -1,17 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= @@ -21,95 +7,68 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -124,126 +83,38 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/vcs/go.mod b/vcs/go.mod index 3a09c2ec8..5f5060e6e 100644 --- a/vcs/go.mod +++ b/vcs/go.mod @@ -16,7 +16,7 @@ require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -24,7 +24,7 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect diff --git a/vcs/go.sum b/vcs/go.sum index 4dacc7235..e4c471386 100644 --- a/vcs/go.sum +++ b/vcs/go.sum @@ -15,8 +15,8 @@ github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUK github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -63,8 +63,8 @@ github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFz github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= From e8f694eab76e7b17d4b9604c3a837217d17fd062 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 17:48:18 +0800 Subject: [PATCH 004/106] fix: dependency issue --- backend/go.mod | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index a5b47a32c..6c39b7491 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -39,10 +39,10 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/crawlab-team/crawlab/db v0.6.0-1 // indirect - github.com/crawlab-team/crawlab/fs v0.0.0-20240614083850-ff2830bd7994 // indirect - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614083850-ff2830bd7994 // indirect - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614083850-ff2830bd7994 // indirect - github.com/crawlab-team/crawlab/vcs v0.1.1 // indirect + github.com/crawlab-team/crawlab/fs v0.0.0-20240614093256-0834d6771b27 // indirect + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614093256-0834d6771b27 // indirect + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614093256-0834d6771b27 // indirect + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614093256-0834d6771b27 // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect @@ -141,7 +141,6 @@ require ( go.mongodb.org/mongo-driver v1.15.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.10.0 // indirect - go.uber.org/goleak v1.1.11 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.23.0 // indirect From 7b4ee8399ab0b4bfae3e53ccb2aaecb1ab637bed Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 17:52:18 +0800 Subject: [PATCH 005/106] fix: dependency issue --- backend/go.mod | 10 ++++---- core/go.mod | 12 +++++----- db/go.mod | 2 +- fs/go.mod | 3 ++- fs/go.sum | 1 + template-parser/go.mod | 2 +- template-parser/go.sum | 52 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 14 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 6c39b7491..5c82214ef 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -12,7 +12,7 @@ replace ( github.com/crawlab-team/crawlab/vcs => ../vcs ) -require github.com/crawlab-team/crawlab/core v0.0.0-20240614093256-0834d6771b27 +require github.com/crawlab-team/crawlab/core v0.0.0-20240614094818-e8f694eab76e require ( dario.cat/mergo v1.0.0 // indirect @@ -39,10 +39,10 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/crawlab-team/crawlab/db v0.6.0-1 // indirect - github.com/crawlab-team/crawlab/fs v0.0.0-20240614093256-0834d6771b27 // indirect - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614093256-0834d6771b27 // indirect - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614093256-0834d6771b27 // indirect - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614093256-0834d6771b27 // indirect + github.com/crawlab-team/crawlab/fs v0.0.0-20240614094818-e8f694eab76e // indirect + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614094818-e8f694eab76e // indirect + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614094818-e8f694eab76e // indirect + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614094818-e8f694eab76e // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect diff --git a/core/go.mod b/core/go.mod index 0474d4951..4c09613d4 100644 --- a/core/go.mod +++ b/core/go.mod @@ -15,12 +15,12 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/db v0.0.0-20240614093256-0834d6771b27 - github.com/crawlab-team/crawlab/fs v0.0.0-20240614093256-0834d6771b27 - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614093256-0834d6771b27 - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614093256-0834d6771b27 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614093256-0834d6771b27 - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/db v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/fs v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/trace v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614094818-e8f694eab76e github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 diff --git a/db/go.mod b/db/go.mod index a25cc3911..abe87dba8 100644 --- a/db/go.mod +++ b/db/go.mod @@ -7,7 +7,7 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614093256-0834d6771b27 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614094818-e8f694eab76e github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 github.com/spf13/viper v1.19.0 diff --git a/fs/go.mod b/fs/go.mod index e6ea7ce2f..b67da9935 100644 --- a/fs/go.mod +++ b/fs/go.mod @@ -7,8 +7,8 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614094818-e8f694eab76e github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614093256-0834d6771b27 github.com/emirpasic/gods v1.18.1 github.com/google/uuid v1.6.0 github.com/stretchr/testify v1.9.0 @@ -21,6 +21,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/ztrue/tracerr v0.4.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/fs/go.sum b/fs/go.sum index 4a03f0778..fcf3ee813 100644 --- a/fs/go.sum +++ b/fs/go.sum @@ -64,6 +64,7 @@ github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eN github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/template-parser/go.mod b/template-parser/go.mod index 259e07f35..2fa5094e4 100644 --- a/template-parser/go.mod +++ b/template-parser/go.mod @@ -5,7 +5,7 @@ go 1.22 replace github.com/crawlab-team/crawlab/db => ../db require ( - github.com/crawlab-team/crawlab/db v0.6.0-1 + github.com/crawlab-team/crawlab/db v0.0.0-20240614094818-e8f694eab76e github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.15.0 diff --git a/template-parser/go.sum b/template-parser/go.sum index 537225e29..f0fac8a58 100644 --- a/template-parser/go.sum +++ b/template-parser/go.sum @@ -10,14 +10,17 @@ github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -26,6 +29,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -33,42 +37,61 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -79,37 +102,65 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -118,3 +169,4 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 14394fa72d5713cd2d8f4a6eac8a1ab39f2c2260 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 17:58:42 +0800 Subject: [PATCH 006/106] fix: dependency issue --- backend/go.mod | 12 ++++++------ core/go.mod | 12 ++++++------ core/go.sum | 2 ++ db/go.mod | 2 +- fs/go.mod | 2 +- template-parser/go.mod | 2 +- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 5c82214ef..e5f04cc25 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -12,7 +12,7 @@ replace ( github.com/crawlab-team/crawlab/vcs => ../vcs ) -require github.com/crawlab-team/crawlab/core v0.0.0-20240614094818-e8f694eab76e +require github.com/crawlab-team/crawlab/core v0.0.0-20240614095218-7b4ee8399ab0 require ( dario.cat/mergo v1.0.0 // indirect @@ -38,11 +38,11 @@ require ( github.com/cenkalti/backoff/v4 v4.1.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/crawlab-team/crawlab/db v0.6.0-1 // indirect - github.com/crawlab-team/crawlab/fs v0.0.0-20240614094818-e8f694eab76e // indirect - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614094818-e8f694eab76e // indirect - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614094818-e8f694eab76e // indirect - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614094818-e8f694eab76e // indirect + github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect diff --git a/core/go.mod b/core/go.mod index 4c09613d4..f4f15f3e5 100644 --- a/core/go.mod +++ b/core/go.mod @@ -15,12 +15,12 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/db v0.0.0-20240614094818-e8f694eab76e - github.com/crawlab-team/crawlab/fs v0.0.0-20240614094818-e8f694eab76e - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614094818-e8f694eab76e - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614094818-e8f694eab76e - github.com/crawlab-team/crawlab/trace v0.0.0-20240614094818-e8f694eab76e - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 diff --git a/core/go.sum b/core/go.sum index 6d1aae6d0..19a5c9d56 100644 --- a/core/go.sum +++ b/core/go.sum @@ -361,6 +361,7 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -785,6 +786,7 @@ go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= diff --git a/db/go.mod b/db/go.mod index abe87dba8..d48c39dd7 100644 --- a/db/go.mod +++ b/db/go.mod @@ -7,7 +7,7 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 github.com/spf13/viper v1.19.0 diff --git a/fs/go.mod b/fs/go.mod index b67da9935..daadb5f52 100644 --- a/fs/go.mod +++ b/fs/go.mod @@ -7,7 +7,7 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 github.com/emirpasic/gods v1.18.1 github.com/google/uuid v1.6.0 diff --git a/template-parser/go.mod b/template-parser/go.mod index 2fa5094e4..5602ba475 100644 --- a/template-parser/go.mod +++ b/template-parser/go.mod @@ -5,7 +5,7 @@ go 1.22 replace github.com/crawlab-team/crawlab/db => ../db require ( - github.com/crawlab-team/crawlab/db v0.0.0-20240614094818-e8f694eab76e + github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.15.0 From 4764f56b4cc1e99e672a211c091335f5469a1538 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 18:21:50 +0800 Subject: [PATCH 007/106] fix: deps issue --- backend/go.mod | 1 - trace | 1 - trace/.gitignore | 18 ++++++++++++++++++ trace/LICENSE | 29 +++++++++++++++++++++++++++++ trace/README.md | 2 ++ trace/go.mod | 5 +++++ trace/go.sum | 2 ++ trace/trace.go | 22 ++++++++++++++++++++++ 8 files changed, 78 insertions(+), 2 deletions(-) delete mode 160000 trace create mode 100644 trace/.gitignore create mode 100644 trace/LICENSE create mode 100644 trace/README.md create mode 100644 trace/go.mod create mode 100644 trace/go.sum create mode 100644 trace/trace.go diff --git a/backend/go.mod b/backend/go.mod index e5f04cc25..7de0fc4ab 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -16,7 +16,6 @@ require github.com/crawlab-team/crawlab/core v0.0.0-20240614095218-7b4ee8399ab0 require ( dario.cat/mergo v1.0.0 // indirect - github.com/crawlab-team/crawlab/trace v0.1.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect diff --git a/trace b/trace deleted file mode 160000 index 188cd58fe..000000000 --- a/trace +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 188cd58fec6e9cbee13060594f065bf811dea212 diff --git a/trace/.gitignore b/trace/.gitignore new file mode 100644 index 000000000..ec5b69a88 --- /dev/null +++ b/trace/.gitignore @@ -0,0 +1,18 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +.idea +.DS_Store \ No newline at end of file diff --git a/trace/LICENSE b/trace/LICENSE new file mode 100644 index 000000000..3c0946b9b --- /dev/null +++ b/trace/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Crawlab Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/trace/README.md b/trace/README.md new file mode 100644 index 000000000..168e0d406 --- /dev/null +++ b/trace/README.md @@ -0,0 +1,2 @@ +# go-trace +Trace stack tools diff --git a/trace/go.mod b/trace/go.mod new file mode 100644 index 000000000..368c0f2bd --- /dev/null +++ b/trace/go.mod @@ -0,0 +1,5 @@ +module github.com/crawlab-team/crawlab/trace + +go 1.22 + +require github.com/ztrue/tracerr v0.4.0 diff --git a/trace/go.sum b/trace/go.sum new file mode 100644 index 000000000..ed0407a42 --- /dev/null +++ b/trace/go.sum @@ -0,0 +1,2 @@ +github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= +github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= diff --git a/trace/trace.go b/trace/trace.go new file mode 100644 index 000000000..52b595452 --- /dev/null +++ b/trace/trace.go @@ -0,0 +1,22 @@ +package trace + +import "github.com/ztrue/tracerr" + +func PrintError(err error) { + err = tracerr.Wrap(err) + if err != nil { + tracerr.Print(err) + } +} + +func TraceError(err error) error { + err = tracerr.Wrap(err) + if err != nil { + tracerr.Print(err) + } + return err +} + +func Error(err error) error { + return tracerr.Wrap(err) +} From e5b20af9a40b5eec67a5b8c1c543d63b0ee29817 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 19:17:23 +0800 Subject: [PATCH 008/106] feat: updated grpc proto --- backend/go.mod | 1 + core/models/models/base_v2.go | 8 +- grpc/dependencies_service_v2.pb.go | 572 ++++++++++++++++++ grpc/dependencies_service_v2_grpc.pb.go | 244 ++++++++ grpc/dependency_service_v2.pb.go | 108 ---- grpc/dependency_service_v2_grpc.pb.go | 312 ---------- .../dependencies_service_v2_request.proto | 32 - .../services/dependencies_service_v2.proto | 46 ++ .../services/dependency_service_v2.proto | 14 - 9 files changed, 867 insertions(+), 470 deletions(-) create mode 100644 grpc/dependencies_service_v2.pb.go create mode 100644 grpc/dependencies_service_v2_grpc.pb.go delete mode 100644 grpc/dependency_service_v2.pb.go delete mode 100644 grpc/dependency_service_v2_grpc.pb.go delete mode 100644 grpc/proto/entity/dependencies_service_v2_request.proto create mode 100644 grpc/proto/services/dependencies_service_v2.proto delete mode 100644 grpc/proto/services/dependency_service_v2.proto diff --git a/backend/go.mod b/backend/go.mod index 7de0fc4ab..bfc6cc274 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -41,6 +41,7 @@ require ( github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/grpc v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect diff --git a/core/models/models/base_v2.go b/core/models/models/base_v2.go index cc1468e67..13a33abe3 100644 --- a/core/models/models/base_v2.go +++ b/core/models/models/base_v2.go @@ -7,10 +7,10 @@ import ( type BaseModelV2[T any] struct { Id primitive.ObjectID `json:"_id" bson:"_id"` - CreatedAt time.Time `json:"created_ts" bson:"created_ts"` - CreatedBy primitive.ObjectID `json:"created_by" bson:"created_by"` - UpdatedAt time.Time `json:"updated_ts" bson:"updated_ts"` - UpdatedBy primitive.ObjectID `json:"updated_by" bson:"updated_by"` + CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty"` + CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty"` + UpdatedAt time.Time `json:"updated_ts,omitempty" bson:"updated_ts,omitempty"` + UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty"` } func (m *BaseModelV2[T]) GetId() primitive.ObjectID { diff --git a/grpc/dependencies_service_v2.pb.go b/grpc/dependencies_service_v2.pb.go new file mode 100644 index 000000000..baf1bcad5 --- /dev/null +++ b/grpc/dependencies_service_v2.pb.go @@ -0,0 +1,572 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/dependencies_service_v2.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type DependenciesServiceV2Code int32 + +const ( + DependenciesServiceV2Code_SYNC DependenciesServiceV2Code = 0 + DependenciesServiceV2Code_INSTALL DependenciesServiceV2Code = 1 + DependenciesServiceV2Code_UNINSTALL DependenciesServiceV2Code = 2 +) + +// Enum value maps for DependenciesServiceV2Code. +var ( + DependenciesServiceV2Code_name = map[int32]string{ + 0: "SYNC", + 1: "INSTALL", + 2: "UNINSTALL", + } + DependenciesServiceV2Code_value = map[string]int32{ + "SYNC": 0, + "INSTALL": 1, + "UNINSTALL": 2, + } +) + +func (x DependenciesServiceV2Code) Enum() *DependenciesServiceV2Code { + p := new(DependenciesServiceV2Code) + *p = x + return p +} + +func (x DependenciesServiceV2Code) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DependenciesServiceV2Code) Descriptor() protoreflect.EnumDescriptor { + return file_services_dependencies_service_v2_proto_enumTypes[0].Descriptor() +} + +func (DependenciesServiceV2Code) Type() protoreflect.EnumType { + return &file_services_dependencies_service_v2_proto_enumTypes[0] +} + +func (x DependenciesServiceV2Code) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DependenciesServiceV2Code.Descriptor instead. +func (DependenciesServiceV2Code) EnumDescriptor() ([]byte, []int) { + return file_services_dependencies_service_v2_proto_rawDescGZIP(), []int{0} +} + +type Dependency struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *Dependency) Reset() { + *x = Dependency{} + if protoimpl.UnsafeEnabled { + mi := &file_services_dependencies_service_v2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Dependency) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Dependency) ProtoMessage() {} + +func (x *Dependency) ProtoReflect() protoreflect.Message { + mi := &file_services_dependencies_service_v2_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Dependency.ProtoReflect.Descriptor instead. +func (*Dependency) Descriptor() ([]byte, []int) { + return file_services_dependencies_service_v2_proto_rawDescGZIP(), []int{0} +} + +func (x *Dependency) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Dependency) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +type DependenciesServiceV2ConnectRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` +} + +func (x *DependenciesServiceV2ConnectRequest) Reset() { + *x = DependenciesServiceV2ConnectRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_dependencies_service_v2_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2ConnectRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2ConnectRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_dependencies_service_v2_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2ConnectRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2ConnectRequest) Descriptor() ([]byte, []int) { + return file_services_dependencies_service_v2_proto_rawDescGZIP(), []int{1} +} + +func (x *DependenciesServiceV2ConnectRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +type DependenciesServiceV2ConnectResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code DependenciesServiceV2Code `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.DependenciesServiceV2Code" json:"code,omitempty"` + TaskId string `protobuf:"bytes,2,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + Lang string `protobuf:"bytes,3,opt,name=lang,proto3" json:"lang,omitempty"` + Proxy string `protobuf:"bytes,4,opt,name=proxy,proto3" json:"proxy,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,5,rep,name=dependencies,proto3" json:"dependencies,omitempty"` +} + +func (x *DependenciesServiceV2ConnectResponse) Reset() { + *x = DependenciesServiceV2ConnectResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_services_dependencies_service_v2_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2ConnectResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2ConnectResponse) ProtoMessage() {} + +func (x *DependenciesServiceV2ConnectResponse) ProtoReflect() protoreflect.Message { + mi := &file_services_dependencies_service_v2_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2ConnectResponse.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2ConnectResponse) Descriptor() ([]byte, []int) { + return file_services_dependencies_service_v2_proto_rawDescGZIP(), []int{2} +} + +func (x *DependenciesServiceV2ConnectResponse) GetCode() DependenciesServiceV2Code { + if x != nil { + return x.Code + } + return DependenciesServiceV2Code_SYNC +} + +func (x *DependenciesServiceV2ConnectResponse) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + +func (x *DependenciesServiceV2ConnectResponse) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2ConnectResponse) GetProxy() string { + if x != nil { + return x.Proxy + } + return "" +} + +func (x *DependenciesServiceV2ConnectResponse) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +type DependenciesServiceV2SyncRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` +} + +func (x *DependenciesServiceV2SyncRequest) Reset() { + *x = DependenciesServiceV2SyncRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_dependencies_service_v2_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2SyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2SyncRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2SyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_dependencies_service_v2_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2SyncRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2SyncRequest) Descriptor() ([]byte, []int) { + return file_services_dependencies_service_v2_proto_rawDescGZIP(), []int{3} +} + +func (x *DependenciesServiceV2SyncRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2SyncRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2SyncRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +type DependenciesServiceV2UpdateTaskLogRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` +} + +func (x *DependenciesServiceV2UpdateTaskLogRequest) Reset() { + *x = DependenciesServiceV2UpdateTaskLogRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_dependencies_service_v2_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2UpdateTaskLogRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2UpdateTaskLogRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2UpdateTaskLogRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_dependencies_service_v2_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2UpdateTaskLogRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2UpdateTaskLogRequest) Descriptor() ([]byte, []int) { + return file_services_dependencies_service_v2_proto_rawDescGZIP(), []int{4} +} + +func (x *DependenciesServiceV2UpdateTaskLogRequest) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + +func (x *DependenciesServiceV2UpdateTaskLogRequest) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +var File_services_dependencies_service_v2_proto protoreflect.FileDescriptor + +var file_services_dependencies_service_v2_proto_rawDesc = []byte{ + 0x0a, 0x26, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, + 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, + 0x76, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x15, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3a, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, + 0x6e, 0x63, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x22, 0x40, 0x0a, 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, + 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, + 0x4b, 0x65, 0x79, 0x22, 0xd4, 0x01, 0x0a, 0x24, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, + 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, + 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, + 0x6e, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x14, + 0x0a, 0x05, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, 0x64, 0x65, + 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x22, 0x87, 0x01, 0x0a, 0x20, 0x44, + 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, + 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, + 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x22, 0x5e, 0x0a, 0x29, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x2a, 0x41, 0x0a, 0x19, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, + 0x65, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x59, 0x4e, 0x43, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, + 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x49, 0x4e, + 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x10, 0x02, 0x32, 0x97, 0x02, 0x0a, 0x15, 0x44, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x12, 0x66, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x29, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, + 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x40, 0x0a, 0x04, 0x53, 0x79, 0x6e, + 0x63, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, + 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, + 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x12, 0x2f, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, + 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, + 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_services_dependencies_service_v2_proto_rawDescOnce sync.Once + file_services_dependencies_service_v2_proto_rawDescData = file_services_dependencies_service_v2_proto_rawDesc +) + +func file_services_dependencies_service_v2_proto_rawDescGZIP() []byte { + file_services_dependencies_service_v2_proto_rawDescOnce.Do(func() { + file_services_dependencies_service_v2_proto_rawDescData = protoimpl.X.CompressGZIP(file_services_dependencies_service_v2_proto_rawDescData) + }) + return file_services_dependencies_service_v2_proto_rawDescData +} + +var file_services_dependencies_service_v2_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_services_dependencies_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_services_dependencies_service_v2_proto_goTypes = []interface{}{ + (DependenciesServiceV2Code)(0), // 0: grpc.DependenciesServiceV2Code + (*Dependency)(nil), // 1: grpc.Dependency + (*DependenciesServiceV2ConnectRequest)(nil), // 2: grpc.DependenciesServiceV2ConnectRequest + (*DependenciesServiceV2ConnectResponse)(nil), // 3: grpc.DependenciesServiceV2ConnectResponse + (*DependenciesServiceV2SyncRequest)(nil), // 4: grpc.DependenciesServiceV2SyncRequest + (*DependenciesServiceV2UpdateTaskLogRequest)(nil), // 5: grpc.DependenciesServiceV2UpdateTaskLogRequest + (*Response)(nil), // 6: grpc.Response +} +var file_services_dependencies_service_v2_proto_depIdxs = []int32{ + 0, // 0: grpc.DependenciesServiceV2ConnectResponse.code:type_name -> grpc.DependenciesServiceV2Code + 1, // 1: grpc.DependenciesServiceV2ConnectResponse.dependencies:type_name -> grpc.Dependency + 1, // 2: grpc.DependenciesServiceV2SyncRequest.dependencies:type_name -> grpc.Dependency + 2, // 3: grpc.DependenciesServiceV2.Connect:input_type -> grpc.DependenciesServiceV2ConnectRequest + 4, // 4: grpc.DependenciesServiceV2.Sync:input_type -> grpc.DependenciesServiceV2SyncRequest + 5, // 5: grpc.DependenciesServiceV2.UpdateTaskLog:input_type -> grpc.DependenciesServiceV2UpdateTaskLogRequest + 3, // 6: grpc.DependenciesServiceV2.Connect:output_type -> grpc.DependenciesServiceV2ConnectResponse + 6, // 7: grpc.DependenciesServiceV2.Sync:output_type -> grpc.Response + 6, // 8: grpc.DependenciesServiceV2.UpdateTaskLog:output_type -> grpc.Response + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_services_dependencies_service_v2_proto_init() } +func file_services_dependencies_service_v2_proto_init() { + if File_services_dependencies_service_v2_proto != nil { + return + } + file_entity_response_proto_init() + if !protoimpl.UnsafeEnabled { + file_services_dependencies_service_v2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Dependency); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_dependencies_service_v2_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2ConnectRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_dependencies_service_v2_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2ConnectResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_dependencies_service_v2_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2SyncRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_dependencies_service_v2_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2UpdateTaskLogRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_dependencies_service_v2_proto_rawDesc, + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_dependencies_service_v2_proto_goTypes, + DependencyIndexes: file_services_dependencies_service_v2_proto_depIdxs, + EnumInfos: file_services_dependencies_service_v2_proto_enumTypes, + MessageInfos: file_services_dependencies_service_v2_proto_msgTypes, + }.Build() + File_services_dependencies_service_v2_proto = out.File + file_services_dependencies_service_v2_proto_rawDesc = nil + file_services_dependencies_service_v2_proto_goTypes = nil + file_services_dependencies_service_v2_proto_depIdxs = nil +} diff --git a/grpc/dependencies_service_v2_grpc.pb.go b/grpc/dependencies_service_v2_grpc.pb.go new file mode 100644 index 000000000..b96e3178e --- /dev/null +++ b/grpc/dependencies_service_v2_grpc.pb.go @@ -0,0 +1,244 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/dependencies_service_v2.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// DependenciesServiceV2Client is the client API for DependenciesServiceV2 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type DependenciesServiceV2Client interface { + Connect(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) + Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) + UpdateTaskLog(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_UpdateTaskLogClient, error) +} + +type dependenciesServiceV2Client struct { + cc grpc.ClientConnInterface +} + +func NewDependenciesServiceV2Client(cc grpc.ClientConnInterface) DependenciesServiceV2Client { + return &dependenciesServiceV2Client{cc} +} + +func (c *dependenciesServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) { + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], "/grpc.DependenciesServiceV2/Connect", opts...) + if err != nil { + return nil, err + } + x := &dependenciesServiceV2ConnectClient{stream} + return x, nil +} + +type DependenciesServiceV2_ConnectClient interface { + Send(*DependenciesServiceV2ConnectRequest) error + Recv() (*DependenciesServiceV2ConnectResponse, error) + grpc.ClientStream +} + +type dependenciesServiceV2ConnectClient struct { + grpc.ClientStream +} + +func (x *dependenciesServiceV2ConnectClient) Send(m *DependenciesServiceV2ConnectRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependenciesServiceV2ConnectClient) Recv() (*DependenciesServiceV2ConnectResponse, error) { + m := new(DependenciesServiceV2ConnectResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.DependenciesServiceV2/Sync", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dependenciesServiceV2Client) UpdateTaskLog(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_UpdateTaskLogClient, error) { + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], "/grpc.DependenciesServiceV2/UpdateTaskLog", opts...) + if err != nil { + return nil, err + } + x := &dependenciesServiceV2UpdateTaskLogClient{stream} + return x, nil +} + +type DependenciesServiceV2_UpdateTaskLogClient interface { + Send(*DependenciesServiceV2UpdateTaskLogRequest) error + CloseAndRecv() (*Response, error) + grpc.ClientStream +} + +type dependenciesServiceV2UpdateTaskLogClient struct { + grpc.ClientStream +} + +func (x *dependenciesServiceV2UpdateTaskLogClient) Send(m *DependenciesServiceV2UpdateTaskLogRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependenciesServiceV2UpdateTaskLogClient) CloseAndRecv() (*Response, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DependenciesServiceV2Server is the server API for DependenciesServiceV2 service. +// All implementations must embed UnimplementedDependenciesServiceV2Server +// for forward compatibility +type DependenciesServiceV2Server interface { + Connect(DependenciesServiceV2_ConnectServer) error + Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) + UpdateTaskLog(DependenciesServiceV2_UpdateTaskLogServer) error + mustEmbedUnimplementedDependenciesServiceV2Server() +} + +// UnimplementedDependenciesServiceV2Server must be embedded to have forward compatible implementations. +type UnimplementedDependenciesServiceV2Server struct { +} + +func (UnimplementedDependenciesServiceV2Server) Connect(DependenciesServiceV2_ConnectServer) error { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedDependenciesServiceV2Server) Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") +} +func (UnimplementedDependenciesServiceV2Server) UpdateTaskLog(DependenciesServiceV2_UpdateTaskLogServer) error { + return status.Errorf(codes.Unimplemented, "method UpdateTaskLog not implemented") +} +func (UnimplementedDependenciesServiceV2Server) mustEmbedUnimplementedDependenciesServiceV2Server() {} + +// UnsafeDependenciesServiceV2Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DependenciesServiceV2Server will +// result in compilation errors. +type UnsafeDependenciesServiceV2Server interface { + mustEmbedUnimplementedDependenciesServiceV2Server() +} + +func RegisterDependenciesServiceV2Server(s grpc.ServiceRegistrar, srv DependenciesServiceV2Server) { + s.RegisterService(&DependenciesServiceV2_ServiceDesc, srv) +} + +func _DependenciesServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependenciesServiceV2Server).Connect(&dependenciesServiceV2ConnectServer{stream}) +} + +type DependenciesServiceV2_ConnectServer interface { + Send(*DependenciesServiceV2ConnectResponse) error + Recv() (*DependenciesServiceV2ConnectRequest, error) + grpc.ServerStream +} + +type dependenciesServiceV2ConnectServer struct { + grpc.ServerStream +} + +func (x *dependenciesServiceV2ConnectServer) Send(m *DependenciesServiceV2ConnectResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependenciesServiceV2ConnectServer) Recv() (*DependenciesServiceV2ConnectRequest, error) { + m := new(DependenciesServiceV2ConnectRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DependenciesServiceV2SyncRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DependenciesServiceV2Server).Sync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.DependenciesServiceV2/Sync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DependenciesServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DependenciesServiceV2_UpdateTaskLog_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{stream}) +} + +type DependenciesServiceV2_UpdateTaskLogServer interface { + SendAndClose(*Response) error + Recv() (*DependenciesServiceV2UpdateTaskLogRequest, error) + grpc.ServerStream +} + +type dependenciesServiceV2UpdateTaskLogServer struct { + grpc.ServerStream +} + +func (x *dependenciesServiceV2UpdateTaskLogServer) SendAndClose(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependenciesServiceV2UpdateTaskLogServer) Recv() (*DependenciesServiceV2UpdateTaskLogRequest, error) { + m := new(DependenciesServiceV2UpdateTaskLogRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DependenciesServiceV2_ServiceDesc is the grpc.ServiceDesc for DependenciesServiceV2 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DependenciesServiceV2_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.DependenciesServiceV2", + HandlerType: (*DependenciesServiceV2Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Sync", + Handler: _DependenciesServiceV2_Sync_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _DependenciesServiceV2_Connect_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "UpdateTaskLog", + Handler: _DependenciesServiceV2_UpdateTaskLog_Handler, + ClientStreams: true, + }, + }, + Metadata: "services/dependencies_service_v2.proto", +} diff --git a/grpc/dependency_service_v2.pb.go b/grpc/dependency_service_v2.pb.go deleted file mode 100644 index 7057a9cba..000000000 --- a/grpc/dependency_service_v2.pb.go +++ /dev/null @@ -1,108 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 -// source: services/dependency_service_v2.proto - -package grpc - -import ( - proto "github.com/golang/protobuf/proto" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - -var File_services_dependency_service_v2_proto protoreflect.FileDescriptor - -var file_services_dependency_service_v2_proto_rawDesc = []byte{ - 0x0a, 0x24, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, - 0x64, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x2c, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, - 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x32, 0xc9, 0x02, 0x0a, 0x13, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x12, 0x48, 0x0a, 0x07, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, - 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x28, 0x01, 0x12, 0x40, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x26, 0x2e, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4a, 0x0a, 0x07, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, - 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, - 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, - 0x01, 0x12, 0x5a, 0x0a, 0x15, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x44, 0x65, - 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x2e, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a, - 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var file_services_dependency_service_v2_proto_goTypes = []interface{}{ - (*DependenciesServiceV2ConnectRequest)(nil), // 0: grpc.DependenciesServiceV2ConnectRequest - (*DependenciesServiceV2SyncRequest)(nil), // 1: grpc.DependenciesServiceV2SyncRequest - (*DependenciesServiceV2InstallRequest)(nil), // 2: grpc.DependenciesServiceV2InstallRequest - (*DependenciesServiceV2UninstallRequest)(nil), // 3: grpc.DependenciesServiceV2UninstallRequest - (*Response)(nil), // 4: grpc.Response -} -var file_services_dependency_service_v2_proto_depIdxs = []int32{ - 0, // 0: grpc.DependencyServiceV2.Connect:input_type -> grpc.DependenciesServiceV2ConnectRequest - 1, // 1: grpc.DependencyServiceV2.Sync:input_type -> grpc.DependenciesServiceV2SyncRequest - 2, // 2: grpc.DependencyServiceV2.Install:input_type -> grpc.DependenciesServiceV2InstallRequest - 3, // 3: grpc.DependencyServiceV2.UninstallDependencies:input_type -> grpc.DependenciesServiceV2UninstallRequest - 4, // 4: grpc.DependencyServiceV2.Connect:output_type -> grpc.Response - 4, // 5: grpc.DependencyServiceV2.Sync:output_type -> grpc.Response - 4, // 6: grpc.DependencyServiceV2.Install:output_type -> grpc.Response - 4, // 7: grpc.DependencyServiceV2.UninstallDependencies:output_type -> grpc.Response - 4, // [4:8] is the sub-list for method output_type - 0, // [0:4] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_services_dependency_service_v2_proto_init() } -func file_services_dependency_service_v2_proto_init() { - if File_services_dependency_service_v2_proto != nil { - return - } - file_entity_dependencies_service_v2_request_proto_init() - file_entity_response_proto_init() - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_services_dependency_service_v2_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_services_dependency_service_v2_proto_goTypes, - DependencyIndexes: file_services_dependency_service_v2_proto_depIdxs, - }.Build() - File_services_dependency_service_v2_proto = out.File - file_services_dependency_service_v2_proto_rawDesc = nil - file_services_dependency_service_v2_proto_goTypes = nil - file_services_dependency_service_v2_proto_depIdxs = nil -} diff --git a/grpc/dependency_service_v2_grpc.pb.go b/grpc/dependency_service_v2_grpc.pb.go deleted file mode 100644 index 3d0a9eb96..000000000 --- a/grpc/dependency_service_v2_grpc.pb.go +++ /dev/null @@ -1,312 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 -// source: services/dependency_service_v2.proto - -package grpc - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// DependencyServiceV2Client is the client API for DependencyServiceV2 service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type DependencyServiceV2Client interface { - Connect(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_ConnectClient, error) - Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) - Install(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_InstallClient, error) - UninstallDependencies(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_UninstallDependenciesClient, error) -} - -type dependencyServiceV2Client struct { - cc grpc.ClientConnInterface -} - -func NewDependencyServiceV2Client(cc grpc.ClientConnInterface) DependencyServiceV2Client { - return &dependencyServiceV2Client{cc} -} - -func (c *dependencyServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_ConnectClient, error) { - stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[0], "/grpc.DependencyServiceV2/Connect", opts...) - if err != nil { - return nil, err - } - x := &dependencyServiceV2ConnectClient{stream} - return x, nil -} - -type DependencyServiceV2_ConnectClient interface { - Send(*DependenciesServiceV2ConnectRequest) error - CloseAndRecv() (*Response, error) - grpc.ClientStream -} - -type dependencyServiceV2ConnectClient struct { - grpc.ClientStream -} - -func (x *dependencyServiceV2ConnectClient) Send(m *DependenciesServiceV2ConnectRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *dependencyServiceV2ConnectClient) CloseAndRecv() (*Response, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - m := new(Response) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *dependencyServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { - out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.DependencyServiceV2/Sync", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *dependencyServiceV2Client) Install(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_InstallClient, error) { - stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[1], "/grpc.DependencyServiceV2/Install", opts...) - if err != nil { - return nil, err - } - x := &dependencyServiceV2InstallClient{stream} - return x, nil -} - -type DependencyServiceV2_InstallClient interface { - Send(*DependenciesServiceV2InstallRequest) error - Recv() (*Response, error) - grpc.ClientStream -} - -type dependencyServiceV2InstallClient struct { - grpc.ClientStream -} - -func (x *dependencyServiceV2InstallClient) Send(m *DependenciesServiceV2InstallRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *dependencyServiceV2InstallClient) Recv() (*Response, error) { - m := new(Response) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *dependencyServiceV2Client) UninstallDependencies(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_UninstallDependenciesClient, error) { - stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[2], "/grpc.DependencyServiceV2/UninstallDependencies", opts...) - if err != nil { - return nil, err - } - x := &dependencyServiceV2UninstallDependenciesClient{stream} - return x, nil -} - -type DependencyServiceV2_UninstallDependenciesClient interface { - Send(*DependenciesServiceV2UninstallRequest) error - Recv() (*Response, error) - grpc.ClientStream -} - -type dependencyServiceV2UninstallDependenciesClient struct { - grpc.ClientStream -} - -func (x *dependencyServiceV2UninstallDependenciesClient) Send(m *DependenciesServiceV2UninstallRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *dependencyServiceV2UninstallDependenciesClient) Recv() (*Response, error) { - m := new(Response) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// DependencyServiceV2Server is the server API for DependencyServiceV2 service. -// All implementations must embed UnimplementedDependencyServiceV2Server -// for forward compatibility -type DependencyServiceV2Server interface { - Connect(DependencyServiceV2_ConnectServer) error - Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) - Install(DependencyServiceV2_InstallServer) error - UninstallDependencies(DependencyServiceV2_UninstallDependenciesServer) error - mustEmbedUnimplementedDependencyServiceV2Server() -} - -// UnimplementedDependencyServiceV2Server must be embedded to have forward compatible implementations. -type UnimplementedDependencyServiceV2Server struct { -} - -func (UnimplementedDependencyServiceV2Server) Connect(DependencyServiceV2_ConnectServer) error { - return status.Errorf(codes.Unimplemented, "method Connect not implemented") -} -func (UnimplementedDependencyServiceV2Server) Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) { - return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") -} -func (UnimplementedDependencyServiceV2Server) Install(DependencyServiceV2_InstallServer) error { - return status.Errorf(codes.Unimplemented, "method Install not implemented") -} -func (UnimplementedDependencyServiceV2Server) UninstallDependencies(DependencyServiceV2_UninstallDependenciesServer) error { - return status.Errorf(codes.Unimplemented, "method UninstallDependencies not implemented") -} -func (UnimplementedDependencyServiceV2Server) mustEmbedUnimplementedDependencyServiceV2Server() {} - -// UnsafeDependencyServiceV2Server may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to DependencyServiceV2Server will -// result in compilation errors. -type UnsafeDependencyServiceV2Server interface { - mustEmbedUnimplementedDependencyServiceV2Server() -} - -func RegisterDependencyServiceV2Server(s grpc.ServiceRegistrar, srv DependencyServiceV2Server) { - s.RegisterService(&DependencyServiceV2_ServiceDesc, srv) -} - -func _DependencyServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependencyServiceV2Server).Connect(&dependencyServiceV2ConnectServer{stream}) -} - -type DependencyServiceV2_ConnectServer interface { - SendAndClose(*Response) error - Recv() (*DependenciesServiceV2ConnectRequest, error) - grpc.ServerStream -} - -type dependencyServiceV2ConnectServer struct { - grpc.ServerStream -} - -func (x *dependencyServiceV2ConnectServer) SendAndClose(m *Response) error { - return x.ServerStream.SendMsg(m) -} - -func (x *dependencyServiceV2ConnectServer) Recv() (*DependenciesServiceV2ConnectRequest, error) { - m := new(DependenciesServiceV2ConnectRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _DependencyServiceV2_Sync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DependenciesServiceV2SyncRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(DependencyServiceV2Server).Sync(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.DependencyServiceV2/Sync", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(DependencyServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _DependencyServiceV2_Install_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependencyServiceV2Server).Install(&dependencyServiceV2InstallServer{stream}) -} - -type DependencyServiceV2_InstallServer interface { - Send(*Response) error - Recv() (*DependenciesServiceV2InstallRequest, error) - grpc.ServerStream -} - -type dependencyServiceV2InstallServer struct { - grpc.ServerStream -} - -func (x *dependencyServiceV2InstallServer) Send(m *Response) error { - return x.ServerStream.SendMsg(m) -} - -func (x *dependencyServiceV2InstallServer) Recv() (*DependenciesServiceV2InstallRequest, error) { - m := new(DependenciesServiceV2InstallRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _DependencyServiceV2_UninstallDependencies_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependencyServiceV2Server).UninstallDependencies(&dependencyServiceV2UninstallDependenciesServer{stream}) -} - -type DependencyServiceV2_UninstallDependenciesServer interface { - Send(*Response) error - Recv() (*DependenciesServiceV2UninstallRequest, error) - grpc.ServerStream -} - -type dependencyServiceV2UninstallDependenciesServer struct { - grpc.ServerStream -} - -func (x *dependencyServiceV2UninstallDependenciesServer) Send(m *Response) error { - return x.ServerStream.SendMsg(m) -} - -func (x *dependencyServiceV2UninstallDependenciesServer) Recv() (*DependenciesServiceV2UninstallRequest, error) { - m := new(DependenciesServiceV2UninstallRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// DependencyServiceV2_ServiceDesc is the grpc.ServiceDesc for DependencyServiceV2 service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var DependencyServiceV2_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.DependencyServiceV2", - HandlerType: (*DependencyServiceV2Server)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Sync", - Handler: _DependencyServiceV2_Sync_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Connect", - Handler: _DependencyServiceV2_Connect_Handler, - ClientStreams: true, - }, - { - StreamName: "Install", - Handler: _DependencyServiceV2_Install_Handler, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "UninstallDependencies", - Handler: _DependencyServiceV2_UninstallDependencies_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "services/dependency_service_v2.proto", -} diff --git a/grpc/proto/entity/dependencies_service_v2_request.proto b/grpc/proto/entity/dependencies_service_v2_request.proto deleted file mode 100644 index c7cfa79d5..000000000 --- a/grpc/proto/entity/dependencies_service_v2_request.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package grpc; -option go_package = ".;grpc"; - -message Dependency { - string name = 1; - string version = 2; -} - -message DependenciesServiceV2ConnectRequest { - string node_key = 1; -} - -message DependenciesServiceV2SyncRequest { - string node_key = 1; - string lang = 2; - repeated Dependency dependencies = 3; -} - -message DependenciesServiceV2InstallRequest { - string node_key = 1; - string lang = 2; - repeated Dependency dependencies = 3; - string proxy = 4; -} - -message DependenciesServiceV2UninstallRequest { - string node_key = 1; - string lang = 2; - repeated Dependency dependencies = 3; -} \ No newline at end of file diff --git a/grpc/proto/services/dependencies_service_v2.proto b/grpc/proto/services/dependencies_service_v2.proto new file mode 100644 index 000000000..b09f9c940 --- /dev/null +++ b/grpc/proto/services/dependencies_service_v2.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; + +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +message Dependency { + string name = 1; + string version = 2; +} + +message DependenciesServiceV2ConnectRequest { + string node_key = 1; +} + +enum DependenciesServiceV2Code { + SYNC = 0; + INSTALL = 1; + UNINSTALL = 2; +} + +message DependenciesServiceV2ConnectResponse { + DependenciesServiceV2Code code = 1; + string task_id = 2; + string lang = 3; + string proxy = 4; + repeated Dependency dependencies = 5; +} + +message DependenciesServiceV2SyncRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; +} + +message DependenciesServiceV2UpdateTaskLogRequest { + string task_id = 1; + string content = 2; +} + +service DependenciesServiceV2 { + rpc Connect(stream DependenciesServiceV2ConnectRequest) returns (stream DependenciesServiceV2ConnectResponse){}; + rpc Sync(DependenciesServiceV2SyncRequest) returns (Response){}; + rpc UpdateTaskLog(stream DependenciesServiceV2UpdateTaskLogRequest) returns (Response){}; +} diff --git a/grpc/proto/services/dependency_service_v2.proto b/grpc/proto/services/dependency_service_v2.proto deleted file mode 100644 index dea310d64..000000000 --- a/grpc/proto/services/dependency_service_v2.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -import "entity/dependencies_service_v2_request.proto"; -import "entity/response.proto"; - -package grpc; -option go_package = ".;grpc"; - -service DependencyServiceV2 { - rpc Connect(stream DependenciesServiceV2ConnectRequest) returns (Response){}; - rpc Sync(DependenciesServiceV2SyncRequest) returns (Response){}; - rpc Install(stream DependenciesServiceV2InstallRequest) returns (stream Response){}; - rpc UninstallDependencies(stream DependenciesServiceV2UninstallRequest) returns (stream Response){}; -} From 08d9f4f676118b1c3e01d9d07500d8cd6d84451a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 14 Jun 2024 19:24:09 +0800 Subject: [PATCH 009/106] feat: updated grpc proto --- core/go.mod | 2 +- core/grpc/server/dependencies_server_v2.go | 81 ++++++++++++++++--- .../server/model_base_service_v2_server.go | 3 + core/models/models/dependency_log_v2.go | 10 +++ core/models/models/dependency_task_v2.go | 15 ++++ core/models/models/dependency_v2.go | 14 ++++ 6 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 core/models/models/dependency_log_v2.go create mode 100644 core/models/models/dependency_task_v2.go create mode 100644 core/models/models/dependency_v2.go diff --git a/core/go.mod b/core/go.mod index f4f15f3e5..98fc835d8 100644 --- a/core/go.mod +++ b/core/go.mod @@ -17,7 +17,7 @@ require ( github.com/cenkalti/backoff/v4 v4.1.0 github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614111723-e5b20af9a40b github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 diff --git a/core/grpc/server/dependencies_server_v2.go b/core/grpc/server/dependencies_server_v2.go index aca7e0086..b5b15c355 100644 --- a/core/grpc/server/dependencies_server_v2.go +++ b/core/grpc/server/dependencies_server_v2.go @@ -2,29 +2,84 @@ package server import ( "context" - grpc "github.com/crawlab-team/crawlab/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/grpc" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "sync" ) type DependenciesServerV2 struct { - grpc.UnimplementedDependencyServiceV2Server + grpc.UnimplementedDependenciesServiceV2Server + mu *sync.Mutex + streams map[string]grpc.DependenciesServiceV2_ConnectServer } -func (svr DependenciesServerV2) Connect(stream grpc.DependencyServiceV2_ConnectServer) (err error) { - return status.Errorf(codes.Unimplemented, "method Connect not implemented") +func (svr DependenciesServerV2) Connect(stream grpc.DependenciesServiceV2_ConnectServer) (err error) { + svr.mu.Lock() + defer svr.mu.Unlock() + req, err := stream.Recv() + if err != nil { + return err + } + svr.streams[req.NodeKey] = stream + return nil } func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) { - return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") + n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) + if err != nil { + return nil, err + } + var deps []models.DependencyV2 + for _, dep := range request.Dependencies { + deps = append(deps, models.DependencyV2{ + Name: dep.Name, + NodeId: n.Id, + Type: request.Lang, + Version: dep.Version, + }) + } + _, err = service.NewModelServiceV2[models.DependencyV2]().InsertMany(deps) + if err != nil { + return nil, err + } + return nil, nil } -func (svr DependenciesServerV2) Install(stream grpc.DependencyServiceV2_InstallServer) (err error) { - return status.Errorf(codes.Unimplemented, "method Install not implemented") -} - -func (svr DependenciesServerV2) UninstallDependencies(stream grpc.DependencyServiceV2_UninstallDependenciesServer) (err error) { - return status.Errorf(codes.Unimplemented, "method UninstallDependencies not implemented") +func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_UpdateTaskLogServer) (err error) { + var t *models.DependencyTaskV2 + for { + req, err := stream.Recv() + if err == io.EOF { + // all messages have been received + return stream.SendAndClose(&grpc.Response{Message: "update task log finished"}) + } + if err != nil { + return err + } + taskId, err := primitive.ObjectIDFromHex(req.TaskId) + if err != nil { + return err + } + if t == nil { + t, err = service.NewModelServiceV2[models.DependencyTaskV2]().GetById(taskId) + if err != nil { + return err + } + } + l := models.DependencyLogV2{ + TaskId: taskId, + Content: req.Content, + } + l.SetCreated(t.CreatedBy) + _, err = service.NewModelServiceV2[models.DependencyLogV2]().InsertOne(l) + if err != nil { + return err + } + } } func NewDependenciesServerV2() *DependenciesServerV2 { diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 793856027..17771925d 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -19,7 +19,10 @@ var ( *new(models.TestModel), *new(models.DataCollectionV2), *new(models.DataSourceV2), + *new(models.DependencyV2), + *new(models.DependencyLogV2), *new(models.DependencySettingV2), + *new(models.DependencyTaskV2), *new(models.EnvironmentV2), *new(models.GitV2), *new(models.NodeV2), diff --git a/core/models/models/dependency_log_v2.go b/core/models/models/dependency_log_v2.go new file mode 100644 index 000000000..f0178c2e4 --- /dev/null +++ b/core/models/models/dependency_log_v2.go @@ -0,0 +1,10 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyLogV2 struct { + any `collection:"dependency_logs"` + BaseModelV2[DependencyLogV2] `bson:",inline"` + TaskId primitive.ObjectID `json:"task_id" bson:"task_id"` + Content string `json:"content" bson:"content"` +} diff --git a/core/models/models/dependency_task_v2.go b/core/models/models/dependency_task_v2.go new file mode 100644 index 000000000..c9fd40d66 --- /dev/null +++ b/core/models/models/dependency_task_v2.go @@ -0,0 +1,15 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyTaskV2 struct { + any `collection:"dependency_tasks"` + BaseModelV2[DependencyTaskV2] `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Action string `json:"action" bson:"action"` + DepNames []string `json:"dep_names" bson:"dep_names"` +} diff --git a/core/models/models/dependency_v2.go b/core/models/models/dependency_v2.go new file mode 100644 index 000000000..4d3811d6f --- /dev/null +++ b/core/models/models/dependency_v2.go @@ -0,0 +1,14 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyV2 struct { + any `collection:"dependencies"` + BaseModelV2[DependencyV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Type string `json:"type" bson:"type"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + Version string `json:"version" bson:"version"` +} From 972713959f243763810f7f1f0c00452cf9e5719c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 15 Jun 2024 23:25:24 +0800 Subject: [PATCH 010/106] feat: updated grpc for dependencies service --- core/controllers/base_v2.go | 12 +- core/entity/dependency.go | 14 + core/grpc/client/client_v2.go | 4 +- core/grpc/server/dependencies_server_v2.go | 135 ++++- core/grpc/server/server_v2.go | 23 +- core/models/common/index_service_v2.go | 160 ++++++ core/models/models/dependency_v2.go | 18 +- core/models/models/git_v2.go | 1 + core/node/service/master_service_v2.go | 2 +- grpc/dependencies_service_v2.pb.go | 62 +-- grpc/dependencies_service_v2_grpc.pb.go | 35 +- grpc/dependencies_service_v2_request.pb.go | 488 ------------------ .../services/dependencies_service_v2.proto | 4 +- 13 files changed, 371 insertions(+), 587 deletions(-) create mode 100644 core/entity/dependency.go create mode 100644 core/models/common/index_service_v2.go delete mode 100644 grpc/dependencies_service_v2_request.pb.go diff --git a/core/controllers/base_v2.go b/core/controllers/base_v2.go index ddb9107e1..be6eebff5 100644 --- a/core/controllers/base_v2.go +++ b/core/controllers/base_v2.go @@ -86,7 +86,6 @@ func (ctr *BaseControllerV2[T]) PutById(c *gin.Context) { u := GetUserFromContextV2(c) m := any(&model).(interfaces.ModelV2) - m.SetId(primitive.NewObjectID()) m.SetUpdated(u.Id) if err := ctr.modelSvc.ReplaceById(id, model); err != nil { @@ -170,14 +169,19 @@ func (ctr *BaseControllerV2[T]) DeleteList(c *gin.Context) { } func (ctr *BaseControllerV2[T]) getAll(c *gin.Context) { - models, err := ctr.modelSvc.GetMany(nil, &mongo.FindOptions{ - Sort: bson.D{{"_id", -1}}, + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + if sort == nil { + sort = bson.D{{"_id", -1}} + } + models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ + Sort: sort, }) if err != nil { HandleErrorInternalServerError(c, err) return } - total, err := ctr.modelSvc.Count(nil) + total, err := ctr.modelSvc.Count(query) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/entity/dependency.go b/core/entity/dependency.go new file mode 100644 index 000000000..55f9c6ada --- /dev/null +++ b/core/entity/dependency.go @@ -0,0 +1,14 @@ +package entity + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyResult struct { + Name string `json:"name,omitempty" bson:"name,omitempty"` + NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"node_ids,omitempty"` + Versions []string `json:"versions,omitempty" bson:"versions,omitempty"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + Count int `json:"count,omitempty" bson:"count,omitempty"` + Upgradable bool `json:"upgradable" bson:"upgradable"` + Downgradable bool `json:"downgradable" bson:"downgradable"` + Installable bool `json:"installable" bson:"installable"` +} diff --git a/core/grpc/client/client_v2.go b/core/grpc/client/client_v2.go index fd40cc9c4..f4ddc2e32 100644 --- a/core/grpc/client/client_v2.go +++ b/core/grpc/client/client_v2.go @@ -40,7 +40,7 @@ type GrpcClientV2 struct { NodeClient grpc2.NodeServiceClient TaskClient grpc2.TaskServiceClient ModelBaseServiceV2Client grpc2.ModelBaseServiceV2Client - DependenciesClient grpc2.DependencyServiceV2Client + DependenciesClient grpc2.DependenciesServiceV2Client } func (c *GrpcClientV2) Init() (err error) { @@ -95,7 +95,7 @@ func (c *GrpcClientV2) Register() { c.NodeClient = grpc2.NewNodeServiceClient(c.conn) c.ModelBaseServiceV2Client = grpc2.NewModelBaseServiceV2Client(c.conn) c.TaskClient = grpc2.NewTaskServiceClient(c.conn) - c.DependenciesClient = grpc2.NewDependencyServiceV2Client(c.conn) + c.DependenciesClient = grpc2.NewDependenciesServiceV2Client(c.conn) // log log.Infof("[GrpcClient] grpc client registered client services") diff --git a/core/grpc/server/dependencies_server_v2.go b/core/grpc/server/dependencies_server_v2.go index b5b15c355..f7ede5c05 100644 --- a/core/grpc/server/dependencies_server_v2.go +++ b/core/grpc/server/dependencies_server_v2.go @@ -2,30 +2,41 @@ package server import ( "context" + "errors" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" + mongo2 "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" "io" "sync" + "time" ) type DependenciesServerV2 struct { grpc.UnimplementedDependenciesServiceV2Server mu *sync.Mutex - streams map[string]grpc.DependenciesServiceV2_ConnectServer + streams map[string]*grpc.DependenciesServiceV2_ConnectServer } -func (svr DependenciesServerV2) Connect(stream grpc.DependenciesServiceV2_ConnectServer) (err error) { +func (svr DependenciesServerV2) Connect(req *grpc.DependenciesServiceV2ConnectRequest, stream grpc.DependenciesServiceV2_ConnectServer) (err error) { svr.mu.Lock() - defer svr.mu.Unlock() - req, err := stream.Recv() - if err != nil { - return err + svr.streams[req.NodeKey] = &stream + svr.mu.Unlock() + log.Info("[DependenciesServerV2] connected: " + req.NodeKey) + + // Keep this scope alive because once this scope exits - the stream is closed + for { + select { + case <-stream.Context().Done(): + log.Info("[DependenciesServerV2] disconnected: " + req.NodeKey) + return nil + } } - svr.streams[req.NodeKey] = stream - return nil } func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) { @@ -33,20 +44,75 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen if err != nil { return nil, err } - var deps []models.DependencyV2 + + depsDb, err := service.NewModelServiceV2[models.DependencyV2]().GetMany(bson.M{ + "node_id": n.Id, + "type": request.Lang, + }, nil) + if err != nil { + if !errors.Is(err, mongo.ErrNoDocuments) { + log.Errorf("[DependenciesServiceV2] get dependencies from db error: %v", err) + return nil, err + } + } + + depsDbMap := make(map[string]*models.DependencyV2) + for _, d := range depsDb { + depsDbMap[d.Name] = &d + } + + var depsToInsert []models.DependencyV2 + depsMap := make(map[string]*models.DependencyV2) for _, dep := range request.Dependencies { - deps = append(deps, models.DependencyV2{ + d := models.DependencyV2{ Name: dep.Name, NodeId: n.Id, Type: request.Lang, Version: dep.Version, - }) + } + d.SetCreatedAt(time.Now()) + + depsMap[d.Name] = &d + + _, ok := depsDbMap[d.Name] + if !ok { + depsToInsert = append(depsToInsert, d) + } } - _, err = service.NewModelServiceV2[models.DependencyV2]().InsertMany(deps) - if err != nil { - return nil, err + + var depIdsToDelete []primitive.ObjectID + for _, d := range depsDb { + _, ok := depsMap[d.Name] + if !ok { + depIdsToDelete = append(depIdsToDelete, d.Id) + } } - return nil, nil + + err = mongo2.RunTransaction(func(ctx mongo.SessionContext) (err error) { + if len(depIdsToDelete) > 0 { + err = service.NewModelServiceV2[models.DependencyV2]().DeleteMany(bson.M{ + "_id": bson.M{"$in": depIdsToDelete}, + }) + if err != nil { + log.Errorf("[DependenciesServerV2] delete dependencies in db error: %v", err) + trace.PrintError(err) + return err + } + } + + if len(depsToInsert) > 0 { + _, err = service.NewModelServiceV2[models.DependencyV2]().InsertMany(depsToInsert) + if err != nil { + log.Errorf("[DependenciesServerV2] insert dependencies in db error: %v", err) + trace.PrintError(err) + return err + } + } + + return nil + }) + + return nil, err } func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_UpdateTaskLogServer) (err error) { @@ -70,18 +136,45 @@ func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_ return err } } - l := models.DependencyLogV2{ - TaskId: taskId, - Content: req.Content, + var logs []models.DependencyLogV2 + for _, line := range req.LogLines { + l := models.DependencyLogV2{ + TaskId: taskId, + Content: line, + } + l.SetCreated(t.CreatedBy) + logs = append(logs, l) } - l.SetCreated(t.CreatedBy) - _, err = service.NewModelServiceV2[models.DependencyLogV2]().InsertOne(l) + _, err = service.NewModelServiceV2[models.DependencyLogV2]().InsertMany(logs) if err != nil { return err } } } +func (svr DependenciesServerV2) GetStream(key string) (stream *grpc.DependenciesServiceV2_ConnectServer, err error) { + svr.mu.Lock() + defer svr.mu.Unlock() + stream, ok := svr.streams[key] + if !ok { + return nil, errors.New("stream not found") + } + return stream, nil +} + func NewDependenciesServerV2() *DependenciesServerV2 { - return &DependenciesServerV2{} + return &DependenciesServerV2{ + mu: new(sync.Mutex), + streams: make(map[string]*grpc.DependenciesServiceV2_ConnectServer), + } +} + +var depSvc *DependenciesServerV2 + +func GetDependenciesServerV2() *DependenciesServerV2 { + if depSvc != nil { + return depSvc + } + depSvc = NewDependenciesServerV2() + return depSvc } diff --git a/core/grpc/server/server_v2.go b/core/grpc/server/server_v2.go index ebcb871b7..ad04c5a03 100644 --- a/core/grpc/server/server_v2.go +++ b/core/grpc/server/server_v2.go @@ -42,10 +42,10 @@ type GrpcServerV2 struct { nodeCfgSvc interfaces.NodeConfigService // servers - nodeSvr *NodeServerV2 - taskSvr *TaskServerV2 - modelBaseServiceSvr *ModelBaseServiceServerV2 - dependenciesSvr *DependenciesServerV2 + NodeSvr *NodeServerV2 + TaskSvr *TaskServerV2 + ModelBaseServiceSvr *ModelBaseServiceServerV2 + DependenciesSvr *DependenciesServerV2 } func (svr *GrpcServerV2) GetConfigPath() (path string) { @@ -115,9 +115,10 @@ func (svr *GrpcServerV2) Stop() (err error) { } func (svr *GrpcServerV2) Register() (err error) { - grpc2.RegisterNodeServiceServer(svr.svr, *svr.nodeSvr) // node service - grpc2.RegisterModelBaseServiceV2Server(svr.svr, *svr.modelBaseServiceSvr) - grpc2.RegisterTaskServiceServer(svr.svr, *svr.taskSvr) + grpc2.RegisterNodeServiceServer(svr.svr, *svr.NodeSvr) + grpc2.RegisterModelBaseServiceV2Server(svr.svr, *svr.ModelBaseServiceSvr) + grpc2.RegisterTaskServiceServer(svr.svr, *svr.TaskSvr) + grpc2.RegisterDependenciesServiceV2Server(svr.svr, *svr.DependenciesSvr) return nil } @@ -206,16 +207,16 @@ func NewGrpcServerV2() (svr *GrpcServerV2, err error) { svr.nodeCfgSvc = nodeconfig.GetNodeConfigService() - svr.nodeSvr, err = NewNodeServerV2() + svr.NodeSvr, err = NewNodeServerV2() if err != nil { return nil, err } - svr.modelBaseServiceSvr = NewModelBaseServiceV2Server() - svr.taskSvr, err = NewTaskServerV2() + svr.ModelBaseServiceSvr = NewModelBaseServiceV2Server() + svr.TaskSvr, err = NewTaskServerV2() if err != nil { return nil, err } - svr.dependenciesSvr = NewDependenciesServerV2() + svr.DependenciesSvr = GetDependenciesServerV2() // recovery options recoveryOpts := []grpc_recovery.Option{ diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go new file mode 100644 index 000000000..c46ec3dc5 --- /dev/null +++ b/core/models/common/index_service_v2.go @@ -0,0 +1,160 @@ +package common + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/db/mongo" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func CreateIndexesV2() { + // nodes + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, // key + {Keys: bson.M{"name": 1}}, // name + {Keys: bson.M{"is_master": 1}}, // is_master + {Keys: bson.M{"status": 1}}, // status + {Keys: bson.M{"enabled": 1}}, // enabled + {Keys: bson.M{"active": 1}}, // active + }) + + // projects + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // spiders + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + {Keys: bson.M{"type": 1}}, + {Keys: bson.M{"col_id": 1}}, + {Keys: bson.M{"project_id": 1}}, + }) + + // tasks + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"spider_id": 1}}, + {Keys: bson.M{"status": 1}}, + {Keys: bson.M{"node_id": 1}}, + {Keys: bson.M{"schedule_id": 1}}, + {Keys: bson.M{"type": 1}}, + {Keys: bson.M{"mode": 1}}, + {Keys: bson.M{"priority": 1}}, + {Keys: bson.M{"parent_id": 1}}, + {Keys: bson.M{"has_sub": 1}}, + {Keys: bson.M{"create_ts": -1}}, + }) + + // task stats + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"create_ts": 1}}, + }) + + // schedules + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + {Keys: bson.M{"spider_id": 1}}, + {Keys: bson.M{"enabled": 1}}, + }) + + // users + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"username": 1}}, + {Keys: bson.M{"role": 1}}, + {Keys: bson.M{"email": 1}}, + }) + + // settings + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, + }) + + // tokens + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // variables + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, + }) + + // data sources + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // data collections + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // roles + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // user role relations + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, + {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // permissions + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // role permission relations + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, + {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // dependencies + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"type", 1}, + {"node_id", 1}, + {"name", 1}, + }, + Options: (&options.IndexOptions{}).SetUnique(true), + }, + }) + + // dependency settings + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"type", 1}, + {"node_id", 1}, + {"name", 1}, + }, + Options: (&options.IndexOptions{}).SetUnique(true), + }, + }) + + // dependency logs + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{{"task_id", 1}}, + }, + { + Keys: bson.D{{"update_ts", 1}}, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24), + }, + }) + + // dependency tasks + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"update_ts", 1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24), + }, + }) +} diff --git a/core/models/models/dependency_v2.go b/core/models/models/dependency_v2.go index 4d3811d6f..6d8d60393 100644 --- a/core/models/models/dependency_v2.go +++ b/core/models/models/dependency_v2.go @@ -1,14 +1,18 @@ package models -import "go.mongodb.org/mongo-driver/bson/primitive" +import ( + "github.com/crawlab-team/crawlab/core/entity" + "go.mongodb.org/mongo-driver/bson/primitive" +) type DependencyV2 struct { any `collection:"dependencies"` BaseModelV2[DependencyV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Type string `json:"type" bson:"type"` - LatestVersion string `json:"latest_version" bson:"latest_version"` - Version string `json:"version" bson:"version"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Type string `json:"type" bson:"type"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + Version string `json:"version" bson:"version"` + Result entity.DependencyResult `json:"result" bson:"-"` } diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go index b72c130a9..1ccced39f 100644 --- a/core/models/models/git_v2.go +++ b/core/models/models/git_v2.go @@ -4,6 +4,7 @@ type GitV2 struct { any `collection:"gits"` BaseModelV2[GitV2] `bson:",inline"` Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` AuthType string `json:"auth_type" bson:"auth_type"` Username string `json:"username" bson:"username"` Password string `json:"password" bson:"password"` diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 1ca7a52aa..13aa69c1e 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -54,7 +54,7 @@ func (svc *MasterServiceV2) Init() (err error) { func (svc *MasterServiceV2) Start() { // create indexes - common.CreateIndexes() + common.CreateIndexesV2() // start grpc server if err := svc.server.Start(); err != nil { diff --git a/grpc/dependencies_service_v2.pb.go b/grpc/dependencies_service_v2.pb.go index baf1bcad5..6966b1d95 100644 --- a/grpc/dependencies_service_v2.pb.go +++ b/grpc/dependencies_service_v2.pb.go @@ -323,8 +323,8 @@ type DependenciesServiceV2UpdateTaskLogRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` - Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + LogLines []string `protobuf:"bytes,2,rep,name=log_lines,json=logLines,proto3" json:"log_lines,omitempty"` } func (x *DependenciesServiceV2UpdateTaskLogRequest) Reset() { @@ -366,11 +366,11 @@ func (x *DependenciesServiceV2UpdateTaskLogRequest) GetTaskId() string { return "" } -func (x *DependenciesServiceV2UpdateTaskLogRequest) GetContent() string { +func (x *DependenciesServiceV2UpdateTaskLogRequest) GetLogLines() []string { if x != nil { - return x.Content + return x.LogLines } - return "" + return nil } var File_services_dependencies_service_v2_proto protoreflect.FileDescriptor @@ -410,36 +410,36 @@ var file_services_dependencies_service_v2_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, - 0x63, 0x69, 0x65, 0x73, 0x22, 0x5e, 0x0a, 0x29, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x22, 0x61, 0x0a, 0x29, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x2a, 0x41, 0x0a, 0x19, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, - 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, - 0x65, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x59, 0x4e, 0x43, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, - 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x49, 0x4e, - 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x10, 0x02, 0x32, 0x97, 0x02, 0x0a, 0x15, 0x44, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, - 0x32, 0x12, 0x66, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x29, 0x2e, 0x67, - 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, + 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6c, 0x6f, + 0x67, 0x5f, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6c, + 0x6f, 0x67, 0x4c, 0x69, 0x6e, 0x65, 0x73, 0x2a, 0x41, 0x0a, 0x19, 0x44, 0x65, 0x70, 0x65, 0x6e, + 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x43, 0x6f, 0x64, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x59, 0x4e, 0x43, 0x10, 0x00, 0x12, 0x0b, + 0x0a, 0x07, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x55, + 0x4e, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x10, 0x02, 0x32, 0x95, 0x02, 0x0a, 0x15, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x40, 0x0a, 0x04, 0x53, 0x79, 0x6e, - 0x63, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, - 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, - 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x12, 0x2f, 0x2e, 0x67, - 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, - 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, - 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, - 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x63, 0x65, 0x56, 0x32, 0x12, 0x64, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, + 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x40, 0x0a, 0x04, 0x53, 0x79, + 0x6e, 0x63, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, + 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, + 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x12, 0x2f, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x54, 0x61, 0x73, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x28, 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/grpc/dependencies_service_v2_grpc.pb.go b/grpc/dependencies_service_v2_grpc.pb.go index b96e3178e..e76d3bda4 100644 --- a/grpc/dependencies_service_v2_grpc.pb.go +++ b/grpc/dependencies_service_v2_grpc.pb.go @@ -22,7 +22,7 @@ const _ = grpc.SupportPackageIsVersion7 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type DependenciesServiceV2Client interface { - Connect(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) + Connect(ctx context.Context, in *DependenciesServiceV2ConnectRequest, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) UpdateTaskLog(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_UpdateTaskLogClient, error) } @@ -35,17 +35,22 @@ func NewDependenciesServiceV2Client(cc grpc.ClientConnInterface) DependenciesSer return &dependenciesServiceV2Client{cc} } -func (c *dependenciesServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) { +func (c *dependenciesServiceV2Client) Connect(ctx context.Context, in *DependenciesServiceV2ConnectRequest, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) { stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], "/grpc.DependenciesServiceV2/Connect", opts...) if err != nil { return nil, err } x := &dependenciesServiceV2ConnectClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } return x, nil } type DependenciesServiceV2_ConnectClient interface { - Send(*DependenciesServiceV2ConnectRequest) error Recv() (*DependenciesServiceV2ConnectResponse, error) grpc.ClientStream } @@ -54,10 +59,6 @@ type dependenciesServiceV2ConnectClient struct { grpc.ClientStream } -func (x *dependenciesServiceV2ConnectClient) Send(m *DependenciesServiceV2ConnectRequest) error { - return x.ClientStream.SendMsg(m) -} - func (x *dependenciesServiceV2ConnectClient) Recv() (*DependenciesServiceV2ConnectResponse, error) { m := new(DependenciesServiceV2ConnectResponse) if err := x.ClientStream.RecvMsg(m); err != nil { @@ -113,7 +114,7 @@ func (x *dependenciesServiceV2UpdateTaskLogClient) CloseAndRecv() (*Response, er // All implementations must embed UnimplementedDependenciesServiceV2Server // for forward compatibility type DependenciesServiceV2Server interface { - Connect(DependenciesServiceV2_ConnectServer) error + Connect(*DependenciesServiceV2ConnectRequest, DependenciesServiceV2_ConnectServer) error Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) UpdateTaskLog(DependenciesServiceV2_UpdateTaskLogServer) error mustEmbedUnimplementedDependenciesServiceV2Server() @@ -123,7 +124,7 @@ type DependenciesServiceV2Server interface { type UnimplementedDependenciesServiceV2Server struct { } -func (UnimplementedDependenciesServiceV2Server) Connect(DependenciesServiceV2_ConnectServer) error { +func (UnimplementedDependenciesServiceV2Server) Connect(*DependenciesServiceV2ConnectRequest, DependenciesServiceV2_ConnectServer) error { return status.Errorf(codes.Unimplemented, "method Connect not implemented") } func (UnimplementedDependenciesServiceV2Server) Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) { @@ -146,12 +147,15 @@ func RegisterDependenciesServiceV2Server(s grpc.ServiceRegistrar, srv Dependenci } func _DependenciesServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependenciesServiceV2Server).Connect(&dependenciesServiceV2ConnectServer{stream}) + m := new(DependenciesServiceV2ConnectRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{stream}) } type DependenciesServiceV2_ConnectServer interface { Send(*DependenciesServiceV2ConnectResponse) error - Recv() (*DependenciesServiceV2ConnectRequest, error) grpc.ServerStream } @@ -163,14 +167,6 @@ func (x *dependenciesServiceV2ConnectServer) Send(m *DependenciesServiceV2Connec return x.ServerStream.SendMsg(m) } -func (x *dependenciesServiceV2ConnectServer) Recv() (*DependenciesServiceV2ConnectRequest, error) { - m := new(DependenciesServiceV2ConnectRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DependenciesServiceV2SyncRequest) if err := dec(in); err != nil { @@ -232,7 +228,6 @@ var DependenciesServiceV2_ServiceDesc = grpc.ServiceDesc{ StreamName: "Connect", Handler: _DependenciesServiceV2_Connect_Handler, ServerStreams: true, - ClientStreams: true, }, { StreamName: "UpdateTaskLog", diff --git a/grpc/dependencies_service_v2_request.pb.go b/grpc/dependencies_service_v2_request.pb.go deleted file mode 100644 index cae03422f..000000000 --- a/grpc/dependencies_service_v2_request.pb.go +++ /dev/null @@ -1,488 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 -// source: entity/dependencies_service_v2_request.proto - -package grpc - -import ( - proto "github.com/golang/protobuf/proto" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - -type Dependency struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` -} - -func (x *Dependency) Reset() { - *x = Dependency{} - if protoimpl.UnsafeEnabled { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Dependency) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Dependency) ProtoMessage() {} - -func (x *Dependency) ProtoReflect() protoreflect.Message { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Dependency.ProtoReflect.Descriptor instead. -func (*Dependency) Descriptor() ([]byte, []int) { - return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{0} -} - -func (x *Dependency) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *Dependency) GetVersion() string { - if x != nil { - return x.Version - } - return "" -} - -type DependenciesServiceV2ConnectRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` -} - -func (x *DependenciesServiceV2ConnectRequest) Reset() { - *x = DependenciesServiceV2ConnectRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DependenciesServiceV2ConnectRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DependenciesServiceV2ConnectRequest) ProtoMessage() {} - -func (x *DependenciesServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DependenciesServiceV2ConnectRequest.ProtoReflect.Descriptor instead. -func (*DependenciesServiceV2ConnectRequest) Descriptor() ([]byte, []int) { - return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{1} -} - -func (x *DependenciesServiceV2ConnectRequest) GetNodeKey() string { - if x != nil { - return x.NodeKey - } - return "" -} - -type DependenciesServiceV2SyncRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` - Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` - Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` -} - -func (x *DependenciesServiceV2SyncRequest) Reset() { - *x = DependenciesServiceV2SyncRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DependenciesServiceV2SyncRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DependenciesServiceV2SyncRequest) ProtoMessage() {} - -func (x *DependenciesServiceV2SyncRequest) ProtoReflect() protoreflect.Message { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DependenciesServiceV2SyncRequest.ProtoReflect.Descriptor instead. -func (*DependenciesServiceV2SyncRequest) Descriptor() ([]byte, []int) { - return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{2} -} - -func (x *DependenciesServiceV2SyncRequest) GetNodeKey() string { - if x != nil { - return x.NodeKey - } - return "" -} - -func (x *DependenciesServiceV2SyncRequest) GetLang() string { - if x != nil { - return x.Lang - } - return "" -} - -func (x *DependenciesServiceV2SyncRequest) GetDependencies() []*Dependency { - if x != nil { - return x.Dependencies - } - return nil -} - -type DependenciesServiceV2InstallRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` - Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` - Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` - Proxy string `protobuf:"bytes,4,opt,name=proxy,proto3" json:"proxy,omitempty"` -} - -func (x *DependenciesServiceV2InstallRequest) Reset() { - *x = DependenciesServiceV2InstallRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DependenciesServiceV2InstallRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DependenciesServiceV2InstallRequest) ProtoMessage() {} - -func (x *DependenciesServiceV2InstallRequest) ProtoReflect() protoreflect.Message { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DependenciesServiceV2InstallRequest.ProtoReflect.Descriptor instead. -func (*DependenciesServiceV2InstallRequest) Descriptor() ([]byte, []int) { - return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{3} -} - -func (x *DependenciesServiceV2InstallRequest) GetNodeKey() string { - if x != nil { - return x.NodeKey - } - return "" -} - -func (x *DependenciesServiceV2InstallRequest) GetLang() string { - if x != nil { - return x.Lang - } - return "" -} - -func (x *DependenciesServiceV2InstallRequest) GetDependencies() []*Dependency { - if x != nil { - return x.Dependencies - } - return nil -} - -func (x *DependenciesServiceV2InstallRequest) GetProxy() string { - if x != nil { - return x.Proxy - } - return "" -} - -type DependenciesServiceV2UninstallRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` - Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` - Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` -} - -func (x *DependenciesServiceV2UninstallRequest) Reset() { - *x = DependenciesServiceV2UninstallRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DependenciesServiceV2UninstallRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DependenciesServiceV2UninstallRequest) ProtoMessage() {} - -func (x *DependenciesServiceV2UninstallRequest) ProtoReflect() protoreflect.Message { - mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DependenciesServiceV2UninstallRequest.ProtoReflect.Descriptor instead. -func (*DependenciesServiceV2UninstallRequest) Descriptor() ([]byte, []int) { - return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{4} -} - -func (x *DependenciesServiceV2UninstallRequest) GetNodeKey() string { - if x != nil { - return x.NodeKey - } - return "" -} - -func (x *DependenciesServiceV2UninstallRequest) GetLang() string { - if x != nil { - return x.Lang - } - return "" -} - -func (x *DependenciesServiceV2UninstallRequest) GetDependencies() []*Dependency { - if x != nil { - return x.Dependencies - } - return nil -} - -var File_entity_dependencies_service_v2_request_proto protoreflect.FileDescriptor - -var file_entity_dependencies_service_v2_request_proto_rawDesc = []byte{ - 0x0a, 0x2c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, - 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, - 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, - 0x67, 0x72, 0x70, 0x63, 0x22, 0x3a, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, - 0x63, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x22, 0x40, 0x0a, 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, - 0x65, 0x79, 0x22, 0x87, 0x01, 0x0a, 0x20, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, - 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, - 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, - 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, - 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, - 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x22, 0xa0, 0x01, 0x0a, - 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, - 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, - 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, - 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, 0x64, 0x65, 0x70, - 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x22, - 0x8c, 0x01, 0x0a, 0x25, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, - 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, - 0x65, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, - 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, - 0x52, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x42, 0x08, - 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_entity_dependencies_service_v2_request_proto_rawDescOnce sync.Once - file_entity_dependencies_service_v2_request_proto_rawDescData = file_entity_dependencies_service_v2_request_proto_rawDesc -) - -func file_entity_dependencies_service_v2_request_proto_rawDescGZIP() []byte { - file_entity_dependencies_service_v2_request_proto_rawDescOnce.Do(func() { - file_entity_dependencies_service_v2_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_dependencies_service_v2_request_proto_rawDescData) - }) - return file_entity_dependencies_service_v2_request_proto_rawDescData -} - -var file_entity_dependencies_service_v2_request_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_entity_dependencies_service_v2_request_proto_goTypes = []interface{}{ - (*Dependency)(nil), // 0: grpc.Dependency - (*DependenciesServiceV2ConnectRequest)(nil), // 1: grpc.DependenciesServiceV2ConnectRequest - (*DependenciesServiceV2SyncRequest)(nil), // 2: grpc.DependenciesServiceV2SyncRequest - (*DependenciesServiceV2InstallRequest)(nil), // 3: grpc.DependenciesServiceV2InstallRequest - (*DependenciesServiceV2UninstallRequest)(nil), // 4: grpc.DependenciesServiceV2UninstallRequest -} -var file_entity_dependencies_service_v2_request_proto_depIdxs = []int32{ - 0, // 0: grpc.DependenciesServiceV2SyncRequest.dependencies:type_name -> grpc.Dependency - 0, // 1: grpc.DependenciesServiceV2InstallRequest.dependencies:type_name -> grpc.Dependency - 0, // 2: grpc.DependenciesServiceV2UninstallRequest.dependencies:type_name -> grpc.Dependency - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name -} - -func init() { file_entity_dependencies_service_v2_request_proto_init() } -func file_entity_dependencies_service_v2_request_proto_init() { - if File_entity_dependencies_service_v2_request_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_entity_dependencies_service_v2_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Dependency); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_entity_dependencies_service_v2_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DependenciesServiceV2ConnectRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_entity_dependencies_service_v2_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DependenciesServiceV2SyncRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_entity_dependencies_service_v2_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DependenciesServiceV2InstallRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_entity_dependencies_service_v2_request_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DependenciesServiceV2UninstallRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_entity_dependencies_service_v2_request_proto_rawDesc, - NumEnums: 0, - NumMessages: 5, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_entity_dependencies_service_v2_request_proto_goTypes, - DependencyIndexes: file_entity_dependencies_service_v2_request_proto_depIdxs, - MessageInfos: file_entity_dependencies_service_v2_request_proto_msgTypes, - }.Build() - File_entity_dependencies_service_v2_request_proto = out.File - file_entity_dependencies_service_v2_request_proto_rawDesc = nil - file_entity_dependencies_service_v2_request_proto_goTypes = nil - file_entity_dependencies_service_v2_request_proto_depIdxs = nil -} diff --git a/grpc/proto/services/dependencies_service_v2.proto b/grpc/proto/services/dependencies_service_v2.proto index b09f9c940..6f4e0578e 100644 --- a/grpc/proto/services/dependencies_service_v2.proto +++ b/grpc/proto/services/dependencies_service_v2.proto @@ -36,11 +36,11 @@ message DependenciesServiceV2SyncRequest { message DependenciesServiceV2UpdateTaskLogRequest { string task_id = 1; - string content = 2; + repeated string log_lines = 2; } service DependenciesServiceV2 { - rpc Connect(stream DependenciesServiceV2ConnectRequest) returns (stream DependenciesServiceV2ConnectResponse){}; + rpc Connect(DependenciesServiceV2ConnectRequest) returns (stream DependenciesServiceV2ConnectResponse){}; rpc Sync(DependenciesServiceV2SyncRequest) returns (Response){}; rpc UpdateTaskLog(stream DependenciesServiceV2UpdateTaskLogRequest) returns (Response){}; } From 1a3411b91ff43bef97f762f537afb722d52f77d1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 16 Jun 2024 12:24:48 +0800 Subject: [PATCH 011/106] feat: migrated git from spider --- core/controllers/router_v2.go | 27 +-- core/controllers/spider_v2.go | 429 ---------------------------------- 2 files changed, 1 insertion(+), 455 deletions(-) diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 91e42ae8b..908368bdf 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -58,7 +58,6 @@ func InitRoutes(app *gin.Engine) (err error) { RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]()) RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]()) RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) - RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]()) RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]()) RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) @@ -172,31 +171,7 @@ func InitRoutes(app *gin.Engine) (err error) { Path: "/:id/run", HandlerFunc: PostSpiderRun, }, - Action{ - Method: http.MethodGet, - Path: "/:id/git", - HandlerFunc: GetSpiderGit, - }, - Action{ - Method: http.MethodGet, - Path: "/:id/git/remote-refs", - HandlerFunc: GetSpiderGitRemoteRefs, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/checkout", - HandlerFunc: PostSpiderGitCheckout, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/pull", - HandlerFunc: PostSpiderGitPull, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/commit", - HandlerFunc: PostSpiderGitCommit, - }, + Action{ Method: http.MethodGet, Path: "/:id/data-source", diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 9e81c6a28..11f1c4913 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -5,19 +5,13 @@ import ( "fmt" log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" - "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" "github.com/gin-gonic/gin" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -26,7 +20,6 @@ import ( "math" "os" "path/filepath" - "strings" "sync" ) @@ -756,266 +749,6 @@ func PostSpiderRun(c *gin.Context) { HandleSuccessWithData(c, taskIds) } -func GetSpiderGit(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // branches - branches, err := gitClient.GetBranches() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if branches == nil || len(branches) == 0 && currentBranch != "" { - branches = []vcs.GitRef{{Name: currentBranch}} - } - - // changes - changes, err := gitClient.GetStatus() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // logs - logs, err := gitClient.GetLogsWithRefs() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // ignore - ignore, err := getSpiderGitIgnore(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // git - _git, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) - return - } - } - - // response - res := bson.M{ - "current_branch": currentBranch, - "branches": branches, - "changes": changes, - "logs": logs, - "ignore": ignore, - "git": _git, - } - - HandleSuccessWithData(c, res) -} - -func GetSpiderGitRemoteRefs(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // remote name - remoteName := c.Query("remote") - if remoteName == "" { - remoteName = vcs.GitRemoteNameOrigin - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // refs - refs, err := gitClient.GetRemoteRefs(remoteName) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, refs) -} - -func PostSpiderGitCheckout(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload struct { - Paths []string `json:"paths"` - CommitMessage string `json:"commit_message"` - Branch string `json:"branch"` - Tag string `json:"tag"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // branch to pull - var branch string - if payload.Branch == "" { - // by default current branch - branch, err = gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - // payload branch - branch = payload.Branch - } - - // checkout - if err := gitSpiderCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func PostSpiderGitPull(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload struct { - Paths []string `json:"paths"` - CommitMessage string `json:"commit_message"` - Branch string `json:"branch"` - Tag string `json:"tag"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git - g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // attempt to sync git - adminSvc, err := admin.GetSpiderAdminServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - _ = adminSvc.SyncGitOne(g) - - HandleSuccess(c) -} - -func PostSpiderGitCommit(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // add - for _, p := range payload.Paths { - if err := gitClient.Add(p); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // commit - if err := gitClient.Commit(payload.CommitMessage); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // push - if err := gitClient.Push( - vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), - ); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - func GetSpiderDataSource(c *gin.Context) { // spider id id, err := primitive.ObjectIDFromHex(c.Param("id")) @@ -1107,168 +840,6 @@ func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 { return fsSvc } -func getSpiderGitClient(id primitive.ObjectID) (client *vcs.GitClient, err error) { - // git - g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - if !errors.Is(err, mongo2.ErrNoDocuments) { - return nil, trace.TraceError(err) - } - return nil, nil - } - - // git client - workspacePath := viper.GetString("workspace") - client, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) - if err != nil { - return nil, err - } - - // set auth - utils.InitGitClientAuthV2(g, client) - - // remote name - remoteName := vcs.GitRemoteNameOrigin - - // update remote - r, err := client.GetRemote(remoteName) - if errors.Is(err, git.ErrRemoteNotFound) { - // remote not exists, create - if _, err := client.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } else if err == nil { - // remote exists, update if different - if g.Url != r.Config().URLs[0] { - if err := client.DeleteRemote(remoteName); err != nil { - return nil, trace.TraceError(err) - } - if _, err := client.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } - client.SetRemoteUrl(g.Url) - } else { - // error - return nil, trace.TraceError(err) - } - - // check if head reference exists - _, err = client.GetRepository().Head() - if err == nil { - return client, nil - } - - // align master/main branch - alignSpiderGitBranch(client) - - return client, nil -} - -func alignSpiderGitBranch(gitClient *vcs.GitClient) { - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - trace.PrintError(err) - return - } - - // skip if current branch is not master - if currentBranch != vcs.GitBranchNameMaster { - return - } - - // remote refs - refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) - if err != nil { - trace.PrintError(err) - return - } - - // main branch - defaultRemoteBranch, err := getSpiderDefaultRemoteBranch(refs) - if err != nil || defaultRemoteBranch == "" { - return - } - - // move branch - if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { - trace.PrintError(err) - } -} - -func getSpiderDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { - // remote branch name - for _, r := range refs { - if r.Type != vcs.GitRefTypeBranch { - continue - } - - if r.Name == vcs.GitBranchNameMain { - defaultRemoteBranchName = r.Name - break - } - - if r.Name == vcs.GitBranchNameMaster { - defaultRemoteBranchName = r.Name - continue - } - - if defaultRemoteBranchName == "" { - defaultRemoteBranchName = r.Name - continue - } - } - - return defaultRemoteBranchName, nil -} - -func getSpiderGitIgnore(id primitive.ObjectID) (ignore []string, err error) { - workspacePath := viper.GetString("workspace") - filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") - if !utils.Exists(filePath) { - return nil, nil - } - data, err := os.ReadFile(filePath) - if err != nil { - return nil, trace.TraceError(err) - } - ignore = strings.Split(string(data), "\n") - return ignore, nil -} - -func gitSpiderCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { - if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { - return trace.TraceError(err) - } - - // pull - return spiderGitPull(gitClient, remote, branch) -} - -func spiderGitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { - // pull - if err := gitClient.Pull( - vcs.WithRemoteNamePull(remote), - vcs.WithBranchNamePull(branch), - ); err != nil { - return trace.TraceError(err) - } - - // reset - if err := gitClient.Reset(); err != nil { - return trace.TraceError(err) - } - - return nil -} - func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { modelSvc := service.NewModelServiceV2[models.DataCollectionV2]() if s.ColId.IsZero() { From 3727ea6970acbd1815cfaedfb9d27f3b181ab6c5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Jun 2024 17:30:23 +0800 Subject: [PATCH 012/106] fix: deps issue --- backend/go.mod | 4 +--- backend/go.sum | 6 ------ core/go.mod | 2 -- core/go.sum | 6 ------ fs/go.mod | 3 +-- fs/go.sum | 7 ------- fs/utils.go | 2 +- go.work.sum | 9 +++++++-- 8 files changed, 10 insertions(+), 29 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index bfc6cc274..5ade919d0 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -39,11 +39,10 @@ require ( github.com/cloudflare/circl v1.3.7 // indirect github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 // indirect - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614111723-e5b20af9a40b // indirect github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 // indirect - github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect github.com/elastic/go-elasticsearch/v8 v8.7.0 // indirect @@ -90,7 +89,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect - github.com/linxGnu/gumble v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matcornic/hermes/v2 v2.1.0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect diff --git a/backend/go.sum b/backend/go.sum index 63fb69cd8..7de11f847 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -146,8 +146,6 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -482,8 +480,6 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= -github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -604,7 +600,6 @@ github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWR github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= @@ -701,7 +696,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= -github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe h1:9YnI5plmy+ad6BM+JCLJb2ZV7/TNiE5l7SNKfumYKgc= diff --git a/core/go.mod b/core/go.mod index 98fc835d8..cab919688 100644 --- a/core/go.mod +++ b/core/go.mod @@ -70,7 +70,6 @@ require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect @@ -117,7 +116,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect - github.com/linxGnu/gumble v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect diff --git a/core/go.sum b/core/go.sum index 19a5c9d56..12f6ea4cd 100644 --- a/core/go.sum +++ b/core/go.sum @@ -148,8 +148,6 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -489,8 +487,6 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= -github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -616,7 +612,6 @@ github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWR github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= @@ -716,7 +711,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= -github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= diff --git a/fs/go.mod b/fs/go.mod index daadb5f52..0cf9826b6 100644 --- a/fs/go.mod +++ b/fs/go.mod @@ -8,7 +8,7 @@ require ( github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 - github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 + github.com/crawlab-team/goseaweedfs v0.6.3 github.com/emirpasic/gods v1.18.1 github.com/google/uuid v1.6.0 github.com/stretchr/testify v1.9.0 @@ -17,7 +17,6 @@ require ( require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/kr/pretty v0.3.1 // indirect - github.com/linxGnu/gumble v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect diff --git a/fs/go.sum b/fs/go.sum index fcf3ee813..da7b0b402 100644 --- a/fs/go.sum +++ b/fs/go.sum @@ -7,8 +7,6 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -30,8 +28,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= -github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -46,14 +42,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -63,7 +57,6 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/fs/utils.go b/fs/utils.go index a34382bae..c17a461b8 100644 --- a/fs/utils.go +++ b/fs/utils.go @@ -41,7 +41,7 @@ func getFilesAndFilesMaps(f *goseaweedfs.Filer, localPath, remotePath string) (l remoteFilesMap = map[string]goseaweedfs.FilerFileInfo{} // cache local files info - localFiles, err = goseaweedfs.ListLocalFilesRecursive(localPath) + localFiles, err = goseaweedfs.ListFilesRecursive(localPath) if err != nil { return localFiles, remoteFiles, localFilesMap, remoteFilesMap, err } diff --git a/go.work.sum b/go.work.sum index 9c9482bac..1fdba63d3 100644 --- a/go.work.sum +++ b/go.work.sum @@ -562,6 +562,8 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= +github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -596,6 +598,7 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= @@ -606,6 +609,7 @@ github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -680,7 +684,6 @@ github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sL github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= @@ -704,6 +707,7 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= @@ -714,12 +718,14 @@ github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/lyft/protoc-gen-star v0.5.3 h1:zSGLzsUew8RT+ZKPHc3jnf8XLaVyHzTcAFBzHtCNR20= github.com/lyft/protoc-gen-star/v2 v2.0.3 h1:/3+/2sWyXeMLzKd1bX+ixWKgEMsULrIivpDsuaF441o= github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= @@ -858,7 +864,6 @@ go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzc go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= From 41e7edf9f9e58b7ed4511a1d63ae73fb78160604 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 17 Jun 2024 23:31:34 +0800 Subject: [PATCH 013/106] feat: optimized code --- core/controllers/base_file_v2.go | 305 +++++++++++++++++++++++++++++++ core/controllers/spider_v2.go | 257 +++++--------------------- core/fs/service_v2.go | 11 ++ core/interfaces/fs_service_v2.go | 1 + core/models/models/spider_v2.go | 10 +- core/spider/admin/service_v2.go | 23 --- 6 files changed, 364 insertions(+), 243 deletions(-) create mode 100644 core/controllers/base_file_v2.go diff --git a/core/controllers/base_file_v2.go b/core/controllers/base_file_v2.go new file mode 100644 index 000000000..6c398abdd --- /dev/null +++ b/core/controllers/base_file_v2.go @@ -0,0 +1,305 @@ +package controllers + +import ( + "errors" + "fmt" + log2 "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "os" + "path/filepath" + "sync" +) + +func GetBaseFileListDir(rootPath string, c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + files, err := fsSvc.List(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccessWithData(c, files) +} + +func GetBaseFileFile(rootPath string, c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + data, err := fsSvc.GetFile(path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, string(data)) +} + +func GetBaseFileFileInfo(rootPath string, c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + info, err := fsSvc.GetFileInfo(path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, info) +} + +func PostBaseFileSaveFile(rootPath string, c *gin.Context) { + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + if c.GetHeader("Content-Type") == "application/json" { + var payload struct { + Path string `json:"path"` + Data string `json:"data"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } else { + path, ok := c.GetPostForm("path") + if !ok { + HandleErrorBadRequest(c, errors.New("missing required field 'path'")) + return + } + file, err := c.FormFile("file") + if err != nil { + HandleErrorBadRequest(c, err) + return + } + f, err := file.Open() + if err != nil { + HandleErrorBadRequest(c, err) + return + } + fileData, err := io.ReadAll(f) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := fsSvc.Save(path, fileData); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccess(c) +} + +func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + form, err := c.MultipartForm() + if err != nil { + HandleErrorBadRequest(c, err) + return + } + wg := sync.WaitGroup{} + wg.Add(len(form.File)) + for path := range form.File { + go func(path string) { + file, err := c.FormFile(path) + if err != nil { + log2.Warnf("invalid file header: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + f, err := file.Open() + if err != nil { + log2.Warnf("unable to open file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + fileData, err := io.ReadAll(f) + if err != nil { + log2.Warnf("unable to read file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + if err := fsSvc.Save(path, fileData); err != nil { + log2.Warnf("unable to save file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + wg.Done() + }(path) + } + wg.Wait() + + HandleSuccess(c) +} + +func PostBaseFileSaveDir(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + Data string `json:"data"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.CreateDir(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostBaseFileRenameFile(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } +} + +func DeleteBaseFileFile(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if payload.Path == "~" { + payload.Path = "." + } + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Delete(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + _, err = fsSvc.GetFileInfo(".") + if err != nil { + _ = fsSvc.CreateDir("/") + } + + HandleSuccess(c) +} + +func PostBaseFileCopyFile(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostBaseFileExport(rootPath string, c *gin.Context) { + fsSvc, err := getBaseFileFsSvc(rootPath, c) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // zip file path + zipFilePath, err := fsSvc.Export() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // download + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) + c.File(zipFilePath) +} + +func getBaseFileFsSvc(rootPath string, c *gin.Context) (svc interfaces.FsServiceV2, err error) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + return nil, err + } + + workspacePath := viper.GetString("workspace") + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex(), rootPath)) + + return fsSvc, nil +} diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 11f1c4913..9b28b94df 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -2,7 +2,6 @@ package controllers import ( "errors" - "fmt" log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/fs" @@ -10,13 +9,13 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" + "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" "math" "os" "path/filepath" @@ -432,286 +431,93 @@ func DeleteSpiderList(c *gin.Context) { } func GetSpiderListDir(c *gin.Context) { - path := c.Query("path") - - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorInternalServerError(c, err) return } - - files, err := fsSvc.List(path) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccessWithData(c, files) + GetBaseFileListDir(s.GitRootPath, c) } func GetSpiderFile(c *gin.Context) { - path := c.Query("path") - - fsSvc, err := getSpiderFsSvc(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - data, err := fsSvc.GetFile(path) + s, err := allowSpiderGit(c) if err != nil { HandleErrorInternalServerError(c, err) return } - - HandleSuccessWithData(c, string(data)) + GetBaseFileFile(s.GitRootPath, c) } func GetSpiderFileInfo(c *gin.Context) { - path := c.Query("path") - - fsSvc, err := getSpiderFsSvc(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - info, err := fsSvc.GetFileInfo(path) + s, err := allowSpiderGit(c) if err != nil { HandleErrorInternalServerError(c, err) return } - - HandleSuccessWithData(c, info) + GetBaseFileFileInfo(s.GitRootPath, c) } func PostSpiderSaveFile(c *gin.Context) { - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { HandleErrorInternalServerError(c, err) return } - - if c.GetHeader("Content-Type") == "application/json" { - var payload struct { - Path string `json:"path"` - Data string `json:"data"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - path, ok := c.GetPostForm("path") - if !ok { - HandleErrorBadRequest(c, errors.New("missing required field 'path'")) - return - } - file, err := c.FormFile("file") - if err != nil { - HandleErrorBadRequest(c, err) - return - } - f, err := file.Open() - if err != nil { - HandleErrorBadRequest(c, err) - return - } - fileData, err := io.ReadAll(f) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(path, fileData); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) + PostBaseFileSaveFile(s.GitRootPath, c) } func PostSpiderSaveFiles(c *gin.Context) { - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { HandleErrorInternalServerError(c, err) return } - - form, err := c.MultipartForm() - if err != nil { - HandleErrorBadRequest(c, err) - return - } - wg := sync.WaitGroup{} - wg.Add(len(form.File)) - for path := range form.File { - go func(path string) { - file, err := c.FormFile(path) - if err != nil { - log2.Warnf("invalid file header: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - f, err := file.Open() - if err != nil { - log2.Warnf("unable to open file: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - fileData, err := io.ReadAll(f) - if err != nil { - log2.Warnf("unable to read file: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - if err := fsSvc.Save(path, fileData); err != nil { - log2.Warnf("unable to save file: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - wg.Done() - }(path) - } - wg.Wait() - - HandleSuccess(c) + PostBaseFileSaveFiles(s.GitRootPath, c) } func PostSpiderSaveDir(c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - Data string `json:"data"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.CreateDir(payload.Path); err != nil { HandleErrorInternalServerError(c, err) return } - - HandleSuccess(c) + PostBaseFileSaveDir(s.GitRootPath, c) } func PostSpiderRenameFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { HandleErrorInternalServerError(c, err) return } + PostBaseFileRenameFile(s.GitRootPath, c) } func DeleteSpiderFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if payload.Path == "~" { - payload.Path = "." - } - - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Delete(payload.Path); err != nil { HandleErrorInternalServerError(c, err) return } - _, err = fsSvc.GetFileInfo(".") - if err != nil { - _ = fsSvc.CreateDir("/") - } - - HandleSuccess(c) + DeleteBaseFileFile(s.GitRootPath, c) } func PostSpiderCopyFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := getSpiderFsSvc(c) + s, err := allowSpiderGit(c) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { HandleErrorInternalServerError(c, err) return } - - HandleSuccess(c) + PostBaseFileCopyFile(s.GitRootPath, c) } func PostSpiderExport(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - adminSvc, err := admin.GetSpiderAdminServiceV2() + s, err := allowSpiderGit(c) if err != nil { HandleErrorInternalServerError(c, err) return } - - // zip file path - zipFilePath, err := adminSvc.Export(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // download - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) - c.File(zipFilePath) + PostBaseFileExport(s.GitRootPath, c) } func PostSpiderRun(c *gin.Context) { @@ -878,3 +684,22 @@ func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { } return nil } + +func allowSpiderGit(c *gin.Context) (s models.SpiderV2, err error) { + if utils.IsPro() { + return s, nil + } + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + return s, err + } + _s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + return s, err + } + if s.GitId.IsZero() { + return s, errors.New("git is not allowed in this edition") + } + s = *_s + return s, nil +} diff --git a/core/fs/service_v2.go b/core/fs/service_v2.go index 6ca1f057c..918a35938 100644 --- a/core/fs/service_v2.go +++ b/core/fs/service_v2.go @@ -4,6 +4,8 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/trace" + "github.com/google/uuid" "io" "os" "path/filepath" @@ -159,6 +161,15 @@ func (svc *ServiceV2) Copy(path, newPath string) (err error) { } } +func (svc *ServiceV2) Export() (resultPath string, err error) { + zipFilePath := filepath.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(svc.rootPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + func NewFsServiceV2(path string) (svc interfaces.FsServiceV2) { return &ServiceV2{ rootPath: path, diff --git a/core/interfaces/fs_service_v2.go b/core/interfaces/fs_service_v2.go index 4bc3d7df6..533a9d6b6 100644 --- a/core/interfaces/fs_service_v2.go +++ b/core/interfaces/fs_service_v2.go @@ -9,4 +9,5 @@ type FsServiceV2 interface { Rename(path, newPath string) (err error) Delete(path string) (err error) Copy(path, newPath string) (err error) + Export() (resultPath string, err error) } diff --git a/core/models/models/spider_v2.go b/core/models/models/spider_v2.go index 4f98f6e79..939bad519 100644 --- a/core/models/models/spider_v2.go +++ b/core/models/models/spider_v2.go @@ -17,14 +17,16 @@ type SpiderV2 struct { ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id Mode string `json:"mode" bson:"mode"` // default Task.Mode NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *GitV2 `json:"git,omitempty" bson:"-"` + + // stats + Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` // execution Cmd string `json:"cmd" bson:"cmd"` // execute command Param string `json:"param" bson:"param"` // default task param Priority int `json:"priority" bson:"priority"` AutoInstall bool `json:"auto_install" bson:"auto_install"` - - // settings - IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files } diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index c62a5fee9..628a7f485 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -14,13 +14,10 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" "github.com/crawlab-team/crawlab/vcs" - "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "os" - "path" "path/filepath" "sync" "time" @@ -60,26 +57,6 @@ func (svc *ServiceV2) SyncGit() (err error) { return nil } -func (svc *ServiceV2) SyncGitOne(g *models.GitV2) (err error) { - svc.syncGitOne(g) - return nil -} - -func (svc *ServiceV2) Export(id primitive.ObjectID) (filePath string, err error) { - // spider fs - workspacePath := viper.GetString("workspace") - spiderFolderPath := filepath.Join(workspacePath, id.Hex()) - - // zip files in workspace - dirPath := spiderFolderPath - zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") - if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { - return "", trace.TraceError(err) - } - - return zipFilePath, nil -} - func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task mainTask := &models.TaskV2{ From 9d21bad931a9ba837ada9ae7d4ac538924209a89 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 18 Jun 2024 18:16:26 +0800 Subject: [PATCH 014/106] fix: file upload issue --- core/go.mod | 9 +++++++-- core/go.sum | 15 +++++++++++++++ go.work.sum | 13 ++----------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/core/go.mod b/core/go.mod index cab919688..9212fd2b5 100644 --- a/core/go.mod +++ b/core/go.mod @@ -21,7 +21,7 @@ require ( github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 - github.com/elastic/go-elasticsearch/v8 v8.7.0 + github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 github.com/gavv/httpexpect/v2 v2.16.0 @@ -72,13 +72,15 @@ require ( github.com/cloudflare/circl v1.3.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect - github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -168,6 +170,9 @@ require ( github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/goleak v1.1.11 // indirect go.uber.org/multierr v1.9.0 // indirect diff --git a/core/go.sum b/core/go.sum index 12f6ea4cd..b86e23b82 100644 --- a/core/go.sum +++ b/core/go.sum @@ -161,8 +161,12 @@ github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= +github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -220,6 +224,11 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -768,6 +777,12 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/go.work.sum b/go.work.sum index 1fdba63d3..a07598f35 100644 --- a/go.work.sum +++ b/go.work.sum @@ -590,14 +590,9 @@ github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFV github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -843,13 +838,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1: go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= @@ -858,8 +850,6 @@ go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= @@ -921,6 +911,7 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= From b757826867ac0e3d3741bc4e758334f39fa6e041 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 19 Jun 2024 00:17:36 +0800 Subject: [PATCH 015/106] feat: added fields to models --- core/models/models/git_v2.go | 6 +- core/spider/admin/service_v2.go | 110 +------------------------ core/utils/git.go | 15 ---- vcs/git.go | 142 +++++++++++++------------------- vcs/git_options.go | 6 ++ 5 files changed, 67 insertions(+), 212 deletions(-) diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go index 1ccced39f..2351882ac 100644 --- a/core/models/models/git_v2.go +++ b/core/models/models/git_v2.go @@ -9,5 +9,9 @@ type GitV2 struct { Username string `json:"username" bson:"username"` Password string `json:"password" bson:"password"` CurrentBranch string `json:"current_branch" bson:"current_branch"` - AutoPull bool `json:"auto_pull" bson:"auto_pull"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + + // settings + AutoPull bool `json:"auto_pull" bson:"auto_pull"` } diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index 628a7f485..bd6d1e19f 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -1,8 +1,6 @@ package admin import ( - "context" - "github.com/apex/log" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" @@ -11,15 +9,10 @@ import ( "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/scheduler" - "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" "github.com/robfig/cron/v3" - "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "path/filepath" - "sync" "time" ) @@ -35,7 +28,7 @@ type ServiceV2 struct { } func (svc *ServiceV2) Start() (err error) { - return svc.SyncGit() + return nil } func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { @@ -49,14 +42,6 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun return svc.scheduleTasks(s, opts) } -func (svc *ServiceV2) SyncGit() (err error) { - if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { - return trace.TraceError(err) - } - svc.cron.Start() - return nil -} - func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task mainTask := &models.TaskV2{ @@ -175,99 +160,6 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) } } -func (svc *ServiceV2) syncGit() { - if svc.syncLock { - log.Infof("[SpiderAdminService] sync git is locked, skip") - return - } - log.Infof("[SpiderAdminService] start to sync git") - - svc.syncLock = true - defer func() { - svc.syncLock = false - }() - - // spiders - spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(nil, nil) - if err != nil { - trace.PrintError(err) - return - } - - // spider ids - var spiderIds []primitive.ObjectID - for _, s := range spiders { - spiderIds = append(spiderIds, s.Id) - } - - if len(spiderIds) > 0 { - // gits - gits, err := service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - "auto_pull": true, - }, nil) - if err != nil { - trace.PrintError(err) - return - } - - wg := sync.WaitGroup{} - wg.Add(len(gits)) - for _, g := range gits { - go func(g models.GitV2) { - svc.syncGitOne(&g) - wg.Done() - }(g) - } - wg.Wait() - } - - log.Infof("[SpiderAdminService] finished sync git") -} - -func (svc *ServiceV2) syncGitOne(g *models.GitV2) { - log.Infof("[SpiderAdminService] sync git %s", g.Id) - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - // git client - workspacePath := viper.GetString("workspace") - gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.Id.Hex()))) - if err != nil { - return - } - - // set auth - utils.InitGitClientAuthV2(g, gitClient) - - // check if remote has changes - ok, err := gitClient.IsRemoteChanged() - if err != nil { - trace.PrintError(err) - return - } - if !ok { - // no change - return - } - - // pull and sync to workspace - if err := gitClient.Reset(); err != nil { - trace.PrintError(err) - return - } - if err := gitClient.Pull(); err != nil { - trace.PrintError(err) - return - } - - // wait for context to end - <-ctx.Done() -} - func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { svc := &ServiceV2{ nodeCfgSvc: config.GetNodeConfigService(), diff --git a/core/utils/git.go b/core/utils/git.go index 475d2314c..272d06911 100644 --- a/core/utils/git.go +++ b/core/utils/git.go @@ -3,7 +3,6 @@ package utils import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" vcs "github.com/crawlab-team/crawlab/vcs" ) @@ -20,17 +19,3 @@ func InitGitClientAuth(g interfaces.Git, gitClient *vcs.GitClient) { gitClient.SetPrivateKey(g.GetPassword()) } } - -func InitGitClientAuthV2(g *models.GitV2, gitClient *vcs.GitClient) { - // set auth - switch g.AuthType { - case constants.GitAuthTypeHttp: - gitClient.SetAuthType(vcs.GitAuthTypeHTTP) - gitClient.SetUsername(g.Username) - gitClient.SetPassword(g.Password) - case constants.GitAuthTypeSsh: - gitClient.SetAuthType(vcs.GitAuthTypeSSH) - gitClient.SetUsername(g.Username) - gitClient.SetPrivateKey(g.Password) - } -} diff --git a/vcs/git.go b/vcs/git.go index 3ed5a4570..11267a472 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -1,6 +1,7 @@ package vcs import ( + "errors" "github.com/apex/log" "github.com/crawlab-team/crawlab/trace" "github.com/go-git/go-billy/v5" @@ -14,7 +15,7 @@ import ( gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/storage/memory" "golang.org/x/crypto/ssh" - "io/ioutil" + "io" "os" "path" "regexp" @@ -35,6 +36,7 @@ type GitClient struct { privateKey string privateKeyPath string defaultBranch string + defaultInit bool // internals r *git.Repository @@ -51,36 +53,6 @@ func (c *GitClient) Init() (err error) { if err != nil { return err } - - // if remote url is not empty and no remote exists - // create default remote and pull from remote url - remotes, err := c.r.Remotes() - if err != nil { - return err - } - if c.remoteUrl != "" && len(remotes) == 0 { - // attempt to get default remote - if _, err := c.r.Remote(GitRemoteNameOrigin); err != nil { - if err != git.ErrRemoteNotFound { - return trace.TraceError(err) - } - err = nil - - // create default remote - if err := c.createRemote(GitRemoteNameOrigin, c.remoteUrl); err != nil { - return err - } - - //// pull - //opts := []GitPullOption{ - // WithRemoteNamePull(GitRemoteNameOrigin), - //} - //if err := c.Pull(opts...); err != nil { - // return err - //} - } - } - return nil } @@ -97,6 +69,33 @@ func (c *GitClient) Dispose() (err error) { return nil } +func (c *GitClient) Clone() (err error) { + // validate + if c.remoteUrl == "" { + return ErrUnableToCloneWithEmptyRemoteUrl + } + + // auth + auth, err := c.getGitAuth() + if err != nil { + return err + } + + // options + o := &git.CloneOptions{ + URL: c.remoteUrl, + Auth: auth, + } + + // clone + _, err = git.PlainClone(c.path, false, o) + if err != nil { + return err + } + + return nil +} + func (c *GitClient) Checkout(opts ...GitCheckoutOption) (err error) { // worktree wt, err := c.r.Worktree() @@ -163,16 +162,16 @@ func (c *GitClient) Pull(opts ...GitPullOption) (err error) { // pull if err := wt.Pull(o); err != nil { - if err == transport.ErrEmptyRemoteRepository { + if errors.Is(err, transport.ErrEmptyRemoteRepository) { return nil } - if err == transport.ErrEmptyUploadPackRequest { + if errors.Is(err, transport.ErrEmptyUploadPackRequest) { return nil } - if err == git.NoErrAlreadyUpToDate { + if errors.Is(err, git.NoErrAlreadyUpToDate) { return nil } - if err == git.ErrNonFastForwardUpdate { + if errors.Is(err, git.ErrNonFastForwardUpdate) { return nil } return trace.TraceError(err) @@ -260,14 +259,14 @@ func (c *GitClient) CheckoutBranchWithRemote(branch, remote string, ref *plumbin } // check if the branch exists - b, err := c.r.Branch(branch) + _, err = c.r.Branch(branch) if err != nil { - if err == git.ErrBranchNotFound { + if errors.Is(err, git.ErrBranchNotFound) { // create a new branch if it does not exist if err := c.createBranch(branch, remote, ref); err != nil { return err } - b, err = c.r.Branch(branch) + _, err = c.r.Branch(branch) if err != nil { return trace.TraceError(err) } @@ -276,11 +275,6 @@ func (c *GitClient) CheckoutBranchWithRemote(branch, remote string, ref *plumbin } } - // set branch remote - if remote != "" { - b.Remote = remote - } - // add to options opts = append(opts, WithBranch(branch)) @@ -338,14 +332,14 @@ func (c *GitClient) GetLogs() (logs []GitLog, err error) { return nil, trace.TraceError(err) } if err := iter.ForEach(func(commit *object.Commit) error { - log := GitLog{ + gitLog := GitLog{ Hash: commit.Hash.String(), Msg: commit.Message, AuthorName: commit.Author.Name, AuthorEmail: commit.Author.Email, Timestamp: commit.Author.When, } - logs = append(logs, log) + logs = append(logs, gitLog) return nil }); err != nil { return nil, trace.TraceError(err) @@ -524,10 +518,14 @@ func (c *GitClient) GetBranches() (branches []GitRef, err error) { } func (c *GitClient) GetRemoteRefs(remoteName string) (gitRefs []GitRef, err error) { + if remoteName == "" { + remoteName = GitRemoteNameOrigin + } + // remote r, err := c.r.Remote(remoteName) if err != nil { - if err == git.ErrRemoteNotFound { + if errors.Is(err, git.ErrRemoteNotFound) { return nil, nil } return nil, trace.TraceError(err) @@ -542,7 +540,7 @@ func (c *GitClient) GetRemoteRefs(remoteName string) (gitRefs []GitRef, err erro // refs refs, err := r.List(&git.ListOptions{Auth: auth}) if err != nil { - if err != transport.ErrEmptyRemoteRepository { + if !errors.Is(err, transport.ErrEmptyRemoteRepository) { return nil, trace.TraceError(err) } return nil, nil @@ -699,7 +697,7 @@ func (c *GitClient) initMem() (err error) { // attempt to init c.r, err = git.Init(storage, wt) if err != nil { - if err == git.ErrRepositoryAlreadyExists { + if errors.Is(err, git.ErrRepositoryAlreadyExists) { // if already exists, attempt to open c.r, err = git.Open(storage, wt) if err != nil { @@ -730,43 +728,10 @@ func (c *GitClient) initFs() (err error) { // try to open repo c.r, err = git.PlainOpen(c.path) - if err == git.ErrRepositoryNotExists { - // repo not exists, init - c.r, err = git.PlainInit(c.path, false) - if err != nil { - return trace.TraceError(err) - } - } else if err != nil { - // error - return trace.TraceError(err) - } - - return nil -} - -func (c *GitClient) clone() (err error) { - // validate - if c.remoteUrl == "" { - return trace.TraceError(ErrUnableToCloneWithEmptyRemoteUrl) - } - - // auth - auth, err := c.getGitAuth() if err != nil { return err } - // options - o := &git.CloneOptions{ - URL: c.remoteUrl, - Auth: auth, - } - - // clone - if _, err := git.PlainClone(c.path, false, o); err != nil { - return trace.TraceError(err) - } - return nil } @@ -843,7 +808,7 @@ func (c *GitClient) getGitAuth() (auth transport.AuthMethod, err error) { privateKeyData = []byte(c.privateKey) } else if c.privateKeyPath != "" { // read from private key file - privateKeyData, err = ioutil.ReadFile(c.privateKeyPath) + privateKeyData, err = os.ReadFile(c.privateKeyPath) if err != nil { return nil, trace.TraceError(err) } @@ -877,7 +842,7 @@ func (c *GitClient) getHeadRef() (ref string, err error) { if err != nil { return "", trace.TraceError(err) } - data, err := ioutil.ReadAll(fh) + data, err := io.ReadAll(fh) if err != nil { return "", trace.TraceError(err) } @@ -933,7 +898,7 @@ func (c *GitClient) createBranch(branch, remote string, ref *plumbing.Reference) ref, err = c.getBranchHashRef(branch, remote) // if no matched remote branch, set to HEAD - if err == ErrNoMatchedRemoteBranch { + if errors.Is(err, ErrNoMatchedRemoteBranch) { ref, err = c.r.Head() if err != nil { return trace.TraceError(err) @@ -1003,6 +968,7 @@ func NewGitClient(opts ...GitOption) (c *GitClient, err error) { authType: GitAuthTypeNone, username: "git", privateKeyPath: getDefaultPublicKeyPath(), + defaultInit: true, } // apply options @@ -1010,9 +976,11 @@ func NewGitClient(opts ...GitOption) (c *GitClient, err error) { opt(c) } - // init - if err := c.Init(); err != nil { - return c, err + // initialize + if c.defaultInit { + if err = c.Init(); err != nil { + return nil, err + } } return diff --git a/vcs/git_options.go b/vcs/git_options.go index a990c1e1f..00152f965 100644 --- a/vcs/git_options.go +++ b/vcs/git_options.go @@ -59,6 +59,12 @@ func WithDefaultBranch(branch string) GitOption { } } +func WithDefaultInit(init bool) GitOption { + return func(c *GitClient) { + c.defaultInit = init + } +} + func WithPrivateKeyPath(path string) GitOption { return func(c *GitClient) { c.privateKeyPath = path From 7c42d68ad79b318f4af5b99eac3f947cd2e3156f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 19 Jun 2024 15:50:30 +0800 Subject: [PATCH 016/106] feat: added fields for git --- core/controllers/utils_http.go | 4 ++++ core/models/models/git_v2.go | 17 +++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/controllers/utils_http.go b/core/controllers/utils_http.go index 4726f2886..a26f4ea1d 100644 --- a/core/controllers/utils_http.go +++ b/core/controllers/utils_http.go @@ -31,6 +31,10 @@ func HandleErrorBadRequest(c *gin.Context, err error) { HandleError(http.StatusBadRequest, c, err) } +func HandleErrorForbidden(c *gin.Context, err error) { + HandleError(http.StatusForbidden, c, err) +} + func HandleErrorUnauthorized(c *gin.Context, err error) { HandleError(http.StatusUnauthorized, c, err) } diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go index 2351882ac..7e4347fdb 100644 --- a/core/models/models/git_v2.go +++ b/core/models/models/git_v2.go @@ -3,14 +3,15 @@ package models type GitV2 struct { any `collection:"gits"` BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` // settings AutoPull bool `json:"auto_pull" bson:"auto_pull"` From 9e400725321ea106eae606a1b69c46b264b31ba0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 19 Jun 2024 18:33:59 +0800 Subject: [PATCH 017/106] feat: updated ref name for git remote branches --- vcs/git.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/vcs/git.go b/vcs/git.go index 11267a472..0518d7fee 100644 --- a/vcs/git.go +++ b/vcs/git.go @@ -550,21 +550,27 @@ func (c *GitClient) GetRemoteRefs(remoteName string) (gitRefs []GitRef, err erro for _, ref := range refs { // ref type var refType string + var gitRef *GitRef if strings.HasPrefix(ref.Name().String(), "refs/heads") { refType = GitRefTypeBranch + gitRef = &GitRef{ + Type: refType, + Name: remoteName + "/" + ref.Name().Short(), + FullName: ref.Name().String(), + Hash: ref.Hash().String(), + } } else if strings.HasPrefix(ref.Name().String(), "refs/tags") { refType = GitRefTypeTag + gitRef = &GitRef{ + Type: refType, + Name: ref.Name().Short(), + FullName: ref.Name().String(), + Hash: ref.Hash().String(), + } } else { continue } - - // add to branches - gitRefs = append(gitRefs, GitRef{ - Type: refType, - Name: ref.Name().Short(), - FullName: ref.Name().String(), - Hash: ref.Hash().String(), - }) + gitRefs = append(gitRefs, *gitRef) } // logs without tags From 725ee7d9ad18c7c206a93283f1012a432b0766be Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 22 Jun 2024 00:21:32 +0800 Subject: [PATCH 018/106] feat: updated git model --- core/models/models/git_v2.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go index 7e4347fdb..6dc92f117 100644 --- a/core/models/models/git_v2.go +++ b/core/models/models/git_v2.go @@ -1,17 +1,24 @@ package models +import ( + "github.com/crawlab-team/crawlab/vcs" + "time" +) + type GitV2 struct { any `collection:"gits"` BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` // settings AutoPull bool `json:"auto_pull" bson:"auto_pull"` From 460c8d958a07e123a5baf7795b3ea70e5cd24cc5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 23 Jun 2024 23:20:58 +0800 Subject: [PATCH 019/106] fix: missing relational models issues --- core/controllers/base_file_v2.go | 30 ++--- core/controllers/project_v2.go | 89 +++++++++++++ core/controllers/router_v2.go | 102 ++++++++------- core/controllers/spider_v2.go | 212 +++++++++++++++++++++++-------- core/controllers/sync.go | 9 +- core/controllers/task_v2.go | 57 ++++++--- core/task/handler/runner_v2.go | 59 ++++++--- core/task/handler/service_v2.go | 5 +- vcs/entity.go | 11 +- 9 files changed, 406 insertions(+), 168 deletions(-) create mode 100644 core/controllers/project_v2.go diff --git a/core/controllers/base_file_v2.go b/core/controllers/base_file_v2.go index 6c398abdd..38987be79 100644 --- a/core/controllers/base_file_v2.go +++ b/core/controllers/base_file_v2.go @@ -8,7 +8,6 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/gin-gonic/gin" "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson/primitive" "io" "os" "path/filepath" @@ -18,7 +17,7 @@ import ( func GetBaseFileListDir(rootPath string, c *gin.Context) { path := c.Query("path") - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -38,7 +37,7 @@ func GetBaseFileListDir(rootPath string, c *gin.Context) { func GetBaseFileFile(rootPath string, c *gin.Context) { path := c.Query("path") - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -56,7 +55,7 @@ func GetBaseFileFile(rootPath string, c *gin.Context) { func GetBaseFileFileInfo(rootPath string, c *gin.Context) { path := c.Query("path") - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -72,7 +71,7 @@ func GetBaseFileFileInfo(rootPath string, c *gin.Context) { } func PostBaseFileSaveFile(rootPath string, c *gin.Context) { - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorInternalServerError(c, err) return @@ -122,7 +121,7 @@ func PostBaseFileSaveFile(rootPath string, c *gin.Context) { } func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorInternalServerError(c, err) return @@ -183,7 +182,7 @@ func PostBaseFileSaveDir(rootPath string, c *gin.Context) { return } - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -207,7 +206,7 @@ func PostBaseFileRenameFile(rootPath string, c *gin.Context) { return } - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -231,7 +230,7 @@ func DeleteBaseFileFile(rootPath string, c *gin.Context) { payload.Path = "." } - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -259,7 +258,7 @@ func PostBaseFileCopyFile(rootPath string, c *gin.Context) { return } - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -274,7 +273,7 @@ func PostBaseFileCopyFile(rootPath string, c *gin.Context) { } func PostBaseFileExport(rootPath string, c *gin.Context) { - fsSvc, err := getBaseFileFsSvc(rootPath, c) + fsSvc, err := getBaseFileFsSvc(rootPath) if err != nil { HandleErrorBadRequest(c, err) return @@ -292,14 +291,9 @@ func PostBaseFileExport(rootPath string, c *gin.Context) { c.File(zipFilePath) } -func getBaseFileFsSvc(rootPath string, c *gin.Context) (svc interfaces.FsServiceV2, err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - return nil, err - } - +func getBaseFileFsSvc(rootPath string) (svc interfaces.FsServiceV2, err error) { workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex(), rootPath)) + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, rootPath)) return fsSvc, nil } diff --git a/core/controllers/project_v2.go b/core/controllers/project_v2.go new file mode 100644 index 000000000..dd2d89263 --- /dev/null +++ b/core/controllers/project_v2.go @@ -0,0 +1,89 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/db/mongo" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +func GetProjectList(c *gin.Context) { + // get all list + all := MustGetFilterAll(c) + if all { + NewControllerV2[models.ProjectV2]().getAll(c) + return + } + + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + projects, err := service.NewModelServiceV2[models.ProjectV2]().GetMany(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + HandleErrorInternalServerError(c, err) + } + return + } + if len(projects) == 0 { + HandleSuccessWithListData(c, []models.ProjectV2{}, 0) + return + } + + // total count + total, err := service.NewModelServiceV2[models.ProjectV2]().Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // project ids + var ids []primitive.ObjectID + + // count cache + cache := map[primitive.ObjectID]int{} + + // iterate + for _, p := range projects { + ids = append(ids, p.Id) + cache[p.Id] = 0 + } + + // spiders + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{ + "project_id": bson.M{ + "$in": ids, + }, + }, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + for _, s := range spiders { + _, ok := cache[s.ProjectId] + if !ok { + HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache) + return + } + cache[s.ProjectId]++ + } + + // assign + for _, p := range projects { + p.Spiders = cache[p.Id] + projects = append(projects, p) + } + + HandleSuccessWithListData(c, projects, total) +} diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 908368bdf..1e274bffe 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -61,204 +61,210 @@ func InitRoutes(app *gin.Engine) (err error) { RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]()) RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) - RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]()) + RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]([]Action{ + { + Method: http.MethodGet, + Path: "", + HandlerFunc: GetProjectList, + }, + }...)) RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]()) - RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]( - Action{ + RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]([]Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostSchedule, }, - Action{ + { Method: http.MethodPut, Path: "/:id", HandlerFunc: PutScheduleById, }, - Action{ + { Method: http.MethodPost, Path: "/:id/enable", HandlerFunc: PostScheduleEnable, }, - Action{ + { Method: http.MethodPost, Path: "/:id/disable", HandlerFunc: PostScheduleDisable, }, - )) - RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2]([]Action{ + { Method: http.MethodGet, Path: "/:id", HandlerFunc: GetSpiderById, }, - Action{ + { Method: http.MethodGet, Path: "", HandlerFunc: GetSpiderList, }, - Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostSpider, }, - Action{ + { Method: http.MethodPut, Path: "/:id", HandlerFunc: PutSpiderById, }, - Action{ + { Method: http.MethodDelete, Path: "/:id", HandlerFunc: DeleteSpiderById, }, - Action{ + { Method: http.MethodDelete, Path: "", HandlerFunc: DeleteSpiderList, }, - Action{ + { Method: http.MethodGet, Path: "/:id/files/list", HandlerFunc: GetSpiderListDir, }, - Action{ + { Method: http.MethodGet, Path: "/:id/files/get", HandlerFunc: GetSpiderFile, }, - Action{ + { Method: http.MethodGet, Path: "/:id/files/info", HandlerFunc: GetSpiderFileInfo, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/save", HandlerFunc: PostSpiderSaveFile, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/save/batch", HandlerFunc: PostSpiderSaveFiles, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/save/dir", HandlerFunc: PostSpiderSaveDir, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/rename", HandlerFunc: PostSpiderRenameFile, }, - Action{ + { Method: http.MethodDelete, Path: "/:id/files", HandlerFunc: DeleteSpiderFile, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/copy", HandlerFunc: PostSpiderCopyFile, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/export", HandlerFunc: PostSpiderExport, }, - Action{ + { Method: http.MethodPost, Path: "/:id/run", HandlerFunc: PostSpiderRun, }, - Action{ + { Method: http.MethodGet, Path: "/:id/data-source", HandlerFunc: GetSpiderDataSource, }, - Action{ + { Method: http.MethodPost, Path: "/:id/data-source/:ds_id", HandlerFunc: PostSpiderDataSource, }, - )) - RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2]([]Action{ + { Method: http.MethodGet, Path: "/:id", HandlerFunc: GetTaskById, }, - Action{ + { Method: http.MethodGet, Path: "", HandlerFunc: GetTaskList, }, - Action{ + { Method: http.MethodDelete, Path: "/:id", HandlerFunc: DeleteTaskById, }, - Action{ + { Method: http.MethodDelete, Path: "", HandlerFunc: DeleteList, }, - Action{ + { Method: http.MethodPost, Path: "/run", HandlerFunc: PostTaskRun, }, - Action{ + { Method: http.MethodPost, Path: "/:id/restart", HandlerFunc: PostTaskRestart, }, - Action{ + { Method: http.MethodPost, Path: "/:id/cancel", HandlerFunc: PostTaskCancel, }, - Action{ + { Method: http.MethodGet, Path: "/:id/logs", HandlerFunc: GetTaskLogs, }, - Action{ + { Method: http.MethodGet, Path: "/:id/data", HandlerFunc: GetTaskData, }, - )) - RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2]([]Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostToken, }, - )) - RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2]([]Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostUser, }, - Action{ + { Method: http.MethodPost, Path: "/:id/change-password", HandlerFunc: PostUserChangePassword, }, - Action{ + { Method: http.MethodGet, Path: "/me", HandlerFunc: GetUserMe, }, - Action{ + { Method: http.MethodPut, Path: "/me", HandlerFunc: PutUserById, }, - )) + }...)) RegisterActions(groups.AuthGroup, "/results", []Action{ { diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 9b28b94df..78b4d2492 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -2,7 +2,7 @@ package controllers import ( "errors" - log2 "github.com/apex/log" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" @@ -11,6 +11,7 @@ import ( "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/trace" "github.com/gin-gonic/gin" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" @@ -64,12 +65,25 @@ func GetSpiderById(c *gin.Context) { } func GetSpiderList(c *gin.Context) { + // get all list + all := MustGetFilterAll(c) + if all { + NewControllerV2[models.ProjectV2]().getAll(c) + return + } + + // get list withStats := c.Query("stats") if withStats == "" { NewControllerV2[models.SpiderV2]().GetList(c) return } + // get list with stats + getSpiderListWithStats(c) +} + +func getSpiderListWithStats(c *gin.Context) { // params pagination := MustGetPagination(c) query := MustGetFilterQuery(c) @@ -205,6 +219,7 @@ func PostSpider(c *gin.Context) { return } + // user u := GetUserFromContextV2(c) // add @@ -229,7 +244,12 @@ func PostSpider(c *gin.Context) { } // create folder - err = getSpiderFsSvcById(id).CreateDir(".") + fsSvc, err := getSpiderFsSvcById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + err = fsSvc.CreateDir(".") if err != nil { HandleErrorInternalServerError(c, err) return @@ -336,7 +356,7 @@ func DeleteSpiderById(c *gin.Context) { // delete task logs logPath := filepath.Join(viper.GetString("log.path"), id) if err := os.RemoveAll(logPath); err != nil { - log2.Warnf("failed to remove task log directory: %s", logPath) + log.Warnf("failed to remove task log directory: %s", logPath) } wg.Done() }(id.Hex()) @@ -349,6 +369,35 @@ func DeleteSpiderById(c *gin.Context) { return } + go func() { + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + log.Errorf("failed to get spider: %s", err.Error()) + trace.PrintError(err) + return + } + + // skip spider with git + if !s.GitId.IsZero() { + return + } + + // delete spider directory + fsSvc, err := getSpiderFsSvcById(id) + if err != nil { + log.Errorf("failed to get spider fs service: %s", err.Error()) + trace.PrintError(err) + return + } + err = fsSvc.Delete(".") + if err != nil { + log.Errorf("failed to delete spider directory: %s", err.Error()) + trace.PrintError(err) + return + } + }() + HandleSuccess(c) } @@ -414,7 +463,7 @@ func DeleteSpiderList(c *gin.Context) { // delete task logs logPath := filepath.Join(viper.GetString("log.path"), id) if err := os.RemoveAll(logPath); err != nil { - log2.Warnf("failed to remove task log directory: %s", logPath) + log.Warnf("failed to remove task log directory: %s", logPath) } wg.Done() }(id.Hex()) @@ -427,97 +476,136 @@ func DeleteSpiderList(c *gin.Context) { return } + // delete spider directories + go func() { + wg := sync.WaitGroup{} + wg.Add(len(payload.Ids)) + for _, id := range payload.Ids { + go func(id primitive.ObjectID) { + defer wg.Done() + + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + log.Errorf("failed to get spider: %s", err.Error()) + trace.PrintError(err) + return + } + + // skip spider with git + if !s.GitId.IsZero() { + return + } + + // delete spider directory + fsSvc, err := getSpiderFsSvcById(id) + if err != nil { + log.Errorf("failed to get spider fs service: %s", err.Error()) + trace.PrintError(err) + return + } + err = fsSvc.Delete(".") + if err != nil { + log.Errorf("failed to delete spider directory: %s", err.Error()) + trace.PrintError(err) + return + } + }(id) + } + wg.Wait() + }() + HandleSuccess(c) } func GetSpiderListDir(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - GetBaseFileListDir(s.GitRootPath, c) + GetBaseFileListDir(rootPath, c) } func GetSpiderFile(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - GetBaseFileFile(s.GitRootPath, c) + GetBaseFileFile(rootPath, c) } func GetSpiderFileInfo(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - GetBaseFileFileInfo(s.GitRootPath, c) + GetBaseFileFileInfo(rootPath, c) } func PostSpiderSaveFile(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - PostBaseFileSaveFile(s.GitRootPath, c) + PostBaseFileSaveFile(rootPath, c) } func PostSpiderSaveFiles(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - PostBaseFileSaveFiles(s.GitRootPath, c) + PostBaseFileSaveFiles(rootPath, c) } func PostSpiderSaveDir(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - PostBaseFileSaveDir(s.GitRootPath, c) + PostBaseFileSaveDir(rootPath, c) } func PostSpiderRenameFile(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - PostBaseFileRenameFile(s.GitRootPath, c) + PostBaseFileRenameFile(rootPath, c) } func DeleteSpiderFile(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - DeleteBaseFileFile(s.GitRootPath, c) + DeleteBaseFileFile(rootPath, c) } func PostSpiderCopyFile(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - PostBaseFileCopyFile(s.GitRootPath, c) + PostBaseFileCopyFile(rootPath, c) } func PostSpiderExport(c *gin.Context) { - s, err := allowSpiderGit(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - PostBaseFileExport(s.GitRootPath, c) + PostBaseFileExport(rootPath, c) } func PostSpiderRun(c *gin.Context) { @@ -628,22 +716,25 @@ func PostSpiderDataSource(c *gin.Context) { HandleSuccess(c) } -func getSpiderFsSvc(c *gin.Context) (svc interfaces.FsServiceV2, err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - return nil, err - } - +func getSpiderFsSvc(s *models.SpiderV2) (svc interfaces.FsServiceV2, err error) { workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, s.Id.Hex())) return fsSvc, nil } -func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 { - workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) - return fsSvc +func GetSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) { + return getSpiderFsSvcById(id) +} + +func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) { + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + log.Errorf("failed to get spider: %s", err.Error()) + trace.PrintError(err) + return nil, err + } + return getSpiderFsSvc(s) } func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { @@ -685,21 +776,32 @@ func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { return nil } -func allowSpiderGit(c *gin.Context) (s models.SpiderV2, err error) { - if utils.IsPro() { - return s, nil - } +func UpsertSpiderDataCollection(s *models.SpiderV2) (err error) { + return upsertSpiderDataCollection(s) +} + +func getSpiderRootPath(c *gin.Context) (rootPath string, err error) { + // spider id id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { - return s, err + return "", err } - _s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) if err != nil { - return s, err + return "", err } + + // check git permission + if !utils.IsPro() && !s.GitId.IsZero() { + return "", errors.New("git is not allowed in the community version") + } + + // if git id is zero, return spider id as root path if s.GitId.IsZero() { - return s, errors.New("git is not allowed in this edition") + return id.Hex(), nil } - s = *_s - return s, nil + + return filepath.Join(s.GitId.Hex(), rootPath), nil } diff --git a/core/controllers/sync.go b/core/controllers/sync.go index 674bff76f..5f9874c8f 100644 --- a/core/controllers/sync.go +++ b/core/controllers/sync.go @@ -31,7 +31,8 @@ type syncContext struct { func (ctx *syncContext) scan(c *gin.Context) { id := c.Param("id") - dir := ctx._getDir(id) + path := c.Query("path") + dir := ctx._getDir(id, path) files, err := utils.ScanDirectory(dir) if err != nil { HandleErrorInternalServerError(c, err) @@ -43,13 +44,13 @@ func (ctx *syncContext) scan(c *gin.Context) { func (ctx *syncContext) download(c *gin.Context) { id := c.Param("id") filePath := c.Query("path") - dir := ctx._getDir(id) + dir := ctx._getDir(id, "") c.File(filepath.Join(dir, filePath)) } -func (ctx *syncContext) _getDir(id string) string { +func (ctx *syncContext) _getDir(id string, path string) string { workspacePath := viper.GetString("workspace") - return filepath.Join(workspacePath, id) + return filepath.Join(workspacePath, id, path) } func newSyncContext() syncContext { diff --git a/core/controllers/task_v2.go b/core/controllers/task_v2.go index 95c58de60..aad4fcf3f 100644 --- a/core/controllers/task_v2.go +++ b/core/controllers/task_v2.go @@ -71,8 +71,8 @@ func GetTaskList(c *gin.Context) { query := MustGetFilterQuery(c) sort := MustGetSortOption(c) - // get list - list, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ + // get tasks + tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -87,15 +87,17 @@ func GetTaskList(c *gin.Context) { } // check empty list - if len(list) == 0 { + if len(tasks) == 0 { HandleSuccessWithListData(c, nil, 0) return } // ids - var ids []primitive.ObjectID - for _, t := range list { - ids = append(ids, t.Id) + var taskIds []primitive.ObjectID + var spiderIds []primitive.ObjectID + for _, t := range tasks { + taskIds = append(taskIds, t.Id) + spiderIds = append(spiderIds, t.SpiderId) } // total count @@ -106,33 +108,56 @@ func GetTaskList(c *gin.Context) { } // stat list - query = bson.M{ + stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ "_id": bson.M{ - "$in": ids, + "$in": taskIds, }, - } - stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(query, nil) + }, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // cache stat list to dict - dict := map[primitive.ObjectID]models.TaskStatV2{} + statsDict := map[primitive.ObjectID]models.TaskStatV2{} for _, s := range stats { - dict[s.Id] = s + statsDict[s.Id] = s + } + + // spider list + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + }, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache spider list to dict + spiderDict := map[primitive.ObjectID]models.SpiderV2{} + for _, s := range spiders { + spiderDict[s.Id] = s } // iterate list again - for i, t := range list { - ts, ok := dict[t.Id] + for i, t := range tasks { + // task stat + ts, ok := statsDict[t.Id] + if ok { + tasks[i].Stat = &ts + } + + // spider + s, ok := spiderDict[t.SpiderId] if ok { - list[i].Stat = &ts + tasks[i].Spider = &s } } // response - HandleSuccessWithListData(c, list, total) + HandleSuccessWithListData(c, tasks, total) } func DeleteTaskById(c *gin.Context) { diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 7424a2c18..57f95edb3 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -71,16 +71,8 @@ func (r *RunnerV2) Init() (err error) { // start grpc client if !r.c.IsStarted() { - r.c.Start() - } - - // working directory - workspacePath := viper.GetString("workspace") - r.cwd = filepath.Join(workspacePath, r.s.Id.Hex()) - - // sync files from master - if !utils.IsMaster() { - if err := r.syncFiles(); err != nil { + err := r.c.Start() + if err != nil { return err } } @@ -97,6 +89,16 @@ func (r *RunnerV2) Run() (err error) { // log task started log.Infof("task[%s] started", r.tid.Hex()) + // configure working directory + r.configureCwd() + + // sync files worker nodes + if !utils.IsMaster() { + if err := r.syncFiles(); err != nil { + return err + } + } + // configure cmd r.configureCmd() @@ -317,26 +319,31 @@ func (r *RunnerV2) configureEnv() { } func (r *RunnerV2) syncFiles() (err error) { - masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.Id.Hex()) - workspacePath := viper.GetString("workspace") - workerDir := filepath.Join(workspacePath, r.s.Id.Hex()) + var id string + if r.s.GitId.IsZero() { + id = r.s.Id.Hex() + } else { + id = r.s.GitId.Hex() + } + masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), id) + workerDir := r.cwd // get file list from master - resp, err := http.Get(masterURL + "/scan") + resp, err := http.Get(masterURL + "/scan?path=" + workerDir) if err != nil { - fmt.Println("Error getting file list from master:", err) + log.Errorf("Error getting file list from master: %v", err) return trace.TraceError(err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - fmt.Println("Error reading response body:", err) + log.Errorf("Error reading response body: %v", err) return trace.TraceError(err) } var masterFiles map[string]entity.FsFileInfo err = json.Unmarshal(body, &masterFiles) if err != nil { - fmt.Println("Error unmarshaling JSON:", err) + log.Errorf("Error unmarshaling JSON: %v", err) return trace.TraceError(err) } @@ -349,7 +356,7 @@ func (r *RunnerV2) syncFiles() (err error) { // create worker directory if not exists if _, err := os.Stat(workerDir); os.IsNotExist(err) { if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { - fmt.Println("Error creating worker directory:", err) + log.Errorf("Error creating worker directory: %v", err) return trace.TraceError(err) } } @@ -357,7 +364,7 @@ func (r *RunnerV2) syncFiles() (err error) { // get file list from worker workerFiles, err := utils.ScanDirectory(workerDir) if err != nil { - fmt.Println("Error scanning worker directory:", err) + log.Errorf("Error scanning worker directory: %v", err) return trace.TraceError(err) } @@ -368,10 +375,10 @@ func (r *RunnerV2) syncFiles() (err error) { // delete files that are deleted on master node for path, workerFile := range workerFiles { if _, exists := masterFilesMap[path]; !exists { - fmt.Println("Deleting file:", path) + log.Infof("Deleting file: %s", path) err := os.Remove(workerFile.FullPath) if err != nil { - fmt.Println("Error deleting file:", err) + log.Errorf("Error deleting file: %v", err) } } } @@ -619,7 +626,17 @@ func (r *RunnerV2) _updateSpiderStat(status string) { return } } +} +func (r *RunnerV2) configureCwd() { + workspacePath := viper.GetString("workspace") + if r.s.GitId.IsZero() { + // not git + r.cwd = filepath.Join(workspacePath, r.s.Id.Hex()) + } else { + // git + r.cwd = filepath.Join(workspacePath, r.s.GitId.Hex(), r.s.GitRootPath) + } } func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err error) { diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index ad0dcdd9c..4635ef48b 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -43,7 +43,10 @@ type ServiceV2 struct { func (svc *ServiceV2) Start() { // Initialize gRPC if not started if !svc.c.IsStarted() { - svc.c.Start() + err := svc.c.Start() + if err != nil { + return + } } go svc.ReportStatus() diff --git a/vcs/entity.go b/vcs/entity.go index 509e3784b..e96c2feb3 100644 --- a/vcs/entity.go +++ b/vcs/entity.go @@ -9,11 +9,12 @@ type GitOptions struct { } type GitRef struct { - Type string `json:"type"` - Name string `json:"name"` - FullName string `json:"full_name"` - Hash string `json:"hash"` - Timestamp time.Time `json:"timestamp"` + Type string `json:"type"` + Name string `json:"name"` + FullName string `json:"full_name"` + Hash string `json:"hash"` + Timestamp time.Time `json:"timestamp"` + RemoteTrack string `json:"remote_track"` } type GitLog struct { From 5daeccb87d1ac566e1ba246dc85e1f2cb18f34c1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 25 Jun 2024 14:58:54 +0800 Subject: [PATCH 020/106] fix: unable to sync files and save data issues --- core/controllers/project_v2.go | 5 +++-- core/controllers/router_v2.go | 12 +++++++++++ core/controllers/sync_v2.go | 31 ++++++++++++++++++++++++++++ core/grpc/server/node_server_v2.go | 2 +- core/node/config/config_service.go | 7 ++++--- core/sys_exec/sys_exec_darwin.go | 10 +++++++-- core/sys_exec/sys_exec_linux.go | 10 +++++++-- core/sys_exec/sys_exec_windows.go | 14 ++++++++++--- core/task/handler/runner.go | 2 +- core/task/handler/runner_v2.go | 33 +++++++++++++++++++----------- core/task/stats/service_v2.go | 3 +-- core/utils/demo.go | 15 +++++++++++--- 12 files changed, 113 insertions(+), 31 deletions(-) create mode 100644 core/controllers/sync_v2.go diff --git a/core/controllers/project_v2.go b/core/controllers/project_v2.go index dd2d89263..34fb6b8dd 100644 --- a/core/controllers/project_v2.go +++ b/core/controllers/project_v2.go @@ -80,10 +80,11 @@ func GetProjectList(c *gin.Context) { } // assign + var data []models.ProjectV2 for _, p := range projects { p.Spiders = cache[p.Id] - projects = append(projects, p) + data = append(data, p) } - HandleSuccessWithListData(c, projects, total) + HandleSuccessWithListData(c, data, total) } diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 1e274bffe..ec23515e0 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -363,6 +363,18 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostLogout, }, }) + RegisterActions(groups.AnonymousGroup, "/sync", []Action{ + { + Method: http.MethodGet, + Path: "/:id/scan", + HandlerFunc: GetSyncScan, + }, + { + Method: http.MethodGet, + Path: "/:id/download", + HandlerFunc: GetSyncDownload, + }, + }) return nil } diff --git a/core/controllers/sync_v2.go b/core/controllers/sync_v2.go new file mode 100644 index 000000000..5e44fcd44 --- /dev/null +++ b/core/controllers/sync_v2.go @@ -0,0 +1,31 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net/http" + "path/filepath" +) + +func GetSyncScan(c *gin.Context) { + id := c.Param("id") + path := c.Query("path") + + workspacePath := viper.GetString("workspace") + dirPath := filepath.Join(workspacePath, id, path) + files, err := utils.ScanDirectory(dirPath) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + c.AbortWithStatusJSON(http.StatusOK, files) +} + +func GetSyncDownload(c *gin.Context) { + id := c.Param("id") + path := c.Query("path") + workspacePath := viper.GetString("workspace") + filePath := filepath.Join(workspacePath, id, path) + c.File(filePath) +} diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index ac9499ca8..6c69b14bc 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -83,7 +83,7 @@ func (svr NodeServerV2) Register(ctx context.Context, req *grpc.Request) (res *g } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - _, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(*nodeDb) + node.Id, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(node) if err != nil { return HandleError(err) } diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go index 7e05adde4..c91c58bf3 100644 --- a/core/node/config/config_service.go +++ b/core/node/config/config_service.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" "os" - "path" + "path/filepath" ) type Service struct { @@ -18,7 +18,7 @@ type Service struct { func (svc *Service) Init() (err error) { // check config directory path - configDirPath := path.Dir(svc.path) + configDirPath := filepath.Dir(svc.path) if !utils.Exists(configDirPath) { if err := os.MkdirAll(configDirPath, os.FileMode(0766)); err != nil { return trace.TraceError(err) @@ -55,13 +55,14 @@ func (svc *Service) Reload() (err error) { } func (svc *Service) GetBasicNodeInfo() (res interfaces.Entity) { - return &entity.NodeInfo{ + res = &entity.NodeInfo{ Key: svc.GetNodeKey(), Name: svc.GetNodeName(), IsMaster: svc.IsMaster(), AuthKey: svc.GetAuthKey(), MaxRunners: svc.GetMaxRunners(), } + return res } func (svc *Service) GetNodeKey() (res string) { diff --git a/core/sys_exec/sys_exec_darwin.go b/core/sys_exec/sys_exec_darwin.go index 130b7cb82..b6db18c21 100644 --- a/core/sys_exec/sys_exec_darwin.go +++ b/core/sys_exec/sys_exec_darwin.go @@ -4,12 +4,18 @@ package sys_exec import ( + "errors" "os/exec" + "strings" "syscall" ) -func BuildCmd(cmdStr string) *exec.Cmd { - return exec.Command("sh", "-c", cmdStr) +func BuildCmd(cmdStr string) (cmd *exec.Cmd, err error) { + if cmdStr == "" { + return nil, errors.New("command string is empty") + } + args := strings.Split(cmdStr, " ") + return exec.Command(args[0], args[1:]...), nil } func SetPgid(cmd *exec.Cmd) { diff --git a/core/sys_exec/sys_exec_linux.go b/core/sys_exec/sys_exec_linux.go index bf532a43d..33af73e32 100644 --- a/core/sys_exec/sys_exec_linux.go +++ b/core/sys_exec/sys_exec_linux.go @@ -4,12 +4,18 @@ package sys_exec import ( + "errors" "os/exec" + "strings" "syscall" ) -func BuildCmd(cmdStr string) *exec.Cmd { - return exec.Command("sh", "-c", cmdStr) +func BuildCmd(cmdStr string) (cmd *exec.Cmd, err error) { + if cmdStr == "" { + return nil, errors.New("command string is empty") + } + args := strings.Split(cmdStr, " ") + return exec.Command(args[0], args[1:]...), nil } func SetPgid(cmd *exec.Cmd) { diff --git a/core/sys_exec/sys_exec_windows.go b/core/sys_exec/sys_exec_windows.go index e2678ccd8..1f1afd5a3 100644 --- a/core/sys_exec/sys_exec_windows.go +++ b/core/sys_exec/sys_exec_windows.go @@ -3,8 +3,16 @@ package sys_exec -import "os/exec" +import ( + "errors" + "os/exec" + "strings" +) -func BuildCmd(cmdStr string) *exec.Cmd { - return exec.Command("cmd", "/C", cmdStr) +func BuildCmd(cmdStr string) (cmd *exec.Cmd, err error) { + if cmdStr == "" { + return nil, errors.New("command string is empty") + } + args := strings.Split(cmdStr, " ") + return exec.Command(args[0], args[1:]...), nil } diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go index b39d9445c..32c7e5bff 100644 --- a/core/task/handler/runner.go +++ b/core/task/handler/runner.go @@ -237,7 +237,7 @@ func (r *Runner) configureCmd() { } // get cmd instance - r.cmd = sys_exec.BuildCmd(cmdStr) + r.cmd, _ = sys_exec.BuildCmd(cmdStr) // set working directory r.cmd.Dir = r.cwd diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 57f95edb3..341ff408c 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -95,12 +95,15 @@ func (r *RunnerV2) Run() (err error) { // sync files worker nodes if !utils.IsMaster() { if err := r.syncFiles(); err != nil { - return err + return r.updateTask(constants.TaskStatusError, err) } } // configure cmd - r.configureCmd() + err = r.configureCmd() + if err != nil { + return r.updateTask(constants.TaskStatusError, err) + } // configure environment variables r.configureEnv() @@ -205,7 +208,7 @@ func (r *RunnerV2) GetTaskId() (id primitive.ObjectID) { return r.tid } -func (r *RunnerV2) configureCmd() { +func (r *RunnerV2) configureCmd() (err error) { var cmdStr string // customized spider @@ -223,13 +226,17 @@ func (r *RunnerV2) configureCmd() { } // get cmd instance - r.cmd = sys_exec.BuildCmd(cmdStr) + r.cmd, err = sys_exec.BuildCmd(cmdStr) + if err != nil { + log.Errorf("Error building command: %v", err) + trace.PrintError(err) + return err + } // set working directory r.cmd.Dir = r.cwd - // configure pgid to allow killing sub processes - //sys_exec.SetPgid(r.cmd) + return nil } func (r *RunnerV2) configureLogging() { @@ -320,16 +327,18 @@ func (r *RunnerV2) configureEnv() { func (r *RunnerV2) syncFiles() (err error) { var id string + var workingDir string if r.s.GitId.IsZero() { id = r.s.Id.Hex() + workingDir = "" } else { id = r.s.GitId.Hex() + workingDir = r.s.GitRootPath } masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), id) - workerDir := r.cwd // get file list from master - resp, err := http.Get(masterURL + "/scan?path=" + workerDir) + resp, err := http.Get(masterURL + "/scan?path=" + workingDir) if err != nil { log.Errorf("Error getting file list from master: %v", err) return trace.TraceError(err) @@ -354,15 +363,15 @@ func (r *RunnerV2) syncFiles() (err error) { } // create worker directory if not exists - if _, err := os.Stat(workerDir); os.IsNotExist(err) { - if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { + if _, err := os.Stat(r.cwd); os.IsNotExist(err) { + if err := os.MkdirAll(r.cwd, os.ModePerm); err != nil { log.Errorf("Error creating worker directory: %v", err) return trace.TraceError(err) } } // get file list from worker - workerFiles, err := utils.ScanDirectory(workerDir) + workerFiles, err := utils.ScanDirectory(r.cwd) if err != nil { log.Errorf("Error scanning worker directory: %v", err) return trace.TraceError(err) @@ -391,7 +400,7 @@ func (r *RunnerV2) syncFiles() (err error) { go func(path string, masterFile entity.FsFileInfo) { defer wg.Done() logrus.Infof("File needs to be synchronized: %s", path) - err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) + err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(r.cwd, path)) if err != nil { logrus.Errorf("Error downloading file: %v", err) select { diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go index b74b34411..fc39cbd50 100644 --- a/core/task/stats/service_v2.go +++ b/core/task/stats/service_v2.go @@ -17,7 +17,6 @@ import ( type ServiceV2 struct { // dependencies nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService // internals mu sync.Mutex @@ -64,7 +63,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa } // task - t, err := svc.modelSvc.GetTaskById(id) + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) if err != nil { return nil, err } diff --git a/core/utils/demo.go b/core/utils/demo.go index bf6676476..70b450444 100644 --- a/core/utils/demo.go +++ b/core/utils/demo.go @@ -31,7 +31,10 @@ func InitializedDemo() (ok bool) { func ImportDemo() (err error) { cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo import", GetApiAddress()) - cmd := sys_exec.BuildCmd(cmdStr) + cmd, err := sys_exec.BuildCmd(cmdStr) + if err != nil { + return err + } if err := cmd.Run(); err != nil { trace.PrintError(err) } @@ -40,7 +43,10 @@ func ImportDemo() (err error) { func ReimportDemo() (err error) { cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) - cmd := sys_exec.BuildCmd(cmdStr) + cmd, err := sys_exec.BuildCmd(cmdStr) + if err != nil { + return err + } if err := cmd.Run(); err != nil { trace.PrintError(err) } @@ -49,7 +55,10 @@ func ReimportDemo() (err error) { func CleanupDemo() (err error) { cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) - cmd := sys_exec.BuildCmd(cmdStr) + cmd, err := sys_exec.BuildCmd(cmdStr) + if err != nil { + return err + } if err := cmd.Run(); err != nil { trace.PrintError(err) } From 326a8d67d02806b3b68581ca732d24a62653e83d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 26 Jun 2024 12:37:24 +0800 Subject: [PATCH 021/106] fix: missing data source issue --- core/controllers/data_source_v2.go | 120 ++++++++ core/controllers/router_v2.go | 20 +- core/ds/service_v2.go | 271 ++++++++++++++++++ .../server/model_base_service_v2_server.go | 1 + core/models/models/data_source_v2.go | 2 +- core/models/models/notification_setting_v2.go | 24 +- core/models/models/spider_v2.go | 2 +- core/node/service/master_service_v2.go | 8 +- core/notification/service_v2.go | 4 - core/system/service_v2.go | 67 +++++ core/utils/cockroachdb.go | 45 +++ core/utils/es.go | 85 ++++-- core/utils/kafka.go | 27 ++ core/utils/mongo.go | 45 +++ core/utils/mssql.go | 45 +++ core/utils/mysql.go | 45 +++ core/utils/postgresql.go | 45 +++ core/utils/sqlite.go | 32 +++ 18 files changed, 843 insertions(+), 45 deletions(-) create mode 100644 core/controllers/data_source_v2.go create mode 100644 core/ds/service_v2.go create mode 100644 core/system/service_v2.go diff --git a/core/controllers/data_source_v2.go b/core/controllers/data_source_v2.go new file mode 100644 index 000000000..6962dc957 --- /dev/null +++ b/core/controllers/data_source_v2.go @@ -0,0 +1,120 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/ds" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func PostDataSource(c *gin.Context) { + // data source + var payload struct { + Name string `json:"name"` + Type string `json:"type"` + Description string `json:"description"` + Host string `json:"host"` + Port string `json:"port"` + Url string `json:"url"` + Hosts []string `json:"hosts"` + Database string `json:"database"` + Username string `json:"username"` + Password string `json:"-,omitempty"` + ConnectType string `json:"connect_type"` + Status string `json:"status"` + Error string `json:"error"` + Extra map[string]string `json:"extra,omitempty"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + u := GetUserFromContextV2(c) + + // add data source to db + dataSource := models.DataSourceV2{ + Name: payload.Name, + Type: payload.Type, + Description: payload.Description, + Host: payload.Host, + Port: payload.Port, + Url: payload.Url, + Hosts: payload.Hosts, + Database: payload.Database, + Username: payload.Username, + Password: payload.Password, + ConnectType: payload.ConnectType, + Status: payload.Status, + Error: payload.Error, + Extra: payload.Extra, + } + dataSource.SetCreated(u.Id) + dataSource.SetUpdated(u.Id) + id, err := service.NewModelServiceV2[models.DataSourceV2]().InsertOne(dataSource) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + dataSource.Id = id + + // check data source status + go func() { + _ = ds.GetDataSourceServiceV2().CheckStatus(id) + }() + + HandleSuccessWithData(c, dataSource) +} + +func PutDataSourceById(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // data source + var dataSource models.DataSourceV2 + if err := c.ShouldBindJSON(&dataSource); err != nil { + HandleErrorBadRequest(c, err) + return + } + + err = service.NewModelServiceV2[models.DataSourceV2]().ReplaceById(id, dataSource) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // check data source status + go func() { + _ = ds.GetDataSourceServiceV2().CheckStatus(id) + }() +} + +func PostDataSourceChangePassword(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + var payload struct { + Password string `json:"password"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if payload.Password == "" { + HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields) + return + } + u := GetUserFromContextV2(c) + if err := ds.GetDataSourceServiceV2().ChangePassword(id, payload.Password, u.Id); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index ec23515e0..fa8a47a6e 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -56,10 +56,26 @@ func InitRoutes(app *gin.Engine) (err error) { groups := NewRouterGroups(app) RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]()) - RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]()) + RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]([]Action{ + { + Method: http.MethodPost, + Path: "", + HandlerFunc: PostDataSource, + }, + { + Method: http.MethodPut, + Path: "/:id", + HandlerFunc: PutDataSourceById, + }, + { + Method: http.MethodPost, + Path: "/:id/change-password", + HandlerFunc: PostDataSourceChangePassword, + }, + }...)) RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) - RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]()) + RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.NotificationSettingV2]()) RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]([]Action{ { diff --git a/core/ds/service_v2.go b/core/ds/service_v2.go new file mode 100644 index 000000000..120939494 --- /dev/null +++ b/core/ds/service_v2.go @@ -0,0 +1,271 @@ +package ds + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + constants2 "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/utils" + utils2 "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // internals + timeout time.Duration + monitorInterval time.Duration + stopped bool +} + +func (svc *ServiceV2) Init() { + // result service registry + reg := result.GetResultServiceRegistry() + + // register result services + reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService) + reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService) + reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService) + reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService) + reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService) + reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService) + reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService) + reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService) +} + +func (svc *ServiceV2) Start() { + // start monitoring + go svc.Monitor() +} + +func (svc *ServiceV2) Wait() { + utils.DefaultWait() +} + +func (svc *ServiceV2) Stop() { + svc.stopped = true +} + +func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { + dataSource, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(id) + if err != nil { + return err + } + dataSource.Password, err = utils.EncryptAES(password) + if err != nil { + return err + } + dataSource.SetUpdated(by) + err = service.NewModelServiceV2[models.DataSourceV2]().ReplaceById(id, *dataSource) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) Monitor() { + for { + // return if stopped + if svc.stopped { + return + } + + // monitor + if err := svc.monitor(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.monitorInterval) + } +} + +func (svc *ServiceV2) CheckStatus(id primitive.ObjectID) (err error) { + ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(id) + if err != nil { + return err + } + return svc.checkStatus(ds, true) +} + +func (svc *ServiceV2) SetTimeout(duration time.Duration) { + svc.timeout = duration +} + +func (svc *ServiceV2) SetMonitorInterval(duration time.Duration) { + svc.monitorInterval = duration +} + +func (svc *ServiceV2) monitor() (err error) { + // start + tic := time.Now() + log.Debugf("[DataSourceService] start monitoring") + + // data source list + dataSources, err := service.NewModelServiceV2[models.DataSourceV2]().GetMany(nil, nil) + if err != nil { + return err + } + + // waiting group + wg := sync.WaitGroup{} + wg.Add(len(dataSources)) + + // iterate data source list + for _, ds := range dataSources { + // async operation + go func(ds *models.DataSourceV2) { + // check status and save + _ = svc.checkStatus(ds, true) + + // release + wg.Done() + }(&ds) + } + + // wait + wg.Wait() + + // finish + toc := time.Now() + log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds()) + + return nil +} + +func (svc *ServiceV2) checkStatus(ds *models.DataSourceV2, save bool) (err error) { + // check status + if err := svc._checkStatus(ds); err != nil { + ds.Status = constants2.DataSourceStatusOffline + ds.Error = err.Error() + } else { + ds.Status = constants2.DataSourceStatusOnline + ds.Error = "" + } + + // save + if save { + return svc._save(ds) + } + + return nil +} + +func (svc *ServiceV2) _save(ds *models.DataSourceV2) (err error) { + log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error) + return service.NewModelServiceV2[models.DataSourceV2]().ReplaceById(ds.Id, *ds) +} + +func (svc *ServiceV2) _checkStatus(ds *models.DataSourceV2) (err error) { + switch ds.Type { + case constants.DataSourceTypeMongo: + _, err := utils2.GetMongoClientWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + case constants.DataSourceTypeMysql: + s, err := utils2.GetMysqlSessionWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + err := s.Close() + if err != nil { + return err + } + } + case constants.DataSourceTypePostgresql: + s, err := utils2.GetPostgresqlSessionWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + err := s.Close() + if err != nil { + return err + } + } + case constants.DataSourceTypeMssql: + s, err := utils2.GetMssqlSessionWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + err := s.Close() + if err != nil { + return err + } + } + case constants.DataSourceTypeSqlite: + s, err := utils2.GetSqliteSessionWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + err := s.Close() + if err != nil { + return err + } + } + case constants.DataSourceTypeCockroachdb: + s, err := utils2.GetCockroachdbSessionWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + err := s.Close() + if err != nil { + return err + } + } + case constants.DataSourceTypeElasticSearch: + _, err := utils2.GetElasticsearchClientWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + case constants.DataSourceTypeKafka: + c, err := utils2.GetKafkaConnectionWithTimeoutV2(ds, svc.timeout) + if err != nil { + return err + } + if c != nil { + err := c.Close() + if err != nil { + return err + } + } + default: + log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type) + } + return nil +} + +func NewDataSourceServiceV2() *ServiceV2 { + // service + svc := &ServiceV2{ + monitorInterval: 15 * time.Second, + timeout: 10 * time.Second, + } + + // initialize + svc.Init() + + // start + svc.Start() + + return svc +} + +var _dsSvcV2 *ServiceV2 + +func GetDataSourceServiceV2() *ServiceV2 { + if _dsSvcV2 != nil { + return _dsSvcV2 + } + _dsSvcV2 = NewDataSourceServiceV2() + return _dsSvcV2 +} diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 17771925d..3050e118d 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -26,6 +26,7 @@ var ( *new(models.EnvironmentV2), *new(models.GitV2), *new(models.NodeV2), + *new(models.NotificationSettingV2), *new(models.PermissionV2), *new(models.ProjectV2), *new(models.RolePermissionV2), diff --git a/core/models/models/data_source_v2.go b/core/models/models/data_source_v2.go index eb81e199f..16415510c 100644 --- a/core/models/models/data_source_v2.go +++ b/core/models/models/data_source_v2.go @@ -12,7 +12,7 @@ type DataSourceV2 struct { Hosts []string `json:"hosts" bson:"hosts"` Database string `json:"database" bson:"database"` Username string `json:"username" bson:"username"` - Password string `json:"password,omitempty" bson:"-"` + Password string `json:"-,omitempty" bson:"password"` ConnectType string `json:"connect_type" bson:"connect_type"` Status string `json:"status" bson:"status"` Error string `json:"error" bson:"error"` diff --git a/core/models/models/notification_setting_v2.go b/core/models/models/notification_setting_v2.go index 6b5654109..f4e183626 100644 --- a/core/models/models/notification_setting_v2.go +++ b/core/models/models/notification_setting_v2.go @@ -3,17 +3,19 @@ package models import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationSettingV2 struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` + any `collection:"notification_settings"` + BaseModelV2[NotificationSettingV2] `bson:",inline"` + Id primitive.ObjectID `json:"_id" bson:"_id"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` } type NotificationSettingMail struct { diff --git a/core/models/models/spider_v2.go b/core/models/models/spider_v2.go index 939bad519..ac3bf3007 100644 --- a/core/models/models/spider_v2.go +++ b/core/models/models/spider_v2.go @@ -5,7 +5,7 @@ import ( ) type SpiderV2 struct { - any `collection:"spiders"` // spider id + any `collection:"spiders"` BaseModelV2[SpiderV2] `bson:",inline"` Name string `json:"name" bson:"name"` // spider name Type string `json:"type" bson:"type"` // spider type diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 13aa69c1e..6c168c106 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -36,9 +36,9 @@ type MasterServiceV2 struct { schedulerSvc *scheduler.ServiceV2 handlerSvc *handler.ServiceV2 scheduleSvc *schedule.ServiceV2 - notificationSvc *notification.Service + notificationSvc *notification.ServiceV2 spiderAdminSvc *admin.ServiceV2 - systemSvc *system.Service + systemSvc *system.ServiceV2 // settings cfgPath string @@ -368,7 +368,7 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { } // notification service - svc.notificationSvc = notification.GetService() + svc.notificationSvc = notification.GetServiceV2() // spider admin service svc.spiderAdminSvc, err = admin.GetSpiderAdminServiceV2() @@ -377,7 +377,7 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { } // system service - svc.systemSvc = system.GetService() + svc.systemSvc = system.GetServiceV2() // init if err := svc.Init(); err != nil { diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 6db6a01a5..10ddca3e4 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -41,7 +41,6 @@ func (svc *ServiceV2) initData() (err error) { // data to initialize settings := []models.NotificationSettingV2{ { - Id: primitive.NewObjectID(), Type: TypeMail, Enabled: true, Name: "任务通知（邮件）", @@ -77,7 +76,6 @@ func (svc *ServiceV2) initData() (err error) { }, }, { - Id: primitive.NewObjectID(), Type: TypeMail, Enabled: true, Name: "Task Change (Mail)", @@ -113,7 +111,6 @@ Please find the task data as below. }, }, { - Id: primitive.NewObjectID(), Type: TypeMobile, Enabled: true, Name: "任务通知（移动端）", @@ -142,7 +139,6 @@ Please find the task data as below. Mobile: models.NotificationSettingMobile{}, }, { - Id: primitive.NewObjectID(), Type: TypeMobile, Enabled: true, Name: "Task Change (Mobile)", diff --git a/core/system/service_v2.go b/core/system/service_v2.go new file mode 100644 index 000000000..e61513ed7 --- /dev/null +++ b/core/system/service_v2.go @@ -0,0 +1,67 @@ +package system + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "go.mongodb.org/mongo-driver/bson" +) + +type ServiceV2 struct { +} + +func (svc *ServiceV2) Init() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) initData() (err error) { + total, err := service.NewModelServiceV2[models.SettingV2]().Count(bson.M{ + "key": "site_title", + }) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.SettingV2{ + { + Key: "site_title", + Value: bson.M{ + "customize_site_title": false, + "site_title": "", + }, + }, + } + _, err = service.NewModelServiceV2[models.SettingV2]().InsertMany(settings) + if err != nil { + return err + } + return nil +} + +func NewServiceV2() *ServiceV2 { + // service + svc := &ServiceV2{} + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _serviceV2 *ServiceV2 + +func GetServiceV2() *ServiceV2 { + if _serviceV2 == nil { + _serviceV2 = NewServiceV2() + } + return _serviceV2 +} diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go index 650c32f1e..feccc5eca 100644 --- a/core/utils/cockroachdb.go +++ b/core/utils/cockroachdb.go @@ -58,3 +58,48 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses return s, err } + +func GetCockroachdbSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getCockroachdbSessionV2(ctx, ds) +} + +func getCockroachdbSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultCockroachdbPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/es.go b/core/utils/es.go index 6636d0da4..7723205b0 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -50,34 +50,75 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas Addresses: addresses, Username: ds.Username, Password: ds.Password, - //CloudID: "", - //APIKey: "", - //ServiceToken: "", - //CertificateFingerprint: "", - //Header: nil, - //CACert: nil, - //RetryOnStatus: nil, - //DisableRetry: false, - //EnableRetryOnTimeout: false, - //MaxRetries: 0, - //CompressRequestBody: false, - //DiscoverNodesOnStart: false, - //DiscoverNodesInterval: 0, - //EnableMetrics: false, - //EnableDebugLogger: false, - //EnableCompatibilityMode: false, - //DisableMetaHeader: false, - //UseResponseCheckOnly: false, RetryBackoff: func(i int) time.Duration { if i == 1 { rb.Reset() } return rb.NextBackOff() }, - //Transport: nil, - //Logger: nil, - //Selector: nil, - //ConnectionPoolFunc: nil, + } + + // es client + done := make(chan struct{}) + go func() { + c, err = elasticsearch.NewClient(cfg) + if err != nil { + return + } + var res *esapi.Response + res, err = c.Info() + fmt.Println(res) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return c, err +} + +func GetElasticsearchClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getElasticsearchClientV2(ctx, ds) +} + +func getElasticsearchClientV2(ctx context.Context, ds *models.DataSourceV2) (c *elasticsearch.Client, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultElasticsearchPort + } + + // es hosts + addresses := []string{ + fmt.Sprintf("http://%s:%s", host, port), + } + + // retry backoff + rb := backoff.NewExponentialBackOff() + + // es client options + cfg := elasticsearch.Config{ + Addresses: addresses, + Username: ds.Username, + Password: ds.Password, + RetryBackoff: func(i int) time.Duration { + if i == 1 { + rb.Reset() + } + return rb.NextBackOff() + }, } // es client diff --git a/core/utils/kafka.go b/core/utils/kafka.go index 1bf005f2a..e33603922 100644 --- a/core/utils/kafka.go +++ b/core/utils/kafka.go @@ -39,3 +39,30 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co // kafka connection return kafka.DialLeader(ctx, network, address, topic, partition) } + +func GetKafkaConnectionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getKafkaConnectionV2(ctx, ds) +} + +func getKafkaConnectionV2(ctx context.Context, ds *models.DataSourceV2) (c *kafka.Conn, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultKafkaPort + } + + // kafka connection address + network := "tcp" + address := fmt.Sprintf("%s:%s", host, port) + topic := ds.Database + partition := 0 // TODO: parameterize + + // kafka connection + return kafka.DialLeader(ctx, network, address, topic, partition) +} diff --git a/core/utils/mongo.go b/core/utils/mongo.go index 5807f24aa..23e92541d 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -54,6 +54,12 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c return getMongoClient(ctx, ds) } +func GetMongoClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMongoClientV2(ctx, ds) +} + func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Client, err error) { // normalize settings if ds.Host == "" { @@ -92,3 +98,42 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien // client return mongo.GetMongoClient(opts...) } + +func getMongoClientV2(ctx context.Context, ds *models.DataSourceV2) (c *mongo2.Client, err error) { + // normalize settings + if ds.Host == "" { + ds.Host = constants.DefaultHost + } + if ds.Port == "" { + ds.Port = constants.DefaultMongoPort + } + + // options + var opts []mongo.ClientOption + opts = append(opts, mongo.WithContext(ctx)) + opts = append(opts, mongo.WithUri(ds.Url)) + opts = append(opts, mongo.WithHost(ds.Host)) + opts = append(opts, mongo.WithPort(ds.Port)) + opts = append(opts, mongo.WithDb(ds.Database)) + opts = append(opts, mongo.WithUsername(ds.Username)) + opts = append(opts, mongo.WithPassword(ds.Password)) + opts = append(opts, mongo.WithHosts(ds.Hosts)) + + // extra + if ds.Extra != nil { + // auth source + authSource, ok := ds.Extra["auth_source"] + if ok { + opts = append(opts, mongo.WithAuthSource(authSource)) + } + + // auth mechanism + authMechanism, ok := ds.Extra["auth_mechanism"] + if ok { + opts = append(opts, mongo.WithAuthMechanism(authMechanism)) + } + } + + // client + return mongo.GetMongoClient(opts...) +} diff --git a/core/utils/mssql.go b/core/utils/mssql.go index 0fb21353b..03329eae5 100644 --- a/core/utils/mssql.go +++ b/core/utils/mssql.go @@ -58,3 +58,48 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } + +func GetMssqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMssqlSessionV2(ctx, ds) +} + +func getMssqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMssqlPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/mysql.go b/core/utils/mysql.go index 8fc9ae18f..c1e78a20c 100644 --- a/core/utils/mysql.go +++ b/core/utils/mysql.go @@ -58,3 +58,48 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } + +func GetMysqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMysqlSessionV2(ctx, ds) +} + +func getMysqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMysqlPort + } + + // connect settings + settings := mysql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mysql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go index 40c8a208a..cf3ba7dc5 100644 --- a/core/utils/postgresql.go +++ b/core/utils/postgresql.go @@ -58,3 +58,48 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess return s, err } + +func GetPostgresqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getPostgresqlSessionV2(ctx, ds) +} + +func getPostgresqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultPostgresqlPort + } + + // connect settings + settings := postgresql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = postgresql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go index 1d6ff682a..83a06cbcf 100644 --- a/core/utils/sqlite.go +++ b/core/utils/sqlite.go @@ -43,3 +43,35 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } + +func GetSqliteSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getSqliteSessionV2(ctx, ds) +} + +func getSqliteSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { + // connect settings + settings := sqlite.ConnectionURL{ + Database: ds.Database, + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = sqlite.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} From 73a5a64ba4ce8baa40521de459705f577b9c32b6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 26 Jun 2024 13:18:28 +0800 Subject: [PATCH 022/106] fix: notification issue --- backend/go.mod | 10 ++++++++-- backend/go.sum | 13 +++++++++---- core/go.mod | 1 + core/go.sum | 6 ++---- core/models/models/notification_setting_v2.go | 3 --- core/notification/service_v2.go | 2 +- fs/go.sum | 1 + go.work.sum | 2 ++ 8 files changed, 24 insertions(+), 14 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index 5ade919d0..d2d7ae5c8 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -43,9 +43,10 @@ require ( github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 // indirect github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/goseaweedfs v0.6.3 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect - github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect - github.com/elastic/go-elasticsearch/v8 v8.7.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect + github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -54,6 +55,8 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-git/go-git/v5 v5.12.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -137,6 +140,9 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.mongodb.org/mongo-driver v1.15.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 7de11f847..4407623e3 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -146,6 +146,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -156,10 +157,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= -github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= -github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= -github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -217,6 +216,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -749,6 +750,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/core/go.mod b/core/go.mod index 9212fd2b5..57637c28e 100644 --- a/core/go.mod +++ b/core/go.mod @@ -70,6 +70,7 @@ require ( github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/crawlab-team/goseaweedfs v0.6.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect diff --git a/core/go.sum b/core/go.sum index b86e23b82..15e4f90ca 100644 --- a/core/go.sum +++ b/core/go.sum @@ -148,6 +148,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -159,12 +160,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= -github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= -github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= -github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= @@ -781,6 +778,7 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= diff --git a/core/models/models/notification_setting_v2.go b/core/models/models/notification_setting_v2.go index f4e183626..b07a01970 100644 --- a/core/models/models/notification_setting_v2.go +++ b/core/models/models/notification_setting_v2.go @@ -1,11 +1,8 @@ package models -import "go.mongodb.org/mongo-driver/bson/primitive" - type NotificationSettingV2 struct { any `collection:"notification_settings"` BaseModelV2[NotificationSettingV2] `bson:",inline"` - Id primitive.ObjectID `json:"_id" bson:"_id"` Type string `json:"type" bson:"type"` Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 10ddca3e4..5a00b7c61 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -143,7 +143,7 @@ Please find the task data as below. Enabled: true, Name: "Task Change (Mobile)", Description: "This is the default mobile notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskError, + TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] Task Update: {{$.status}}", Template: `Dear {{$.user.username}}, diff --git a/fs/go.sum b/fs/go.sum index da7b0b402..00dede36a 100644 --- a/fs/go.sum +++ b/fs/go.sum @@ -7,6 +7,7 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= diff --git a/go.work.sum b/go.work.sum index a07598f35..45fa9773c 100644 --- a/go.work.sum +++ b/go.work.sum @@ -604,6 +604,7 @@ github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -702,6 +703,7 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= From 9c620fa2eec37b656c1a115f7192f4f1e17143e7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 26 Jun 2024 17:31:31 +0800 Subject: [PATCH 023/106] fix: get all spider issue --- core/controllers/spider_v2.go | 2 +- core/grpc/server/metrics_server_v2.go | 12 + grpc/dependencies_service_v2.pb.go | 21 +- grpc/dependencies_service_v2_grpc.pb.go | 33 +- grpc/message_service.pb.go | 11 +- grpc/message_service_grpc.pb.go | 19 +- grpc/metrics_service_v2.pb.go | 473 +++++++++++++++++++ grpc/metrics_service_v2_grpc.pb.go | 142 ++++++ grpc/model_base_service.pb.go | 11 +- grpc/model_base_service_grpc.pb.go | 83 ++-- grpc/model_base_service_v2.pb.go | 11 +- grpc/model_base_service_v2_grpc.pb.go | 95 ++-- grpc/model_delegate.pb.go | 11 +- grpc/model_delegate_grpc.pb.go | 17 +- grpc/model_service_v2_request.pb.go | 39 +- grpc/node.pb.go | 13 +- grpc/node_info.pb.go | 13 +- grpc/node_service.pb.go | 11 +- grpc/node_service_grpc.pb.go | 43 +- grpc/plugin_request.pb.go | 13 +- grpc/plugin_service.pb.go | 11 +- grpc/plugin_service_grpc.pb.go | 33 +- grpc/proto/services/metrics_service_v2.proto | 36 ++ grpc/request.pb.go | 13 +- grpc/response.pb.go | 13 +- grpc/response_code.pb.go | 11 +- grpc/stream_message.pb.go | 13 +- grpc/stream_message_code.pb.go | 11 +- grpc/stream_message_data_task.pb.go | 13 +- grpc/task.pb.go | 13 +- grpc/task_service.pb.go | 11 +- grpc/task_service_grpc.pb.go | 31 +- 32 files changed, 979 insertions(+), 303 deletions(-) create mode 100644 core/grpc/server/metrics_server_v2.go create mode 100644 grpc/metrics_service_v2.pb.go create mode 100644 grpc/metrics_service_v2_grpc.pb.go create mode 100644 grpc/proto/services/metrics_service_v2.proto diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 78b4d2492..5887b58d6 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -68,7 +68,7 @@ func GetSpiderList(c *gin.Context) { // get all list all := MustGetFilterAll(c) if all { - NewControllerV2[models.ProjectV2]().getAll(c) + NewControllerV2[models.SpiderV2]().getAll(c) return } diff --git a/core/grpc/server/metrics_server_v2.go b/core/grpc/server/metrics_server_v2.go new file mode 100644 index 000000000..19a0a0d50 --- /dev/null +++ b/core/grpc/server/metrics_server_v2.go @@ -0,0 +1,12 @@ +package server + +import ( + "github.com/crawlab-team/crawlab/grpc" + "sync" +) + +type MetricsServerV2 struct { + grpc.UnimplementedMetricsServiceV2Server + mu *sync.Mutex + streams map[string]*grpc.MetricsServiceV2_ConnectServer +} diff --git a/grpc/dependencies_service_v2.pb.go b/grpc/dependencies_service_v2.pb.go index 6966b1d95..00e716da2 100644 --- a/grpc/dependencies_service_v2.pb.go +++ b/grpc/dependencies_service_v2.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/dependencies_service_v2.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type DependenciesServiceV2Code int32 const ( @@ -456,7 +451,7 @@ func file_services_dependencies_service_v2_proto_rawDescGZIP() []byte { var file_services_dependencies_service_v2_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_services_dependencies_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_services_dependencies_service_v2_proto_goTypes = []interface{}{ +var file_services_dependencies_service_v2_proto_goTypes = []any{ (DependenciesServiceV2Code)(0), // 0: grpc.DependenciesServiceV2Code (*Dependency)(nil), // 1: grpc.Dependency (*DependenciesServiceV2ConnectRequest)(nil), // 2: grpc.DependenciesServiceV2ConnectRequest @@ -489,7 +484,7 @@ func file_services_dependencies_service_v2_proto_init() { } file_entity_response_proto_init() if !protoimpl.UnsafeEnabled { - file_services_dependencies_service_v2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_services_dependencies_service_v2_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Dependency); i { case 0: return &v.state @@ -501,7 +496,7 @@ func file_services_dependencies_service_v2_proto_init() { return nil } } - file_services_dependencies_service_v2_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_services_dependencies_service_v2_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*DependenciesServiceV2ConnectRequest); i { case 0: return &v.state @@ -513,7 +508,7 @@ func file_services_dependencies_service_v2_proto_init() { return nil } } - file_services_dependencies_service_v2_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_services_dependencies_service_v2_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*DependenciesServiceV2ConnectResponse); i { case 0: return &v.state @@ -525,7 +520,7 @@ func file_services_dependencies_service_v2_proto_init() { return nil } } - file_services_dependencies_service_v2_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_services_dependencies_service_v2_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*DependenciesServiceV2SyncRequest); i { case 0: return &v.state @@ -537,7 +532,7 @@ func file_services_dependencies_service_v2_proto_init() { return nil } } - file_services_dependencies_service_v2_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_services_dependencies_service_v2_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*DependenciesServiceV2UpdateTaskLogRequest); i { case 0: return &v.state diff --git a/grpc/dependencies_service_v2_grpc.pb.go b/grpc/dependencies_service_v2_grpc.pb.go index e76d3bda4..7afcf1671 100644 --- a/grpc/dependencies_service_v2_grpc.pb.go +++ b/grpc/dependencies_service_v2_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/dependencies_service_v2.proto package grpc @@ -15,8 +15,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + DependenciesServiceV2_Connect_FullMethodName = "/grpc.DependenciesServiceV2/Connect" + DependenciesServiceV2_Sync_FullMethodName = "/grpc.DependenciesServiceV2/Sync" + DependenciesServiceV2_UpdateTaskLog_FullMethodName = "/grpc.DependenciesServiceV2/UpdateTaskLog" +) // DependenciesServiceV2Client is the client API for DependenciesServiceV2 service. // @@ -36,11 +42,12 @@ func NewDependenciesServiceV2Client(cc grpc.ClientConnInterface) DependenciesSer } func (c *dependenciesServiceV2Client) Connect(ctx context.Context, in *DependenciesServiceV2ConnectRequest, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) { - stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], "/grpc.DependenciesServiceV2/Connect", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], DependenciesServiceV2_Connect_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &dependenciesServiceV2ConnectClient{stream} + x := &dependenciesServiceV2ConnectClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -68,8 +75,9 @@ func (x *dependenciesServiceV2ConnectClient) Recv() (*DependenciesServiceV2Conne } func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.DependenciesServiceV2/Sync", in, out, opts...) + err := c.cc.Invoke(ctx, DependenciesServiceV2_Sync_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -77,11 +85,12 @@ func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *Dependencies } func (c *dependenciesServiceV2Client) UpdateTaskLog(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_UpdateTaskLogClient, error) { - stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], "/grpc.DependenciesServiceV2/UpdateTaskLog", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], DependenciesServiceV2_UpdateTaskLog_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &dependenciesServiceV2UpdateTaskLogClient{stream} + x := &dependenciesServiceV2UpdateTaskLogClient{ClientStream: stream} return x, nil } @@ -151,7 +160,7 @@ func _DependenciesServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerS if err := stream.RecvMsg(m); err != nil { return err } - return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{stream}) + return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{ServerStream: stream}) } type DependenciesServiceV2_ConnectServer interface { @@ -177,7 +186,7 @@ func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.DependenciesServiceV2/Sync", + FullMethod: DependenciesServiceV2_Sync_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(DependenciesServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) @@ -186,7 +195,7 @@ func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, d } func _DependenciesServiceV2_UpdateTaskLog_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{stream}) + return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{ServerStream: stream}) } type DependenciesServiceV2_UpdateTaskLogServer interface { diff --git a/grpc/message_service.pb.go b/grpc/message_service.pb.go index fca04128e..a651f61d4 100644 --- a/grpc/message_service.pb.go +++ b/grpc/message_service.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/message_service.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_message_service_proto protoreflect.FileDescriptor var file_services_message_service_proto_rawDesc = []byte{ @@ -40,7 +35,7 @@ var file_services_message_service_proto_rawDesc = []byte{ 0x6f, 0x33, } -var file_services_message_service_proto_goTypes = []interface{}{ +var file_services_message_service_proto_goTypes = []any{ (*StreamMessage)(nil), // 0: grpc.StreamMessage } var file_services_message_service_proto_depIdxs = []int32{ diff --git a/grpc/message_service_grpc.pb.go b/grpc/message_service_grpc.pb.go index c1cc17b0a..cc6dc28f8 100644 --- a/grpc/message_service_grpc.pb.go +++ b/grpc/message_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/message_service.proto package grpc @@ -15,8 +15,12 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + MessageService_Connect_FullMethodName = "/grpc.MessageService/Connect" +) // MessageServiceClient is the client API for MessageService service. // @@ -34,11 +38,12 @@ func NewMessageServiceClient(cc grpc.ClientConnInterface) MessageServiceClient { } func (c *messageServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) { - stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], "/grpc.MessageService/Connect", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], MessageService_Connect_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &messageServiceConnectClient{stream} + x := &messageServiceConnectClient{ClientStream: stream} return x, nil } @@ -93,7 +98,7 @@ func RegisterMessageServiceServer(s grpc.ServiceRegistrar, srv MessageServiceSer } func _MessageService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{stream}) + return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{ServerStream: stream}) } type MessageService_ConnectServer interface { diff --git a/grpc/metrics_service_v2.pb.go b/grpc/metrics_service_v2.pb.go new file mode 100644 index 000000000..6ab0e542a --- /dev/null +++ b/grpc/metrics_service_v2.pb.go @@ -0,0 +1,473 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.2 +// protoc v3.12.4 +// source: services/metrics_service_v2.proto + +package grpc + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type MetricsServiceV2Code int32 + +const ( + MetricsServiceV2Code_SYNC_METRICS MetricsServiceV2Code = 0 +) + +// Enum value maps for MetricsServiceV2Code. +var ( + MetricsServiceV2Code_name = map[int32]string{ + 0: "SYNC_METRICS", + } + MetricsServiceV2Code_value = map[string]int32{ + "SYNC_METRICS": 0, + } +) + +func (x MetricsServiceV2Code) Enum() *MetricsServiceV2Code { + p := new(MetricsServiceV2Code) + *p = x + return p +} + +func (x MetricsServiceV2Code) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MetricsServiceV2Code) Descriptor() protoreflect.EnumDescriptor { + return file_services_metrics_service_v2_proto_enumTypes[0].Descriptor() +} + +func (MetricsServiceV2Code) Type() protoreflect.EnumType { + return &file_services_metrics_service_v2_proto_enumTypes[0] +} + +func (x MetricsServiceV2Code) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MetricsServiceV2Code.Descriptor instead. +func (MetricsServiceV2Code) EnumDescriptor() ([]byte, []int) { + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{0} +} + +type Metric struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + Subsystem string `protobuf:"bytes,2,opt,name=subsystem,proto3" json:"subsystem,omitempty"` + Module string `protobuf:"bytes,3,opt,name=module,proto3" json:"module,omitempty"` + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + Value float32 `protobuf:"fixed32,5,opt,name=value,proto3" json:"value,omitempty"` + Help string `protobuf:"bytes,6,opt,name=help,proto3" json:"help,omitempty"` + Labels string `protobuf:"bytes,7,opt,name=labels,proto3" json:"labels,omitempty"` + Process *MetricProcess `protobuf:"bytes,8,opt,name=process,proto3" json:"process,omitempty"` +} + +func (x *Metric) Reset() { + *x = Metric{} + if protoimpl.UnsafeEnabled { + mi := &file_services_metrics_service_v2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Metric) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metric) ProtoMessage() {} + +func (x *Metric) ProtoReflect() protoreflect.Message { + mi := &file_services_metrics_service_v2_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metric.ProtoReflect.Descriptor instead. +func (*Metric) Descriptor() ([]byte, []int) { + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{0} +} + +func (x *Metric) GetNamespace() string { + if x != nil { + return x.Namespace + } + return "" +} + +func (x *Metric) GetSubsystem() string { + if x != nil { + return x.Subsystem + } + return "" +} + +func (x *Metric) GetModule() string { + if x != nil { + return x.Module + } + return "" +} + +func (x *Metric) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Metric) GetValue() float32 { + if x != nil { + return x.Value + } + return 0 +} + +func (x *Metric) GetHelp() string { + if x != nil { + return x.Help + } + return "" +} + +func (x *Metric) GetLabels() string { + if x != nil { + return x.Labels + } + return "" +} + +func (x *Metric) GetProcess() *MetricProcess { + if x != nil { + return x.Process + } + return nil +} + +type MetricProcess struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pid int32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` +} + +func (x *MetricProcess) Reset() { + *x = MetricProcess{} + if protoimpl.UnsafeEnabled { + mi := &file_services_metrics_service_v2_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MetricProcess) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricProcess) ProtoMessage() {} + +func (x *MetricProcess) ProtoReflect() protoreflect.Message { + mi := &file_services_metrics_service_v2_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricProcess.ProtoReflect.Descriptor instead. +func (*MetricProcess) Descriptor() ([]byte, []int) { + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{1} +} + +func (x *MetricProcess) GetPid() int32 { + if x != nil { + return x.Pid + } + return 0 +} + +type MetricsServiceV2ConnectRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Metrics []*Metric `protobuf:"bytes,2,rep,name=metrics,proto3" json:"metrics,omitempty"` +} + +func (x *MetricsServiceV2ConnectRequest) Reset() { + *x = MetricsServiceV2ConnectRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_metrics_service_v2_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MetricsServiceV2ConnectRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsServiceV2ConnectRequest) ProtoMessage() {} + +func (x *MetricsServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_metrics_service_v2_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsServiceV2ConnectRequest.ProtoReflect.Descriptor instead. +func (*MetricsServiceV2ConnectRequest) Descriptor() ([]byte, []int) { + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{2} +} + +func (x *MetricsServiceV2ConnectRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *MetricsServiceV2ConnectRequest) GetMetrics() []*Metric { + if x != nil { + return x.Metrics + } + return nil +} + +type MetricsServiceV2ConnectResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code MetricsServiceV2Code `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.MetricsServiceV2Code" json:"code,omitempty"` +} + +func (x *MetricsServiceV2ConnectResponse) Reset() { + *x = MetricsServiceV2ConnectResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_services_metrics_service_v2_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MetricsServiceV2ConnectResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricsServiceV2ConnectResponse) ProtoMessage() {} + +func (x *MetricsServiceV2ConnectResponse) ProtoReflect() protoreflect.Message { + mi := &file_services_metrics_service_v2_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricsServiceV2ConnectResponse.ProtoReflect.Descriptor instead. +func (*MetricsServiceV2ConnectResponse) Descriptor() ([]byte, []int) { + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{3} +} + +func (x *MetricsServiceV2ConnectResponse) GetCode() MetricsServiceV2Code { + if x != nil { + return x.Code + } + return MetricsServiceV2Code_SYNC_METRICS +} + +var File_services_metrics_service_v2_proto protoreflect.FileDescriptor + +var file_services_metrics_service_v2_proto_rawDesc = []byte{ + 0x0a, 0x21, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0xe1, 0x01, 0x0a, 0x06, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x2d, + 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x50, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x22, 0x21, 0x0a, + 0x0d, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x10, + 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, 0x64, + 0x22, 0x63, 0x0a, 0x1e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, + 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0x51, 0x0a, 0x1f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, + 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x2a, 0x28, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, 0x65, + 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x53, + 0x10, 0x00, 0x32, 0x70, 0x0a, 0x10, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x12, 0x5c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_services_metrics_service_v2_proto_rawDescOnce sync.Once + file_services_metrics_service_v2_proto_rawDescData = file_services_metrics_service_v2_proto_rawDesc +) + +func file_services_metrics_service_v2_proto_rawDescGZIP() []byte { + file_services_metrics_service_v2_proto_rawDescOnce.Do(func() { + file_services_metrics_service_v2_proto_rawDescData = protoimpl.X.CompressGZIP(file_services_metrics_service_v2_proto_rawDescData) + }) + return file_services_metrics_service_v2_proto_rawDescData +} + +var file_services_metrics_service_v2_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_services_metrics_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_services_metrics_service_v2_proto_goTypes = []any{ + (MetricsServiceV2Code)(0), // 0: grpc.MetricsServiceV2Code + (*Metric)(nil), // 1: grpc.Metric + (*MetricProcess)(nil), // 2: grpc.MetricProcess + (*MetricsServiceV2ConnectRequest)(nil), // 3: grpc.MetricsServiceV2ConnectRequest + (*MetricsServiceV2ConnectResponse)(nil), // 4: grpc.MetricsServiceV2ConnectResponse +} +var file_services_metrics_service_v2_proto_depIdxs = []int32{ + 2, // 0: grpc.Metric.process:type_name -> grpc.MetricProcess + 1, // 1: grpc.MetricsServiceV2ConnectRequest.metrics:type_name -> grpc.Metric + 0, // 2: grpc.MetricsServiceV2ConnectResponse.code:type_name -> grpc.MetricsServiceV2Code + 3, // 3: grpc.MetricsServiceV2.Connect:input_type -> grpc.MetricsServiceV2ConnectRequest + 4, // 4: grpc.MetricsServiceV2.Connect:output_type -> grpc.MetricsServiceV2ConnectResponse + 4, // [4:5] is the sub-list for method output_type + 3, // [3:4] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_services_metrics_service_v2_proto_init() } +func file_services_metrics_service_v2_proto_init() { + if File_services_metrics_service_v2_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_services_metrics_service_v2_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*Metric); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_metrics_service_v2_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*MetricProcess); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_metrics_service_v2_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*MetricsServiceV2ConnectRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_metrics_service_v2_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*MetricsServiceV2ConnectResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_metrics_service_v2_proto_rawDesc, + NumEnums: 1, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_metrics_service_v2_proto_goTypes, + DependencyIndexes: file_services_metrics_service_v2_proto_depIdxs, + EnumInfos: file_services_metrics_service_v2_proto_enumTypes, + MessageInfos: file_services_metrics_service_v2_proto_msgTypes, + }.Build() + File_services_metrics_service_v2_proto = out.File + file_services_metrics_service_v2_proto_rawDesc = nil + file_services_metrics_service_v2_proto_goTypes = nil + file_services_metrics_service_v2_proto_depIdxs = nil +} diff --git a/grpc/metrics_service_v2_grpc.pb.go b/grpc/metrics_service_v2_grpc.pb.go new file mode 100644 index 000000000..e381d8dba --- /dev/null +++ b/grpc/metrics_service_v2_grpc.pb.go @@ -0,0 +1,142 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 +// source: services/metrics_service_v2.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + MetricsServiceV2_Connect_FullMethodName = "/grpc.MetricsServiceV2/Connect" +) + +// MetricsServiceV2Client is the client API for MetricsServiceV2 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type MetricsServiceV2Client interface { + Connect(ctx context.Context, opts ...grpc.CallOption) (MetricsServiceV2_ConnectClient, error) +} + +type metricsServiceV2Client struct { + cc grpc.ClientConnInterface +} + +func NewMetricsServiceV2Client(cc grpc.ClientConnInterface) MetricsServiceV2Client { + return &metricsServiceV2Client{cc} +} + +func (c *metricsServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (MetricsServiceV2_ConnectClient, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &MetricsServiceV2_ServiceDesc.Streams[0], MetricsServiceV2_Connect_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &metricsServiceV2ConnectClient{ClientStream: stream} + return x, nil +} + +type MetricsServiceV2_ConnectClient interface { + Send(*MetricsServiceV2ConnectRequest) error + Recv() (*MetricsServiceV2ConnectResponse, error) + grpc.ClientStream +} + +type metricsServiceV2ConnectClient struct { + grpc.ClientStream +} + +func (x *metricsServiceV2ConnectClient) Send(m *MetricsServiceV2ConnectRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *metricsServiceV2ConnectClient) Recv() (*MetricsServiceV2ConnectResponse, error) { + m := new(MetricsServiceV2ConnectResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MetricsServiceV2Server is the server API for MetricsServiceV2 service. +// All implementations must embed UnimplementedMetricsServiceV2Server +// for forward compatibility +type MetricsServiceV2Server interface { + Connect(MetricsServiceV2_ConnectServer) error + mustEmbedUnimplementedMetricsServiceV2Server() +} + +// UnimplementedMetricsServiceV2Server must be embedded to have forward compatible implementations. +type UnimplementedMetricsServiceV2Server struct { +} + +func (UnimplementedMetricsServiceV2Server) Connect(MetricsServiceV2_ConnectServer) error { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedMetricsServiceV2Server) mustEmbedUnimplementedMetricsServiceV2Server() {} + +// UnsafeMetricsServiceV2Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to MetricsServiceV2Server will +// result in compilation errors. +type UnsafeMetricsServiceV2Server interface { + mustEmbedUnimplementedMetricsServiceV2Server() +} + +func RegisterMetricsServiceV2Server(s grpc.ServiceRegistrar, srv MetricsServiceV2Server) { + s.RegisterService(&MetricsServiceV2_ServiceDesc, srv) +} + +func _MetricsServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(MetricsServiceV2Server).Connect(&metricsServiceV2ConnectServer{ServerStream: stream}) +} + +type MetricsServiceV2_ConnectServer interface { + Send(*MetricsServiceV2ConnectResponse) error + Recv() (*MetricsServiceV2ConnectRequest, error) + grpc.ServerStream +} + +type metricsServiceV2ConnectServer struct { + grpc.ServerStream +} + +func (x *metricsServiceV2ConnectServer) Send(m *MetricsServiceV2ConnectResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *metricsServiceV2ConnectServer) Recv() (*MetricsServiceV2ConnectRequest, error) { + m := new(MetricsServiceV2ConnectRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MetricsServiceV2_ServiceDesc is the grpc.ServiceDesc for MetricsServiceV2 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var MetricsServiceV2_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.MetricsServiceV2", + HandlerType: (*MetricsServiceV2Server)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _MetricsServiceV2_Connect_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/metrics_service_v2.proto", +} diff --git a/grpc/model_base_service.pb.go b/grpc/model_base_service.pb.go index 0f9af6959..f3994f620 100644 --- a/grpc/model_base_service.pb.go +++ b/grpc/model_base_service.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/model_base_service.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_model_base_service_proto protoreflect.FileDescriptor var file_services_model_base_service_proto_rawDesc = []byte{ @@ -71,7 +66,7 @@ var file_services_model_base_service_proto_rawDesc = []byte{ 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -var file_services_model_base_service_proto_goTypes = []interface{}{ +var file_services_model_base_service_proto_goTypes = []any{ (*Request)(nil), // 0: grpc.Request (*Response)(nil), // 1: grpc.Response } diff --git a/grpc/model_base_service_grpc.pb.go b/grpc/model_base_service_grpc.pb.go index 465355ec7..f512986d0 100644 --- a/grpc/model_base_service_grpc.pb.go +++ b/grpc/model_base_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/model_base_service.proto package grpc @@ -15,8 +15,23 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ModelBaseService_GetById_FullMethodName = "/grpc.ModelBaseService/GetById" + ModelBaseService_Get_FullMethodName = "/grpc.ModelBaseService/Get" + ModelBaseService_GetList_FullMethodName = "/grpc.ModelBaseService/GetList" + ModelBaseService_DeleteById_FullMethodName = "/grpc.ModelBaseService/DeleteById" + ModelBaseService_Delete_FullMethodName = "/grpc.ModelBaseService/Delete" + ModelBaseService_DeleteList_FullMethodName = "/grpc.ModelBaseService/DeleteList" + ModelBaseService_ForceDeleteList_FullMethodName = "/grpc.ModelBaseService/ForceDeleteList" + ModelBaseService_UpdateById_FullMethodName = "/grpc.ModelBaseService/UpdateById" + ModelBaseService_Update_FullMethodName = "/grpc.ModelBaseService/Update" + ModelBaseService_UpdateDoc_FullMethodName = "/grpc.ModelBaseService/UpdateDoc" + ModelBaseService_Insert_FullMethodName = "/grpc.ModelBaseService/Insert" + ModelBaseService_Count_FullMethodName = "/grpc.ModelBaseService/Count" +) // ModelBaseServiceClient is the client API for ModelBaseService service. // @@ -45,8 +60,9 @@ func NewModelBaseServiceClient(cc grpc.ClientConnInterface) ModelBaseServiceClie } func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_GetById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -54,8 +70,9 @@ func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts } func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Get", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Get_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -63,8 +80,9 @@ func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...g } func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetList", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_GetList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -72,8 +90,9 @@ func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts } func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_DeleteById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -81,8 +100,9 @@ func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Delete", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Delete_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -90,8 +110,9 @@ func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteList", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_DeleteList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -99,8 +120,9 @@ func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/ForceDeleteList", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_ForceDeleteList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -108,8 +130,9 @@ func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Reques } func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_UpdateById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -117,8 +140,9 @@ func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Update", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Update_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -126,8 +150,9 @@ func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateDoc", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_UpdateDoc_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -135,8 +160,9 @@ func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opt } func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Insert", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Insert_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -144,8 +170,9 @@ func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Count", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Count_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -234,7 +261,7 @@ func _ModelBaseService_GetById_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/GetById", + FullMethod: ModelBaseService_GetById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).GetById(ctx, req.(*Request)) @@ -252,7 +279,7 @@ func _ModelBaseService_Get_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Get", + FullMethod: ModelBaseService_Get_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Get(ctx, req.(*Request)) @@ -270,7 +297,7 @@ func _ModelBaseService_GetList_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/GetList", + FullMethod: ModelBaseService_GetList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).GetList(ctx, req.(*Request)) @@ -288,7 +315,7 @@ func _ModelBaseService_DeleteById_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/DeleteById", + FullMethod: ModelBaseService_DeleteById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).DeleteById(ctx, req.(*Request)) @@ -306,7 +333,7 @@ func _ModelBaseService_Delete_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Delete", + FullMethod: ModelBaseService_Delete_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Delete(ctx, req.(*Request)) @@ -324,7 +351,7 @@ func _ModelBaseService_DeleteList_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/DeleteList", + FullMethod: ModelBaseService_DeleteList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).DeleteList(ctx, req.(*Request)) @@ -342,7 +369,7 @@ func _ModelBaseService_ForceDeleteList_Handler(srv interface{}, ctx context.Cont } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/ForceDeleteList", + FullMethod: ModelBaseService_ForceDeleteList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, req.(*Request)) @@ -360,7 +387,7 @@ func _ModelBaseService_UpdateById_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/UpdateById", + FullMethod: ModelBaseService_UpdateById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).UpdateById(ctx, req.(*Request)) @@ -378,7 +405,7 @@ func _ModelBaseService_Update_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Update", + FullMethod: ModelBaseService_Update_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Update(ctx, req.(*Request)) @@ -396,7 +423,7 @@ func _ModelBaseService_UpdateDoc_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/UpdateDoc", + FullMethod: ModelBaseService_UpdateDoc_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).UpdateDoc(ctx, req.(*Request)) @@ -414,7 +441,7 @@ func _ModelBaseService_Insert_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Insert", + FullMethod: ModelBaseService_Insert_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Insert(ctx, req.(*Request)) @@ -432,7 +459,7 @@ func _ModelBaseService_Count_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Count", + FullMethod: ModelBaseService_Count_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Count(ctx, req.(*Request)) diff --git a/grpc/model_base_service_v2.pb.go b/grpc/model_base_service_v2.pb.go index 22377ca9e..e29db8937 100644 --- a/grpc/model_base_service_v2.pb.go +++ b/grpc/model_base_service_v2.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/model_base_service_v2.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_model_base_service_v2_proto protoreflect.FileDescriptor var file_services_model_base_service_v2_proto_rawDesc = []byte{ @@ -99,7 +94,7 @@ var file_services_model_base_service_v2_proto_rawDesc = []byte{ 0x74, 0x6f, 0x33, } -var file_services_model_base_service_v2_proto_goTypes = []interface{}{ +var file_services_model_base_service_v2_proto_goTypes = []any{ (*ModelServiceV2GetByIdRequest)(nil), // 0: grpc.ModelServiceV2GetByIdRequest (*ModelServiceV2GetOneRequest)(nil), // 1: grpc.ModelServiceV2GetOneRequest (*ModelServiceV2GetManyRequest)(nil), // 2: grpc.ModelServiceV2GetManyRequest diff --git a/grpc/model_base_service_v2_grpc.pb.go b/grpc/model_base_service_v2_grpc.pb.go index 6f5fac73f..805475d33 100644 --- a/grpc/model_base_service_v2_grpc.pb.go +++ b/grpc/model_base_service_v2_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/model_base_service_v2.proto package grpc @@ -15,8 +15,25 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ModelBaseServiceV2_GetById_FullMethodName = "/grpc.ModelBaseServiceV2/GetById" + ModelBaseServiceV2_GetOne_FullMethodName = "/grpc.ModelBaseServiceV2/GetOne" + ModelBaseServiceV2_GetMany_FullMethodName = "/grpc.ModelBaseServiceV2/GetMany" + ModelBaseServiceV2_DeleteById_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteById" + ModelBaseServiceV2_DeleteOne_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteOne" + ModelBaseServiceV2_DeleteMany_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteMany" + ModelBaseServiceV2_UpdateById_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateById" + ModelBaseServiceV2_UpdateOne_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateOne" + ModelBaseServiceV2_UpdateMany_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateMany" + ModelBaseServiceV2_ReplaceById_FullMethodName = "/grpc.ModelBaseServiceV2/ReplaceById" + ModelBaseServiceV2_ReplaceOne_FullMethodName = "/grpc.ModelBaseServiceV2/ReplaceOne" + ModelBaseServiceV2_InsertOne_FullMethodName = "/grpc.ModelBaseServiceV2/InsertOne" + ModelBaseServiceV2_InsertMany_FullMethodName = "/grpc.ModelBaseServiceV2/InsertMany" + ModelBaseServiceV2_Count_FullMethodName = "/grpc.ModelBaseServiceV2/Count" +) // ModelBaseServiceV2Client is the client API for ModelBaseServiceV2 service. // @@ -47,8 +64,9 @@ func NewModelBaseServiceV2Client(cc grpc.ClientConnInterface) ModelBaseServiceV2 } func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -56,8 +74,9 @@ func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelService } func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -65,8 +84,9 @@ func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV } func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -74,8 +94,9 @@ func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelService } func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -83,8 +104,9 @@ func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -92,8 +114,9 @@ func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -101,8 +124,9 @@ func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -110,8 +134,9 @@ func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -119,8 +144,9 @@ func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -128,8 +154,9 @@ func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_ReplaceById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -137,8 +164,9 @@ func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelSer } func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_ReplaceOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -146,8 +174,9 @@ func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_InsertOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -155,8 +184,9 @@ func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_InsertMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -164,8 +194,9 @@ func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/Count", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_Count_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -262,7 +293,7 @@ func _ModelBaseServiceV2_GetById_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/GetById", + FullMethod: ModelBaseServiceV2_GetById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetById(ctx, req.(*ModelServiceV2GetByIdRequest)) @@ -280,7 +311,7 @@ func _ModelBaseServiceV2_GetOne_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/GetOne", + FullMethod: ModelBaseServiceV2_GetOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetOne(ctx, req.(*ModelServiceV2GetOneRequest)) @@ -298,7 +329,7 @@ func _ModelBaseServiceV2_GetMany_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/GetMany", + FullMethod: ModelBaseServiceV2_GetMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetMany(ctx, req.(*ModelServiceV2GetManyRequest)) @@ -316,7 +347,7 @@ func _ModelBaseServiceV2_DeleteById_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/DeleteById", + FullMethod: ModelBaseServiceV2_DeleteById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteById(ctx, req.(*ModelServiceV2DeleteByIdRequest)) @@ -334,7 +365,7 @@ func _ModelBaseServiceV2_DeleteOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/DeleteOne", + FullMethod: ModelBaseServiceV2_DeleteOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, req.(*ModelServiceV2DeleteOneRequest)) @@ -352,7 +383,7 @@ func _ModelBaseServiceV2_DeleteMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/DeleteMany", + FullMethod: ModelBaseServiceV2_DeleteMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, req.(*ModelServiceV2DeleteManyRequest)) @@ -370,7 +401,7 @@ func _ModelBaseServiceV2_UpdateById_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/UpdateById", + FullMethod: ModelBaseServiceV2_UpdateById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateById(ctx, req.(*ModelServiceV2UpdateByIdRequest)) @@ -388,7 +419,7 @@ func _ModelBaseServiceV2_UpdateOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/UpdateOne", + FullMethod: ModelBaseServiceV2_UpdateOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, req.(*ModelServiceV2UpdateOneRequest)) @@ -406,7 +437,7 @@ func _ModelBaseServiceV2_UpdateMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/UpdateMany", + FullMethod: ModelBaseServiceV2_UpdateMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, req.(*ModelServiceV2UpdateManyRequest)) @@ -424,7 +455,7 @@ func _ModelBaseServiceV2_ReplaceById_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/ReplaceById", + FullMethod: ModelBaseServiceV2_ReplaceById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, req.(*ModelServiceV2ReplaceByIdRequest)) @@ -442,7 +473,7 @@ func _ModelBaseServiceV2_ReplaceOne_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/ReplaceOne", + FullMethod: ModelBaseServiceV2_ReplaceOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, req.(*ModelServiceV2ReplaceOneRequest)) @@ -460,7 +491,7 @@ func _ModelBaseServiceV2_InsertOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/InsertOne", + FullMethod: ModelBaseServiceV2_InsertOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).InsertOne(ctx, req.(*ModelServiceV2InsertOneRequest)) @@ -478,7 +509,7 @@ func _ModelBaseServiceV2_InsertMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/InsertMany", + FullMethod: ModelBaseServiceV2_InsertMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).InsertMany(ctx, req.(*ModelServiceV2InsertManyRequest)) @@ -496,7 +527,7 @@ func _ModelBaseServiceV2_Count_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/Count", + FullMethod: ModelBaseServiceV2_Count_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).Count(ctx, req.(*ModelServiceV2CountRequest)) diff --git a/grpc/model_delegate.pb.go b/grpc/model_delegate.pb.go index 76dfa4897..158ed7759 100644 --- a/grpc/model_delegate.pb.go +++ b/grpc/model_delegate.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/model_delegate.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_model_delegate_proto protoreflect.FileDescriptor var file_services_model_delegate_proto_rawDesc = []byte{ @@ -39,7 +34,7 @@ var file_services_model_delegate_proto_rawDesc = []byte{ 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -var file_services_model_delegate_proto_goTypes = []interface{}{ +var file_services_model_delegate_proto_goTypes = []any{ (*Request)(nil), // 0: grpc.Request (*Response)(nil), // 1: grpc.Response } diff --git a/grpc/model_delegate_grpc.pb.go b/grpc/model_delegate_grpc.pb.go index 3ab1efa50..a0614e45b 100644 --- a/grpc/model_delegate_grpc.pb.go +++ b/grpc/model_delegate_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/model_delegate.proto package grpc @@ -15,8 +15,12 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ModelDelegate_Do_FullMethodName = "/grpc.ModelDelegate/Do" +) // ModelDelegateClient is the client API for ModelDelegate service. // @@ -34,8 +38,9 @@ func NewModelDelegateClient(cc grpc.ClientConnInterface) ModelDelegateClient { } func (c *modelDelegateClient) Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelDelegate/Do", in, out, opts...) + err := c.cc.Invoke(ctx, ModelDelegate_Do_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -80,7 +85,7 @@ func _ModelDelegate_Do_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelDelegate/Do", + FullMethod: ModelDelegate_Do_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelDelegateServer).Do(ctx, req.(*Request)) diff --git a/grpc/model_service_v2_request.pb.go b/grpc/model_service_v2_request.pb.go index b8958a9c6..e56d8d8d8 100644 --- a/grpc/model_service_v2_request.pb.go +++ b/grpc/model_service_v2_request.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/model_service_v2_request.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type ModelServiceV2GetByIdRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1096,7 +1091,7 @@ func file_entity_model_service_v2_request_proto_rawDescGZIP() []byte { } var file_entity_model_service_v2_request_proto_msgTypes = make([]protoimpl.MessageInfo, 14) -var file_entity_model_service_v2_request_proto_goTypes = []interface{}{ +var file_entity_model_service_v2_request_proto_goTypes = []any{ (*ModelServiceV2GetByIdRequest)(nil), // 0: grpc.ModelServiceV2GetByIdRequest (*ModelServiceV2GetOneRequest)(nil), // 1: grpc.ModelServiceV2GetOneRequest (*ModelServiceV2GetManyRequest)(nil), // 2: grpc.ModelServiceV2GetManyRequest @@ -1126,7 +1121,7 @@ func file_entity_model_service_v2_request_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_entity_model_service_v2_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2GetByIdRequest); i { case 0: return &v.state @@ -1138,7 +1133,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2GetOneRequest); i { case 0: return &v.state @@ -1150,7 +1145,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2GetManyRequest); i { case 0: return &v.state @@ -1162,7 +1157,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2DeleteByIdRequest); i { case 0: return &v.state @@ -1174,7 +1169,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2DeleteOneRequest); i { case 0: return &v.state @@ -1186,7 +1181,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2DeleteManyRequest); i { case 0: return &v.state @@ -1198,7 +1193,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2UpdateByIdRequest); i { case 0: return &v.state @@ -1210,7 +1205,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2UpdateOneRequest); i { case 0: return &v.state @@ -1222,7 +1217,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2UpdateManyRequest); i { case 0: return &v.state @@ -1234,7 +1229,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2ReplaceByIdRequest); i { case 0: return &v.state @@ -1246,7 +1241,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2ReplaceOneRequest); i { case 0: return &v.state @@ -1258,7 +1253,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2InsertOneRequest); i { case 0: return &v.state @@ -1270,7 +1265,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2InsertManyRequest); i { case 0: return &v.state @@ -1282,7 +1277,7 @@ func file_entity_model_service_v2_request_proto_init() { return nil } } - file_entity_model_service_v2_request_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_entity_model_service_v2_request_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*ModelServiceV2CountRequest); i { case 0: return &v.state diff --git a/grpc/node.pb.go b/grpc/node.pb.go index 2e32c4002..4285a4b57 100644 --- a/grpc/node.pb.go +++ b/grpc/node.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: models/node.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Node struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -200,7 +195,7 @@ func file_models_node_proto_rawDescGZIP() []byte { } var file_models_node_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_models_node_proto_goTypes = []interface{}{ +var file_models_node_proto_goTypes = []any{ (*Node)(nil), // 0: grpc.Node } var file_models_node_proto_depIdxs = []int32{ @@ -217,7 +212,7 @@ func file_models_node_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_models_node_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_models_node_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Node); i { case 0: return &v.state diff --git a/grpc/node_info.pb.go b/grpc/node_info.pb.go index 135ddcbfb..d2f36793b 100644 --- a/grpc/node_info.pb.go +++ b/grpc/node_info.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/node_info.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type NodeInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -105,7 +100,7 @@ func file_entity_node_info_proto_rawDescGZIP() []byte { } var file_entity_node_info_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_entity_node_info_proto_goTypes = []interface{}{ +var file_entity_node_info_proto_goTypes = []any{ (*NodeInfo)(nil), // 0: grpc.NodeInfo } var file_entity_node_info_proto_depIdxs = []int32{ @@ -122,7 +117,7 @@ func file_entity_node_info_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_entity_node_info_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_node_info_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*NodeInfo); i { case 0: return &v.state diff --git a/grpc/node_service.pb.go b/grpc/node_service.pb.go index ebed25592..3d344f0a5 100644 --- a/grpc/node_service.pb.go +++ b/grpc/node_service.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/node_service.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_node_service_proto protoreflect.FileDescriptor var file_services_node_service_proto_rawDesc = []byte{ @@ -53,7 +48,7 @@ var file_services_node_service_proto_rawDesc = []byte{ 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -var file_services_node_service_proto_goTypes = []interface{}{ +var file_services_node_service_proto_goTypes = []any{ (*Request)(nil), // 0: grpc.Request (*Response)(nil), // 1: grpc.Response (*StreamMessage)(nil), // 2: grpc.StreamMessage diff --git a/grpc/node_service_grpc.pb.go b/grpc/node_service_grpc.pb.go index 1e0372f10..e097cb97e 100644 --- a/grpc/node_service_grpc.pb.go +++ b/grpc/node_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/node_service.proto package grpc @@ -15,8 +15,16 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + NodeService_Register_FullMethodName = "/grpc.NodeService/Register" + NodeService_SendHeartbeat_FullMethodName = "/grpc.NodeService/SendHeartbeat" + NodeService_Ping_FullMethodName = "/grpc.NodeService/Ping" + NodeService_Subscribe_FullMethodName = "/grpc.NodeService/Subscribe" + NodeService_Unsubscribe_FullMethodName = "/grpc.NodeService/Unsubscribe" +) // NodeServiceClient is the client API for NodeService service. // @@ -38,8 +46,9 @@ func NewNodeServiceClient(cc grpc.ClientConnInterface) NodeServiceClient { } func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/Register", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_Register_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -47,8 +56,9 @@ func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...g } func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/SendHeartbeat", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_SendHeartbeat_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -56,8 +66,9 @@ func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts } func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/Ping", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_Ping_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -65,11 +76,12 @@ func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc. } func (c *nodeServiceClient) Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], "/grpc.NodeService/Subscribe", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], NodeService_Subscribe_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &nodeServiceSubscribeClient{stream} + x := &nodeServiceSubscribeClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -97,8 +109,9 @@ func (x *nodeServiceSubscribeClient) Recv() (*StreamMessage, error) { } func (c *nodeServiceClient) Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/Unsubscribe", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_Unsubscribe_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -159,7 +172,7 @@ func _NodeService_Register_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/Register", + FullMethod: NodeService_Register_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Register(ctx, req.(*Request)) @@ -177,7 +190,7 @@ func _NodeService_SendHeartbeat_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/SendHeartbeat", + FullMethod: NodeService_SendHeartbeat_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).SendHeartbeat(ctx, req.(*Request)) @@ -195,7 +208,7 @@ func _NodeService_Ping_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/Ping", + FullMethod: NodeService_Ping_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Ping(ctx, req.(*Request)) @@ -208,7 +221,7 @@ func _NodeService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) e if err := stream.RecvMsg(m); err != nil { return err } - return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{stream}) + return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{ServerStream: stream}) } type NodeService_SubscribeServer interface { @@ -234,7 +247,7 @@ func _NodeService_Unsubscribe_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/Unsubscribe", + FullMethod: NodeService_Unsubscribe_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Unsubscribe(ctx, req.(*Request)) diff --git a/grpc/plugin_request.pb.go b/grpc/plugin_request.pb.go index 3db72ec14..0e6931f7a 100644 --- a/grpc/plugin_request.pb.go +++ b/grpc/plugin_request.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/plugin_request.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type PluginRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -115,7 +110,7 @@ func file_entity_plugin_request_proto_rawDescGZIP() []byte { } var file_entity_plugin_request_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_entity_plugin_request_proto_goTypes = []interface{}{ +var file_entity_plugin_request_proto_goTypes = []any{ (*PluginRequest)(nil), // 0: grpc.PluginRequest } var file_entity_plugin_request_proto_depIdxs = []int32{ @@ -132,7 +127,7 @@ func file_entity_plugin_request_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_entity_plugin_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_plugin_request_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*PluginRequest); i { case 0: return &v.state diff --git a/grpc/plugin_service.pb.go b/grpc/plugin_service.pb.go index d8974eefc..dbf71d195 100644 --- a/grpc/plugin_service.pb.go +++ b/grpc/plugin_service.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/plugin_service.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_plugin_service_proto protoreflect.FileDescriptor var file_services_plugin_service_proto_rawDesc = []byte{ @@ -49,7 +44,7 @@ var file_services_plugin_service_proto_rawDesc = []byte{ 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -var file_services_plugin_service_proto_goTypes = []interface{}{ +var file_services_plugin_service_proto_goTypes = []any{ (*PluginRequest)(nil), // 0: grpc.PluginRequest (*StreamMessage)(nil), // 1: grpc.StreamMessage (*Response)(nil), // 2: grpc.Response diff --git a/grpc/plugin_service_grpc.pb.go b/grpc/plugin_service_grpc.pb.go index 06b93262c..02acf3394 100644 --- a/grpc/plugin_service_grpc.pb.go +++ b/grpc/plugin_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/plugin_service.proto package grpc @@ -15,8 +15,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + PluginService_Register_FullMethodName = "/grpc.PluginService/Register" + PluginService_Subscribe_FullMethodName = "/grpc.PluginService/Subscribe" + PluginService_Poll_FullMethodName = "/grpc.PluginService/Poll" +) // PluginServiceClient is the client API for PluginService service. // @@ -36,8 +42,9 @@ func NewPluginServiceClient(cc grpc.ClientConnInterface) PluginServiceClient { } func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.PluginService/Register", in, out, opts...) + err := c.cc.Invoke(ctx, PluginService_Register_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -45,11 +52,12 @@ func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, o } func (c *pluginServiceClient) Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], "/grpc.PluginService/Subscribe", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], PluginService_Subscribe_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &pluginServiceSubscribeClient{stream} + x := &pluginServiceSubscribeClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -77,11 +85,12 @@ func (x *pluginServiceSubscribeClient) Recv() (*StreamMessage, error) { } func (c *pluginServiceClient) Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) { - stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], "/grpc.PluginService/Poll", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], PluginService_Poll_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &pluginServicePollClient{stream} + x := &pluginServicePollClient{ClientStream: stream} return x, nil } @@ -153,7 +162,7 @@ func _PluginService_Register_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.PluginService/Register", + FullMethod: PluginService_Register_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(PluginServiceServer).Register(ctx, req.(*PluginRequest)) @@ -166,7 +175,7 @@ func _PluginService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{stream}) + return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{ServerStream: stream}) } type PluginService_SubscribeServer interface { @@ -183,7 +192,7 @@ func (x *pluginServiceSubscribeServer) Send(m *StreamMessage) error { } func _PluginService_Poll_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(PluginServiceServer).Poll(&pluginServicePollServer{stream}) + return srv.(PluginServiceServer).Poll(&pluginServicePollServer{ServerStream: stream}) } type PluginService_PollServer interface { diff --git a/grpc/proto/services/metrics_service_v2.proto b/grpc/proto/services/metrics_service_v2.proto new file mode 100644 index 000000000..4dd0618c9 --- /dev/null +++ b/grpc/proto/services/metrics_service_v2.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Metric { + string namespace = 1; + string subsystem = 2; + string module = 3; + string name = 4; + float value = 5; + string help = 6; + string labels = 7; + MetricProcess process = 8; +} + +message MetricProcess { + int32 pid = 1; +} + +message MetricsServiceV2ConnectRequest { + string node_key = 1; + repeated Metric metrics = 2; +} + +enum MetricsServiceV2Code { + SYNC_METRICS = 0; +} + +message MetricsServiceV2ConnectResponse { + MetricsServiceV2Code code = 1; +} + +service MetricsServiceV2 { + rpc Connect(stream MetricsServiceV2ConnectRequest) returns (stream MetricsServiceV2ConnectResponse){}; +} diff --git a/grpc/request.pb.go b/grpc/request.pb.go index 3b5415549..345df3a03 100644 --- a/grpc/request.pb.go +++ b/grpc/request.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/request.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Request struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -105,7 +100,7 @@ func file_entity_request_proto_rawDescGZIP() []byte { } var file_entity_request_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_entity_request_proto_goTypes = []interface{}{ +var file_entity_request_proto_goTypes = []any{ (*Request)(nil), // 0: grpc.Request } var file_entity_request_proto_depIdxs = []int32{ @@ -122,7 +117,7 @@ func file_entity_request_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_entity_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_request_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Request); i { case 0: return &v.state diff --git a/grpc/response.pb.go b/grpc/response.pb.go index ad32f8690..f3aa76142 100644 --- a/grpc/response.pb.go +++ b/grpc/response.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/response.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Response struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -136,7 +131,7 @@ func file_entity_response_proto_rawDescGZIP() []byte { } var file_entity_response_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_entity_response_proto_goTypes = []interface{}{ +var file_entity_response_proto_goTypes = []any{ (*Response)(nil), // 0: grpc.Response (ResponseCode)(0), // 1: ResponseCode } @@ -156,7 +151,7 @@ func file_entity_response_proto_init() { } file_entity_response_code_proto_init() if !protoimpl.UnsafeEnabled { - file_entity_response_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_response_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Response); i { case 0: return &v.state diff --git a/grpc/response_code.pb.go b/grpc/response_code.pb.go index 9c9700a85..817353ce8 100644 --- a/grpc/response_code.pb.go +++ b/grpc/response_code.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/response_code.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type ResponseCode int32 const ( @@ -95,7 +90,7 @@ func file_entity_response_code_proto_rawDescGZIP() []byte { } var file_entity_response_code_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_entity_response_code_proto_goTypes = []interface{}{ +var file_entity_response_code_proto_goTypes = []any{ (ResponseCode)(0), // 0: ResponseCode } var file_entity_response_code_proto_depIdxs = []int32{ diff --git a/grpc/stream_message.pb.go b/grpc/stream_message.pb.go index 2e563dfc8..6960039a4 100644 --- a/grpc/stream_message.pb.go +++ b/grpc/stream_message.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/stream_message.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type StreamMessage struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -156,7 +151,7 @@ func file_entity_stream_message_proto_rawDescGZIP() []byte { } var file_entity_stream_message_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_entity_stream_message_proto_goTypes = []interface{}{ +var file_entity_stream_message_proto_goTypes = []any{ (*StreamMessage)(nil), // 0: grpc.StreamMessage (StreamMessageCode)(0), // 1: grpc.StreamMessageCode } @@ -176,7 +171,7 @@ func file_entity_stream_message_proto_init() { } file_entity_stream_message_code_proto_init() if !protoimpl.UnsafeEnabled { - file_entity_stream_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_stream_message_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*StreamMessage); i { case 0: return &v.state diff --git a/grpc/stream_message_code.pb.go b/grpc/stream_message_code.pb.go index 90e1f9f6c..d2296acb2 100644 --- a/grpc/stream_message_code.pb.go +++ b/grpc/stream_message_code.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/stream_message_code.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type StreamMessageCode int32 const ( @@ -153,7 +148,7 @@ func file_entity_stream_message_code_proto_rawDescGZIP() []byte { } var file_entity_stream_message_code_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_entity_stream_message_code_proto_goTypes = []interface{}{ +var file_entity_stream_message_code_proto_goTypes = []any{ (StreamMessageCode)(0), // 0: grpc.StreamMessageCode } var file_entity_stream_message_code_proto_depIdxs = []int32{ diff --git a/grpc/stream_message_data_task.pb.go b/grpc/stream_message_data_task.pb.go index e2ffac6e7..dbbd97c67 100644 --- a/grpc/stream_message_data_task.pb.go +++ b/grpc/stream_message_data_task.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: entity/stream_message_data_task.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type StreamMessageDataTask struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -107,7 +102,7 @@ func file_entity_stream_message_data_task_proto_rawDescGZIP() []byte { } var file_entity_stream_message_data_task_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_entity_stream_message_data_task_proto_goTypes = []interface{}{ +var file_entity_stream_message_data_task_proto_goTypes = []any{ (*StreamMessageDataTask)(nil), // 0: grpc.StreamMessageDataTask } var file_entity_stream_message_data_task_proto_depIdxs = []int32{ @@ -124,7 +119,7 @@ func file_entity_stream_message_data_task_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_entity_stream_message_data_task_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_entity_stream_message_data_task_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*StreamMessageDataTask); i { case 0: return &v.state diff --git a/grpc/task.pb.go b/grpc/task.pb.go index af5aa474b..cf12e9e79 100644 --- a/grpc/task.pb.go +++ b/grpc/task.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: models/task.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Task struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -190,7 +185,7 @@ func file_models_task_proto_rawDescGZIP() []byte { } var file_models_task_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_models_task_proto_goTypes = []interface{}{ +var file_models_task_proto_goTypes = []any{ (*Task)(nil), // 0: grpc.Task } var file_models_task_proto_depIdxs = []int32{ @@ -207,7 +202,7 @@ func file_models_task_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_models_task_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_models_task_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Task); i { case 0: return &v.state diff --git a/grpc/task_service.pb.go b/grpc/task_service.pb.go index 7ea57caa5..a8c2258d9 100644 --- a/grpc/task_service.pb.go +++ b/grpc/task_service.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.20.1 +// protoc-gen-go v1.34.2 +// protoc v3.12.4 // source: services/task_service.proto package grpc import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,10 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - var File_services_task_service_proto protoreflect.FileDescriptor var file_services_task_service_proto_rawDesc = []byte{ @@ -48,7 +43,7 @@ var file_services_task_service_proto_rawDesc = []byte{ 0x6f, 0x74, 0x6f, 0x33, } -var file_services_task_service_proto_goTypes = []interface{}{ +var file_services_task_service_proto_goTypes = []any{ (*StreamMessage)(nil), // 0: grpc.StreamMessage (*Request)(nil), // 1: grpc.Request (*Response)(nil), // 2: grpc.Response diff --git a/grpc/task_service_grpc.pb.go b/grpc/task_service_grpc.pb.go index 6afa12265..a4b40dc31 100644 --- a/grpc/task_service_grpc.pb.go +++ b/grpc/task_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.20.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.12.4 // source: services/task_service.proto package grpc @@ -15,8 +15,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + TaskService_Subscribe_FullMethodName = "/grpc.TaskService/Subscribe" + TaskService_Fetch_FullMethodName = "/grpc.TaskService/Fetch" + TaskService_SendNotification_FullMethodName = "/grpc.TaskService/SendNotification" +) // TaskServiceClient is the client API for TaskService service. // @@ -36,11 +42,12 @@ func NewTaskServiceClient(cc grpc.ClientConnInterface) TaskServiceClient { } func (c *taskServiceClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], "/grpc.TaskService/Subscribe", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], TaskService_Subscribe_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &taskServiceSubscribeClient{stream} + x := &taskServiceSubscribeClient{ClientStream: stream} return x, nil } @@ -70,8 +77,9 @@ func (x *taskServiceSubscribeClient) CloseAndRecv() (*Response, error) { } func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.TaskService/Fetch", in, out, opts...) + err := c.cc.Invoke(ctx, TaskService_Fetch_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -79,8 +87,9 @@ func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc } func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.TaskService/SendNotification", in, out, opts...) + err := c.cc.Invoke(ctx, TaskService_SendNotification_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -124,7 +133,7 @@ func RegisterTaskServiceServer(s grpc.ServiceRegistrar, srv TaskServiceServer) { } func _TaskService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{stream}) + return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{ServerStream: stream}) } type TaskService_SubscribeServer interface { @@ -159,7 +168,7 @@ func _TaskService_Fetch_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.TaskService/Fetch", + FullMethod: TaskService_Fetch_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TaskServiceServer).Fetch(ctx, req.(*Request)) @@ -177,7 +186,7 @@ func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.TaskService/SendNotification", + FullMethod: TaskService_SendNotification_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TaskServiceServer).SendNotification(ctx, req.(*Request)) From 265b09862f830b2fb4a6bfdf0296c78649384fdf Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 26 Jun 2024 17:42:49 +0800 Subject: [PATCH 024/106] fix: unable to get permission api issue --- core/controllers/router_v2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index fa8a47a6e..6b8f91aaf 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -76,7 +76,7 @@ func InitRoutes(app *gin.Engine) (err error) { RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.NotificationSettingV2]()) - RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) + //RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]([]Action{ { Method: http.MethodGet, From 7bdce1af58eec5b7a9009584e224ba3a1b0fe5b6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 26 Jun 2024 23:23:14 +0800 Subject: [PATCH 025/106] feat: added metrics service v2 --- core/grpc/client/client_v2.go | 43 ++--- core/grpc/server/metrics_server_v2.go | 78 +++++++- core/grpc/server/server_v2.go | 3 + core/models/client/model_service_v2.go | 7 +- core/node/service/worker_service_v2.go | 5 +- core/task/handler/service_v2.go | 5 +- grpc/dependencies_service_v2.pb.go | 2 +- grpc/dependencies_service_v2_grpc.pb.go | 33 ++-- grpc/message_service.pb.go | 2 +- grpc/message_service_grpc.pb.go | 19 +- grpc/metrics_service_v2.pb.go | 180 ++++++------------- grpc/metrics_service_v2_grpc.pb.go | 19 +- grpc/model_base_service.pb.go | 2 +- grpc/model_base_service_grpc.pb.go | 83 +++------ grpc/model_base_service_v2.pb.go | 2 +- grpc/model_base_service_v2_grpc.pb.go | 95 ++++------ grpc/model_delegate.pb.go | 2 +- grpc/model_delegate_grpc.pb.go | 17 +- grpc/model_service_v2_request.pb.go | 2 +- grpc/node.pb.go | 2 +- grpc/node_info.pb.go | 2 +- grpc/node_service.pb.go | 2 +- grpc/node_service_grpc.pb.go | 43 ++--- grpc/plugin_request.pb.go | 2 +- grpc/plugin_service.pb.go | 2 +- grpc/plugin_service_grpc.pb.go | 33 ++-- grpc/proto/services/metrics_service_v2.proto | 7 +- grpc/request.pb.go | 2 +- grpc/response.pb.go | 2 +- grpc/response_code.pb.go | 2 +- grpc/stream_message.pb.go | 2 +- grpc/stream_message_code.pb.go | 2 +- grpc/stream_message_data_task.pb.go | 2 +- grpc/task.pb.go | 2 +- grpc/task_service.pb.go | 2 +- grpc/task_service_grpc.pb.go | 31 ++-- 36 files changed, 302 insertions(+), 437 deletions(-) diff --git a/core/grpc/client/client_v2.go b/core/grpc/client/client_v2.go index f4ddc2e32..1e49733f5 100644 --- a/core/grpc/client/client_v2.go +++ b/core/grpc/client/client_v2.go @@ -19,6 +19,7 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "io" + "sync" "time" ) @@ -41,10 +42,7 @@ type GrpcClientV2 struct { TaskClient grpc2.TaskServiceClient ModelBaseServiceV2Client grpc2.ModelBaseServiceV2Client DependenciesClient grpc2.DependenciesServiceV2Client -} - -func (c *GrpcClientV2) Init() (err error) { - return nil + MetricsClient grpc2.MetricsServiceV2Client } func (c *GrpcClientV2) Start() (err error) { @@ -96,11 +94,7 @@ func (c *GrpcClientV2) Register() { c.ModelBaseServiceV2Client = grpc2.NewModelBaseServiceV2Client(c.conn) c.TaskClient = grpc2.NewTaskServiceClient(c.conn) c.DependenciesClient = grpc2.NewDependenciesServiceV2Client(c.conn) - - // log - log.Infof("[GrpcClient] grpc client registered client services") - log.Debugf("[GrpcClient] NodeClient: %v", c.NodeClient) - log.Debugf("[GrpcClient] ModelBaseServiceV2Client: %v", c.ModelBaseServiceV2Client) + c.MetricsClient = grpc2.NewMetricsServiceV2Client(c.conn) } func (c *GrpcClientV2) Context() (ctx context.Context, cancel context.CancelFunc) { @@ -248,7 +242,7 @@ func (c *GrpcClientV2) handleStreamMessage() { } } -func NewGrpcClientV2() (c *GrpcClientV2, err error) { +func newGrpcClientV2() (c *GrpcClientV2) { client := &GrpcClientV2{ address: entity.NewAddress(&entity.AddressOptions{ Host: constants.DefaultGrpcClientRemoteHost, @@ -260,28 +254,23 @@ func NewGrpcClientV2() (c *GrpcClientV2, err error) { client.nodeCfgSvc = nodeconfig.GetNodeConfigService() if viper.GetString("grpc.address") != "" { - client.address, err = entity.NewAddressFromString(viper.GetString("grpc.address")) + address, err := entity.NewAddressFromString(viper.GetString("grpc.address")) if err != nil { - return nil, trace.TraceError(err) + log.Errorf("failed to parse grpc address: %s", viper.GetString("grpc.address")) + panic(err) } + client.address = address } - if err := client.Init(); err != nil { - return nil, err - } - - return client, nil + return client } -var _clientV2 *GrpcClientV2 +var clientV2 *GrpcClientV2 +var clientV2Once sync.Once -func GetGrpcClientV2() (client *GrpcClientV2, err error) { - if _clientV2 != nil { - return _clientV2, nil - } - _clientV2, err = NewGrpcClientV2() - if err != nil { - return nil, err - } - return _clientV2, nil +func GetGrpcClientV2() *GrpcClientV2 { + clientV2Once.Do(func() { + clientV2 = newGrpcClientV2() + }) + return clientV2 } diff --git a/core/grpc/server/metrics_server_v2.go b/core/grpc/server/metrics_server_v2.go index 19a0a0d50..c694b2ab9 100644 --- a/core/grpc/server/metrics_server_v2.go +++ b/core/grpc/server/metrics_server_v2.go @@ -1,12 +1,86 @@ package server import ( + "errors" + "github.com/apex/log" "github.com/crawlab-team/crawlab/grpc" + "io" "sync" ) type MetricsServerV2 struct { grpc.UnimplementedMetricsServiceV2Server - mu *sync.Mutex - streams map[string]*grpc.MetricsServiceV2_ConnectServer + mu *sync.Mutex + streams map[string]*grpc.MetricsServiceV2_ConnectServer + channels map[string]chan []*grpc.Metric +} + +func (svr MetricsServerV2) Connect(stream grpc.MetricsServiceV2_ConnectServer) (err error) { + // receive first message + req, err := stream.Recv() + if err != nil { + log.Errorf("[MetricsServerV2] receive error: %v", err) + return err + } + + // save stream and channel + svr.mu.Lock() + svr.streams[req.NodeKey] = &stream + svr.channels[req.NodeKey] = make(chan []*grpc.Metric) + svr.mu.Unlock() + + log.Info("[MetricsServerV2] connected: " + req.NodeKey) + + for { + // receive metrics + req, err = stream.Recv() + if errors.Is(err, io.EOF) { + log.Errorf("[MetricsServerV2] receive EOF: %v", err) + return + } + + // send metrics to channel + svr.channels[req.NodeKey] <- req.Metrics + + // keep this scope alive because once this scope exits - the stream is closed + select { + case <-stream.Context().Done(): + log.Info("[MetricsServerV2] disconnected: " + req.NodeKey) + delete(svr.streams, req.NodeKey) + delete(svr.channels, req.NodeKey) + return nil + } + } +} + +func (svr MetricsServerV2) GetStream(nodeKey string) (stream *grpc.MetricsServiceV2_ConnectServer, ok bool) { + svr.mu.Lock() + defer svr.mu.Unlock() + stream, ok = svr.streams[nodeKey] + return stream, ok +} + +func (svr MetricsServerV2) GetChannel(nodeKey string) (ch chan []*grpc.Metric, ok bool) { + svr.mu.Lock() + defer svr.mu.Unlock() + ch, ok = svr.channels[nodeKey] + return ch, ok +} + +func NewMetricsServerV2() *MetricsServerV2 { + return &MetricsServerV2{ + mu: new(sync.Mutex), + streams: make(map[string]*grpc.MetricsServiceV2_ConnectServer), + channels: make(map[string]chan []*grpc.Metric), + } +} + +var metricsServerV2 *MetricsServerV2 + +func GetMetricsServerV2() *MetricsServerV2 { + if metricsServerV2 != nil { + return metricsServerV2 + } + metricsServerV2 = NewMetricsServerV2() + return metricsServerV2 } diff --git a/core/grpc/server/server_v2.go b/core/grpc/server/server_v2.go index ad04c5a03..e8c8ba069 100644 --- a/core/grpc/server/server_v2.go +++ b/core/grpc/server/server_v2.go @@ -46,6 +46,7 @@ type GrpcServerV2 struct { TaskSvr *TaskServerV2 ModelBaseServiceSvr *ModelBaseServiceServerV2 DependenciesSvr *DependenciesServerV2 + MetricsSvr *MetricsServerV2 } func (svr *GrpcServerV2) GetConfigPath() (path string) { @@ -119,6 +120,7 @@ func (svr *GrpcServerV2) Register() (err error) { grpc2.RegisterModelBaseServiceV2Server(svr.svr, *svr.ModelBaseServiceSvr) grpc2.RegisterTaskServiceServer(svr.svr, *svr.TaskSvr) grpc2.RegisterDependenciesServiceV2Server(svr.svr, *svr.DependenciesSvr) + grpc2.RegisterMetricsServiceV2Server(svr.svr, *svr.MetricsSvr) return nil } @@ -217,6 +219,7 @@ func NewGrpcServerV2() (svr *GrpcServerV2, err error) { return nil, err } svr.DependenciesSvr = GetDependenciesServerV2() + svr.MetricsSvr = GetMetricsServerV2() // recovery options recoveryOpts := []grpc_recovery.Option{ diff --git a/core/models/client/model_service_v2.go b/core/models/client/model_service_v2.go index 049f584e9..d8921ab5e 100644 --- a/core/models/client/model_service_v2.go +++ b/core/models/client/model_service_v2.go @@ -336,12 +336,9 @@ func NewModelServiceV2[T any]() *ModelServiceV2[T] { var instance *ModelServiceV2[T] - c, err := client.GetGrpcClientV2() - if err != nil { - panic(err) - } + c := client.GetGrpcClientV2() if !c.IsStarted() { - err = c.Start() + err := c.Start() if err != nil { panic(err) } diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index 0134c54db..e83405233 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -205,10 +205,7 @@ func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { } // grpc client - svc.client, err = client.NewGrpcClientV2() - if err != nil { - return nil, err - } + svc.client = client.GetGrpcClientV2() // handler service svc.handlerSvc, err = handler.GetTaskHandlerServiceV2() diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index 4635ef48b..50afc7813 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -405,10 +405,7 @@ func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { svc.cfgSvc = nodeconfig.GetNodeConfigService() // grpc client - svc.c, err = grpcclient.NewGrpcClientV2() - if err != nil { - return nil, err - } + svc.c = grpcclient.GetGrpcClientV2() log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) diff --git a/grpc/dependencies_service_v2.pb.go b/grpc/dependencies_service_v2.pb.go index 00e716da2..f703c5e7f 100644 --- a/grpc/dependencies_service_v2.pb.go +++ b/grpc/dependencies_service_v2.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/dependencies_service_v2.proto package grpc diff --git a/grpc/dependencies_service_v2_grpc.pb.go b/grpc/dependencies_service_v2_grpc.pb.go index 7afcf1671..90aa999b3 100644 --- a/grpc/dependencies_service_v2_grpc.pb.go +++ b/grpc/dependencies_service_v2_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/dependencies_service_v2.proto package grpc @@ -15,14 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - DependenciesServiceV2_Connect_FullMethodName = "/grpc.DependenciesServiceV2/Connect" - DependenciesServiceV2_Sync_FullMethodName = "/grpc.DependenciesServiceV2/Sync" - DependenciesServiceV2_UpdateTaskLog_FullMethodName = "/grpc.DependenciesServiceV2/UpdateTaskLog" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // DependenciesServiceV2Client is the client API for DependenciesServiceV2 service. // @@ -42,12 +36,11 @@ func NewDependenciesServiceV2Client(cc grpc.ClientConnInterface) DependenciesSer } func (c *dependenciesServiceV2Client) Connect(ctx context.Context, in *DependenciesServiceV2ConnectRequest, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], DependenciesServiceV2_Connect_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], "/grpc.DependenciesServiceV2/Connect", opts...) if err != nil { return nil, err } - x := &dependenciesServiceV2ConnectClient{ClientStream: stream} + x := &dependenciesServiceV2ConnectClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -75,9 +68,8 @@ func (x *dependenciesServiceV2ConnectClient) Recv() (*DependenciesServiceV2Conne } func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, DependenciesServiceV2_Sync_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.DependenciesServiceV2/Sync", in, out, opts...) if err != nil { return nil, err } @@ -85,12 +77,11 @@ func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *Dependencies } func (c *dependenciesServiceV2Client) UpdateTaskLog(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_UpdateTaskLogClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], DependenciesServiceV2_UpdateTaskLog_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], "/grpc.DependenciesServiceV2/UpdateTaskLog", opts...) if err != nil { return nil, err } - x := &dependenciesServiceV2UpdateTaskLogClient{ClientStream: stream} + x := &dependenciesServiceV2UpdateTaskLogClient{stream} return x, nil } @@ -160,7 +151,7 @@ func _DependenciesServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerS if err := stream.RecvMsg(m); err != nil { return err } - return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{ServerStream: stream}) + return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{stream}) } type DependenciesServiceV2_ConnectServer interface { @@ -186,7 +177,7 @@ func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: DependenciesServiceV2_Sync_FullMethodName, + FullMethod: "/grpc.DependenciesServiceV2/Sync", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(DependenciesServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) @@ -195,7 +186,7 @@ func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, d } func _DependenciesServiceV2_UpdateTaskLog_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{ServerStream: stream}) + return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{stream}) } type DependenciesServiceV2_UpdateTaskLogServer interface { diff --git a/grpc/message_service.pb.go b/grpc/message_service.pb.go index a651f61d4..ef667f2ec 100644 --- a/grpc/message_service.pb.go +++ b/grpc/message_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/message_service.proto package grpc diff --git a/grpc/message_service_grpc.pb.go b/grpc/message_service_grpc.pb.go index cc6dc28f8..ab488b00a 100644 --- a/grpc/message_service_grpc.pb.go +++ b/grpc/message_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/message_service.proto package grpc @@ -15,12 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - MessageService_Connect_FullMethodName = "/grpc.MessageService/Connect" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // MessageServiceClient is the client API for MessageService service. // @@ -38,12 +34,11 @@ func NewMessageServiceClient(cc grpc.ClientConnInterface) MessageServiceClient { } func (c *messageServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], MessageService_Connect_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], "/grpc.MessageService/Connect", opts...) if err != nil { return nil, err } - x := &messageServiceConnectClient{ClientStream: stream} + x := &messageServiceConnectClient{stream} return x, nil } @@ -98,7 +93,7 @@ func RegisterMessageServiceServer(s grpc.ServiceRegistrar, srv MessageServiceSer } func _MessageService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{ServerStream: stream}) + return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{stream}) } type MessageService_ConnectServer interface { diff --git a/grpc/metrics_service_v2.pb.go b/grpc/metrics_service_v2.pb.go index 6ab0e542a..62a37ecf4 100644 --- a/grpc/metrics_service_v2.pb.go +++ b/grpc/metrics_service_v2.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/metrics_service_v2.proto package grpc @@ -68,14 +68,13 @@ type Metric struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` - Subsystem string `protobuf:"bytes,2,opt,name=subsystem,proto3" json:"subsystem,omitempty"` - Module string `protobuf:"bytes,3,opt,name=module,proto3" json:"module,omitempty"` - Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` - Value float32 `protobuf:"fixed32,5,opt,name=value,proto3" json:"value,omitempty"` - Help string `protobuf:"bytes,6,opt,name=help,proto3" json:"help,omitempty"` - Labels string `protobuf:"bytes,7,opt,name=labels,proto3" json:"labels,omitempty"` - Process *MetricProcess `protobuf:"bytes,8,opt,name=process,proto3" json:"process,omitempty"` + Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + Subsystem string `protobuf:"bytes,2,opt,name=subsystem,proto3" json:"subsystem,omitempty"` + Module string `protobuf:"bytes,3,opt,name=module,proto3" json:"module,omitempty"` + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + Value float32 `protobuf:"fixed32,5,opt,name=value,proto3" json:"value,omitempty"` + Help string `protobuf:"bytes,6,opt,name=help,proto3" json:"help,omitempty"` + Labels []byte `protobuf:"bytes,7,opt,name=labels,proto3" json:"labels,omitempty"` } func (x *Metric) Reset() { @@ -152,67 +151,13 @@ func (x *Metric) GetHelp() string { return "" } -func (x *Metric) GetLabels() string { +func (x *Metric) GetLabels() []byte { if x != nil { return x.Labels } - return "" -} - -func (x *Metric) GetProcess() *MetricProcess { - if x != nil { - return x.Process - } return nil } -type MetricProcess struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Pid int32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` -} - -func (x *MetricProcess) Reset() { - *x = MetricProcess{} - if protoimpl.UnsafeEnabled { - mi := &file_services_metrics_service_v2_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MetricProcess) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MetricProcess) ProtoMessage() {} - -func (x *MetricProcess) ProtoReflect() protoreflect.Message { - mi := &file_services_metrics_service_v2_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MetricProcess.ProtoReflect.Descriptor instead. -func (*MetricProcess) Descriptor() ([]byte, []int) { - return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{1} -} - -func (x *MetricProcess) GetPid() int32 { - if x != nil { - return x.Pid - } - return 0 -} - type MetricsServiceV2ConnectRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -225,7 +170,7 @@ type MetricsServiceV2ConnectRequest struct { func (x *MetricsServiceV2ConnectRequest) Reset() { *x = MetricsServiceV2ConnectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_services_metrics_service_v2_proto_msgTypes[2] + mi := &file_services_metrics_service_v2_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -238,7 +183,7 @@ func (x *MetricsServiceV2ConnectRequest) String() string { func (*MetricsServiceV2ConnectRequest) ProtoMessage() {} func (x *MetricsServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { - mi := &file_services_metrics_service_v2_proto_msgTypes[2] + mi := &file_services_metrics_service_v2_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -251,7 +196,7 @@ func (x *MetricsServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricsServiceV2ConnectRequest.ProtoReflect.Descriptor instead. func (*MetricsServiceV2ConnectRequest) Descriptor() ([]byte, []int) { - return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{2} + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{1} } func (x *MetricsServiceV2ConnectRequest) GetNodeKey() string { @@ -279,7 +224,7 @@ type MetricsServiceV2ConnectResponse struct { func (x *MetricsServiceV2ConnectResponse) Reset() { *x = MetricsServiceV2ConnectResponse{} if protoimpl.UnsafeEnabled { - mi := &file_services_metrics_service_v2_proto_msgTypes[3] + mi := &file_services_metrics_service_v2_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -292,7 +237,7 @@ func (x *MetricsServiceV2ConnectResponse) String() string { func (*MetricsServiceV2ConnectResponse) ProtoMessage() {} func (x *MetricsServiceV2ConnectResponse) ProtoReflect() protoreflect.Message { - mi := &file_services_metrics_service_v2_proto_msgTypes[3] + mi := &file_services_metrics_service_v2_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -305,7 +250,7 @@ func (x *MetricsServiceV2ConnectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricsServiceV2ConnectResponse.ProtoReflect.Descriptor instead. func (*MetricsServiceV2ConnectResponse) Descriptor() ([]byte, []int) { - return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{3} + return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{2} } func (x *MetricsServiceV2ConnectResponse) GetCode() MetricsServiceV2Code { @@ -320,7 +265,7 @@ var File_services_metrics_service_v2_proto protoreflect.FileDescriptor var file_services_metrics_service_v2_proto_rawDesc = []byte{ 0x0a, 0x21, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0xe1, 0x01, 0x0a, 0x06, 0x4d, 0x65, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0xb2, 0x01, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, @@ -331,35 +276,30 @@ var file_services_metrics_service_v2_proto_rawDesc = []byte{ 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x2d, - 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x50, 0x72, 0x6f, - 0x63, 0x65, 0x73, 0x73, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x22, 0x21, 0x0a, - 0x0d, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x10, - 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, 0x64, - 0x22, 0x63, 0x0a, 0x1e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, - 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, - 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0x51, 0x0a, 0x1f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, - 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x2a, 0x28, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x72, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x22, 0x63, + 0x0a, 0x1e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x07, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x22, 0x51, 0x0a, 0x1f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, 0x65, - 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x53, - 0x10, 0x00, 0x32, 0x70, 0x0a, 0x10, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x12, 0x5c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x2a, 0x28, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x10, + 0x0a, 0x0c, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x53, 0x10, 0x00, + 0x32, 0x70, 0x0a, 0x10, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x56, 0x32, 0x12, 0x5c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, + 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, + 0x30, 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -375,25 +315,23 @@ func file_services_metrics_service_v2_proto_rawDescGZIP() []byte { } var file_services_metrics_service_v2_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_services_metrics_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_services_metrics_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_services_metrics_service_v2_proto_goTypes = []any{ (MetricsServiceV2Code)(0), // 0: grpc.MetricsServiceV2Code (*Metric)(nil), // 1: grpc.Metric - (*MetricProcess)(nil), // 2: grpc.MetricProcess - (*MetricsServiceV2ConnectRequest)(nil), // 3: grpc.MetricsServiceV2ConnectRequest - (*MetricsServiceV2ConnectResponse)(nil), // 4: grpc.MetricsServiceV2ConnectResponse + (*MetricsServiceV2ConnectRequest)(nil), // 2: grpc.MetricsServiceV2ConnectRequest + (*MetricsServiceV2ConnectResponse)(nil), // 3: grpc.MetricsServiceV2ConnectResponse } var file_services_metrics_service_v2_proto_depIdxs = []int32{ - 2, // 0: grpc.Metric.process:type_name -> grpc.MetricProcess - 1, // 1: grpc.MetricsServiceV2ConnectRequest.metrics:type_name -> grpc.Metric - 0, // 2: grpc.MetricsServiceV2ConnectResponse.code:type_name -> grpc.MetricsServiceV2Code - 3, // 3: grpc.MetricsServiceV2.Connect:input_type -> grpc.MetricsServiceV2ConnectRequest - 4, // 4: grpc.MetricsServiceV2.Connect:output_type -> grpc.MetricsServiceV2ConnectResponse - 4, // [4:5] is the sub-list for method output_type - 3, // [3:4] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 1, // 0: grpc.MetricsServiceV2ConnectRequest.metrics:type_name -> grpc.Metric + 0, // 1: grpc.MetricsServiceV2ConnectResponse.code:type_name -> grpc.MetricsServiceV2Code + 2, // 2: grpc.MetricsServiceV2.Connect:input_type -> grpc.MetricsServiceV2ConnectRequest + 3, // 3: grpc.MetricsServiceV2.Connect:output_type -> grpc.MetricsServiceV2ConnectResponse + 3, // [3:4] is the sub-list for method output_type + 2, // [2:3] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_services_metrics_service_v2_proto_init() } @@ -415,18 +353,6 @@ func file_services_metrics_service_v2_proto_init() { } } file_services_metrics_service_v2_proto_msgTypes[1].Exporter = func(v any, i int) any { - switch v := v.(*MetricProcess); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_services_metrics_service_v2_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*MetricsServiceV2ConnectRequest); i { case 0: return &v.state @@ -438,7 +364,7 @@ func file_services_metrics_service_v2_proto_init() { return nil } } - file_services_metrics_service_v2_proto_msgTypes[3].Exporter = func(v any, i int) any { + file_services_metrics_service_v2_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*MetricsServiceV2ConnectResponse); i { case 0: return &v.state @@ -457,7 +383,7 @@ func file_services_metrics_service_v2_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_services_metrics_service_v2_proto_rawDesc, NumEnums: 1, - NumMessages: 4, + NumMessages: 3, NumExtensions: 0, NumServices: 1, }, diff --git a/grpc/metrics_service_v2_grpc.pb.go b/grpc/metrics_service_v2_grpc.pb.go index e381d8dba..23fe5c83b 100644 --- a/grpc/metrics_service_v2_grpc.pb.go +++ b/grpc/metrics_service_v2_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/metrics_service_v2.proto package grpc @@ -15,12 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - MetricsServiceV2_Connect_FullMethodName = "/grpc.MetricsServiceV2/Connect" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // MetricsServiceV2Client is the client API for MetricsServiceV2 service. // @@ -38,12 +34,11 @@ func NewMetricsServiceV2Client(cc grpc.ClientConnInterface) MetricsServiceV2Clie } func (c *metricsServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (MetricsServiceV2_ConnectClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &MetricsServiceV2_ServiceDesc.Streams[0], MetricsServiceV2_Connect_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &MetricsServiceV2_ServiceDesc.Streams[0], "/grpc.MetricsServiceV2/Connect", opts...) if err != nil { return nil, err } - x := &metricsServiceV2ConnectClient{ClientStream: stream} + x := &metricsServiceV2ConnectClient{stream} return x, nil } @@ -98,7 +93,7 @@ func RegisterMetricsServiceV2Server(s grpc.ServiceRegistrar, srv MetricsServiceV } func _MetricsServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(MetricsServiceV2Server).Connect(&metricsServiceV2ConnectServer{ServerStream: stream}) + return srv.(MetricsServiceV2Server).Connect(&metricsServiceV2ConnectServer{stream}) } type MetricsServiceV2_ConnectServer interface { diff --git a/grpc/model_base_service.pb.go b/grpc/model_base_service.pb.go index f3994f620..6d6bde709 100644 --- a/grpc/model_base_service.pb.go +++ b/grpc/model_base_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/model_base_service.proto package grpc diff --git a/grpc/model_base_service_grpc.pb.go b/grpc/model_base_service_grpc.pb.go index f512986d0..efca79065 100644 --- a/grpc/model_base_service_grpc.pb.go +++ b/grpc/model_base_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/model_base_service.proto package grpc @@ -15,23 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - ModelBaseService_GetById_FullMethodName = "/grpc.ModelBaseService/GetById" - ModelBaseService_Get_FullMethodName = "/grpc.ModelBaseService/Get" - ModelBaseService_GetList_FullMethodName = "/grpc.ModelBaseService/GetList" - ModelBaseService_DeleteById_FullMethodName = "/grpc.ModelBaseService/DeleteById" - ModelBaseService_Delete_FullMethodName = "/grpc.ModelBaseService/Delete" - ModelBaseService_DeleteList_FullMethodName = "/grpc.ModelBaseService/DeleteList" - ModelBaseService_ForceDeleteList_FullMethodName = "/grpc.ModelBaseService/ForceDeleteList" - ModelBaseService_UpdateById_FullMethodName = "/grpc.ModelBaseService/UpdateById" - ModelBaseService_Update_FullMethodName = "/grpc.ModelBaseService/Update" - ModelBaseService_UpdateDoc_FullMethodName = "/grpc.ModelBaseService/UpdateDoc" - ModelBaseService_Insert_FullMethodName = "/grpc.ModelBaseService/Insert" - ModelBaseService_Count_FullMethodName = "/grpc.ModelBaseService/Count" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // ModelBaseServiceClient is the client API for ModelBaseService service. // @@ -60,9 +45,8 @@ func NewModelBaseServiceClient(cc grpc.ClientConnInterface) ModelBaseServiceClie } func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_GetById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetById", in, out, opts...) if err != nil { return nil, err } @@ -70,9 +54,8 @@ func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts } func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_Get_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Get", in, out, opts...) if err != nil { return nil, err } @@ -80,9 +63,8 @@ func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...g } func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_GetList_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetList", in, out, opts...) if err != nil { return nil, err } @@ -90,9 +72,8 @@ func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts } func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_DeleteById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteById", in, out, opts...) if err != nil { return nil, err } @@ -100,9 +81,8 @@ func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_Delete_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Delete", in, out, opts...) if err != nil { return nil, err } @@ -110,9 +90,8 @@ func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_DeleteList_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteList", in, out, opts...) if err != nil { return nil, err } @@ -120,9 +99,8 @@ func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_ForceDeleteList_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/ForceDeleteList", in, out, opts...) if err != nil { return nil, err } @@ -130,9 +108,8 @@ func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Reques } func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_UpdateById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateById", in, out, opts...) if err != nil { return nil, err } @@ -140,9 +117,8 @@ func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_Update_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Update", in, out, opts...) if err != nil { return nil, err } @@ -150,9 +126,8 @@ func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_UpdateDoc_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateDoc", in, out, opts...) if err != nil { return nil, err } @@ -160,9 +135,8 @@ func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opt } func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_Insert_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Insert", in, out, opts...) if err != nil { return nil, err } @@ -170,9 +144,8 @@ func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseService_Count_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Count", in, out, opts...) if err != nil { return nil, err } @@ -261,7 +234,7 @@ func _ModelBaseService_GetById_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_GetById_FullMethodName, + FullMethod: "/grpc.ModelBaseService/GetById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).GetById(ctx, req.(*Request)) @@ -279,7 +252,7 @@ func _ModelBaseService_Get_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_Get_FullMethodName, + FullMethod: "/grpc.ModelBaseService/Get", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Get(ctx, req.(*Request)) @@ -297,7 +270,7 @@ func _ModelBaseService_GetList_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_GetList_FullMethodName, + FullMethod: "/grpc.ModelBaseService/GetList", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).GetList(ctx, req.(*Request)) @@ -315,7 +288,7 @@ func _ModelBaseService_DeleteById_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_DeleteById_FullMethodName, + FullMethod: "/grpc.ModelBaseService/DeleteById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).DeleteById(ctx, req.(*Request)) @@ -333,7 +306,7 @@ func _ModelBaseService_Delete_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_Delete_FullMethodName, + FullMethod: "/grpc.ModelBaseService/Delete", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Delete(ctx, req.(*Request)) @@ -351,7 +324,7 @@ func _ModelBaseService_DeleteList_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_DeleteList_FullMethodName, + FullMethod: "/grpc.ModelBaseService/DeleteList", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).DeleteList(ctx, req.(*Request)) @@ -369,7 +342,7 @@ func _ModelBaseService_ForceDeleteList_Handler(srv interface{}, ctx context.Cont } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_ForceDeleteList_FullMethodName, + FullMethod: "/grpc.ModelBaseService/ForceDeleteList", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, req.(*Request)) @@ -387,7 +360,7 @@ func _ModelBaseService_UpdateById_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_UpdateById_FullMethodName, + FullMethod: "/grpc.ModelBaseService/UpdateById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).UpdateById(ctx, req.(*Request)) @@ -405,7 +378,7 @@ func _ModelBaseService_Update_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_Update_FullMethodName, + FullMethod: "/grpc.ModelBaseService/Update", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Update(ctx, req.(*Request)) @@ -423,7 +396,7 @@ func _ModelBaseService_UpdateDoc_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_UpdateDoc_FullMethodName, + FullMethod: "/grpc.ModelBaseService/UpdateDoc", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).UpdateDoc(ctx, req.(*Request)) @@ -441,7 +414,7 @@ func _ModelBaseService_Insert_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_Insert_FullMethodName, + FullMethod: "/grpc.ModelBaseService/Insert", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Insert(ctx, req.(*Request)) @@ -459,7 +432,7 @@ func _ModelBaseService_Count_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseService_Count_FullMethodName, + FullMethod: "/grpc.ModelBaseService/Count", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Count(ctx, req.(*Request)) diff --git a/grpc/model_base_service_v2.pb.go b/grpc/model_base_service_v2.pb.go index e29db8937..c6eca7cb8 100644 --- a/grpc/model_base_service_v2.pb.go +++ b/grpc/model_base_service_v2.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/model_base_service_v2.proto package grpc diff --git a/grpc/model_base_service_v2_grpc.pb.go b/grpc/model_base_service_v2_grpc.pb.go index 805475d33..203049d72 100644 --- a/grpc/model_base_service_v2_grpc.pb.go +++ b/grpc/model_base_service_v2_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/model_base_service_v2.proto package grpc @@ -15,25 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - ModelBaseServiceV2_GetById_FullMethodName = "/grpc.ModelBaseServiceV2/GetById" - ModelBaseServiceV2_GetOne_FullMethodName = "/grpc.ModelBaseServiceV2/GetOne" - ModelBaseServiceV2_GetMany_FullMethodName = "/grpc.ModelBaseServiceV2/GetMany" - ModelBaseServiceV2_DeleteById_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteById" - ModelBaseServiceV2_DeleteOne_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteOne" - ModelBaseServiceV2_DeleteMany_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteMany" - ModelBaseServiceV2_UpdateById_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateById" - ModelBaseServiceV2_UpdateOne_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateOne" - ModelBaseServiceV2_UpdateMany_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateMany" - ModelBaseServiceV2_ReplaceById_FullMethodName = "/grpc.ModelBaseServiceV2/ReplaceById" - ModelBaseServiceV2_ReplaceOne_FullMethodName = "/grpc.ModelBaseServiceV2/ReplaceOne" - ModelBaseServiceV2_InsertOne_FullMethodName = "/grpc.ModelBaseServiceV2/InsertOne" - ModelBaseServiceV2_InsertMany_FullMethodName = "/grpc.ModelBaseServiceV2/InsertMany" - ModelBaseServiceV2_Count_FullMethodName = "/grpc.ModelBaseServiceV2/Count" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // ModelBaseServiceV2Client is the client API for ModelBaseServiceV2 service. // @@ -64,9 +47,8 @@ func NewModelBaseServiceV2Client(cc grpc.ClientConnInterface) ModelBaseServiceV2 } func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetById", in, out, opts...) if err != nil { return nil, err } @@ -74,9 +56,8 @@ func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelService } func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetOne_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetOne", in, out, opts...) if err != nil { return nil, err } @@ -84,9 +65,8 @@ func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV } func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetMany_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetMany", in, out, opts...) if err != nil { return nil, err } @@ -94,9 +74,8 @@ func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelService } func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteById", in, out, opts...) if err != nil { return nil, err } @@ -104,9 +83,8 @@ func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteOne_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteOne", in, out, opts...) if err != nil { return nil, err } @@ -114,9 +92,8 @@ func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteMany_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteMany", in, out, opts...) if err != nil { return nil, err } @@ -124,9 +101,8 @@ func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateById", in, out, opts...) if err != nil { return nil, err } @@ -134,9 +110,8 @@ func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateOne_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateOne", in, out, opts...) if err != nil { return nil, err } @@ -144,9 +119,8 @@ func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateMany_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateMany", in, out, opts...) if err != nil { return nil, err } @@ -154,9 +128,8 @@ func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_ReplaceById_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceById", in, out, opts...) if err != nil { return nil, err } @@ -164,9 +137,8 @@ func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelSer } func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_ReplaceOne_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceOne", in, out, opts...) if err != nil { return nil, err } @@ -174,9 +146,8 @@ func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_InsertOne_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertOne", in, out, opts...) if err != nil { return nil, err } @@ -184,9 +155,8 @@ func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_InsertMany_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertMany", in, out, opts...) if err != nil { return nil, err } @@ -194,9 +164,8 @@ func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelBaseServiceV2_Count_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/Count", in, out, opts...) if err != nil { return nil, err } @@ -293,7 +262,7 @@ func _ModelBaseServiceV2_GetById_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_GetById_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/GetById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetById(ctx, req.(*ModelServiceV2GetByIdRequest)) @@ -311,7 +280,7 @@ func _ModelBaseServiceV2_GetOne_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_GetOne_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/GetOne", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetOne(ctx, req.(*ModelServiceV2GetOneRequest)) @@ -329,7 +298,7 @@ func _ModelBaseServiceV2_GetMany_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_GetMany_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/GetMany", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetMany(ctx, req.(*ModelServiceV2GetManyRequest)) @@ -347,7 +316,7 @@ func _ModelBaseServiceV2_DeleteById_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_DeleteById_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteById(ctx, req.(*ModelServiceV2DeleteByIdRequest)) @@ -365,7 +334,7 @@ func _ModelBaseServiceV2_DeleteOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_DeleteOne_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteOne", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, req.(*ModelServiceV2DeleteOneRequest)) @@ -383,7 +352,7 @@ func _ModelBaseServiceV2_DeleteMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_DeleteMany_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteMany", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, req.(*ModelServiceV2DeleteManyRequest)) @@ -401,7 +370,7 @@ func _ModelBaseServiceV2_UpdateById_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_UpdateById_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateById(ctx, req.(*ModelServiceV2UpdateByIdRequest)) @@ -419,7 +388,7 @@ func _ModelBaseServiceV2_UpdateOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_UpdateOne_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateOne", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, req.(*ModelServiceV2UpdateOneRequest)) @@ -437,7 +406,7 @@ func _ModelBaseServiceV2_UpdateMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_UpdateMany_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateMany", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, req.(*ModelServiceV2UpdateManyRequest)) @@ -455,7 +424,7 @@ func _ModelBaseServiceV2_ReplaceById_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_ReplaceById_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/ReplaceById", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, req.(*ModelServiceV2ReplaceByIdRequest)) @@ -473,7 +442,7 @@ func _ModelBaseServiceV2_ReplaceOne_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_ReplaceOne_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/ReplaceOne", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, req.(*ModelServiceV2ReplaceOneRequest)) @@ -491,7 +460,7 @@ func _ModelBaseServiceV2_InsertOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_InsertOne_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/InsertOne", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).InsertOne(ctx, req.(*ModelServiceV2InsertOneRequest)) @@ -509,7 +478,7 @@ func _ModelBaseServiceV2_InsertMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_InsertMany_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/InsertMany", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).InsertMany(ctx, req.(*ModelServiceV2InsertManyRequest)) @@ -527,7 +496,7 @@ func _ModelBaseServiceV2_Count_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelBaseServiceV2_Count_FullMethodName, + FullMethod: "/grpc.ModelBaseServiceV2/Count", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).Count(ctx, req.(*ModelServiceV2CountRequest)) diff --git a/grpc/model_delegate.pb.go b/grpc/model_delegate.pb.go index 158ed7759..c0c4c9f4c 100644 --- a/grpc/model_delegate.pb.go +++ b/grpc/model_delegate.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/model_delegate.proto package grpc diff --git a/grpc/model_delegate_grpc.pb.go b/grpc/model_delegate_grpc.pb.go index a0614e45b..c8da4cc34 100644 --- a/grpc/model_delegate_grpc.pb.go +++ b/grpc/model_delegate_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/model_delegate.proto package grpc @@ -15,12 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - ModelDelegate_Do_FullMethodName = "/grpc.ModelDelegate/Do" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // ModelDelegateClient is the client API for ModelDelegate service. // @@ -38,9 +34,8 @@ func NewModelDelegateClient(cc grpc.ClientConnInterface) ModelDelegateClient { } func (c *modelDelegateClient) Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, ModelDelegate_Do_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.ModelDelegate/Do", in, out, opts...) if err != nil { return nil, err } @@ -85,7 +80,7 @@ func _ModelDelegate_Do_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ModelDelegate_Do_FullMethodName, + FullMethod: "/grpc.ModelDelegate/Do", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelDelegateServer).Do(ctx, req.(*Request)) diff --git a/grpc/model_service_v2_request.pb.go b/grpc/model_service_v2_request.pb.go index e56d8d8d8..c3a51f20b 100644 --- a/grpc/model_service_v2_request.pb.go +++ b/grpc/model_service_v2_request.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/model_service_v2_request.proto package grpc diff --git a/grpc/node.pb.go b/grpc/node.pb.go index 4285a4b57..6db136bc2 100644 --- a/grpc/node.pb.go +++ b/grpc/node.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: models/node.proto package grpc diff --git a/grpc/node_info.pb.go b/grpc/node_info.pb.go index d2f36793b..00ccac73f 100644 --- a/grpc/node_info.pb.go +++ b/grpc/node_info.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/node_info.proto package grpc diff --git a/grpc/node_service.pb.go b/grpc/node_service.pb.go index 3d344f0a5..b5efe6c38 100644 --- a/grpc/node_service.pb.go +++ b/grpc/node_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/node_service.proto package grpc diff --git a/grpc/node_service_grpc.pb.go b/grpc/node_service_grpc.pb.go index e097cb97e..dbff3b281 100644 --- a/grpc/node_service_grpc.pb.go +++ b/grpc/node_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/node_service.proto package grpc @@ -15,16 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - NodeService_Register_FullMethodName = "/grpc.NodeService/Register" - NodeService_SendHeartbeat_FullMethodName = "/grpc.NodeService/SendHeartbeat" - NodeService_Ping_FullMethodName = "/grpc.NodeService/Ping" - NodeService_Subscribe_FullMethodName = "/grpc.NodeService/Subscribe" - NodeService_Unsubscribe_FullMethodName = "/grpc.NodeService/Unsubscribe" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // NodeServiceClient is the client API for NodeService service. // @@ -46,9 +38,8 @@ func NewNodeServiceClient(cc grpc.ClientConnInterface) NodeServiceClient { } func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, NodeService_Register_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Register", in, out, opts...) if err != nil { return nil, err } @@ -56,9 +47,8 @@ func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...g } func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, NodeService_SendHeartbeat_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.NodeService/SendHeartbeat", in, out, opts...) if err != nil { return nil, err } @@ -66,9 +56,8 @@ func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts } func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, NodeService_Ping_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Ping", in, out, opts...) if err != nil { return nil, err } @@ -76,12 +65,11 @@ func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc. } func (c *nodeServiceClient) Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], NodeService_Subscribe_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], "/grpc.NodeService/Subscribe", opts...) if err != nil { return nil, err } - x := &nodeServiceSubscribeClient{ClientStream: stream} + x := &nodeServiceSubscribeClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -109,9 +97,8 @@ func (x *nodeServiceSubscribeClient) Recv() (*StreamMessage, error) { } func (c *nodeServiceClient) Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, NodeService_Unsubscribe_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Unsubscribe", in, out, opts...) if err != nil { return nil, err } @@ -172,7 +159,7 @@ func _NodeService_Register_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: NodeService_Register_FullMethodName, + FullMethod: "/grpc.NodeService/Register", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Register(ctx, req.(*Request)) @@ -190,7 +177,7 @@ func _NodeService_SendHeartbeat_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: NodeService_SendHeartbeat_FullMethodName, + FullMethod: "/grpc.NodeService/SendHeartbeat", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).SendHeartbeat(ctx, req.(*Request)) @@ -208,7 +195,7 @@ func _NodeService_Ping_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: NodeService_Ping_FullMethodName, + FullMethod: "/grpc.NodeService/Ping", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Ping(ctx, req.(*Request)) @@ -221,7 +208,7 @@ func _NodeService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) e if err := stream.RecvMsg(m); err != nil { return err } - return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{ServerStream: stream}) + return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{stream}) } type NodeService_SubscribeServer interface { @@ -247,7 +234,7 @@ func _NodeService_Unsubscribe_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: NodeService_Unsubscribe_FullMethodName, + FullMethod: "/grpc.NodeService/Unsubscribe", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Unsubscribe(ctx, req.(*Request)) diff --git a/grpc/plugin_request.pb.go b/grpc/plugin_request.pb.go index 0e6931f7a..d002b8251 100644 --- a/grpc/plugin_request.pb.go +++ b/grpc/plugin_request.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/plugin_request.proto package grpc diff --git a/grpc/plugin_service.pb.go b/grpc/plugin_service.pb.go index dbf71d195..dbae49233 100644 --- a/grpc/plugin_service.pb.go +++ b/grpc/plugin_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/plugin_service.proto package grpc diff --git a/grpc/plugin_service_grpc.pb.go b/grpc/plugin_service_grpc.pb.go index 02acf3394..5ddd329da 100644 --- a/grpc/plugin_service_grpc.pb.go +++ b/grpc/plugin_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/plugin_service.proto package grpc @@ -15,14 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - PluginService_Register_FullMethodName = "/grpc.PluginService/Register" - PluginService_Subscribe_FullMethodName = "/grpc.PluginService/Subscribe" - PluginService_Poll_FullMethodName = "/grpc.PluginService/Poll" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // PluginServiceClient is the client API for PluginService service. // @@ -42,9 +36,8 @@ func NewPluginServiceClient(cc grpc.ClientConnInterface) PluginServiceClient { } func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, PluginService_Register_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.PluginService/Register", in, out, opts...) if err != nil { return nil, err } @@ -52,12 +45,11 @@ func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, o } func (c *pluginServiceClient) Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], PluginService_Subscribe_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], "/grpc.PluginService/Subscribe", opts...) if err != nil { return nil, err } - x := &pluginServiceSubscribeClient{ClientStream: stream} + x := &pluginServiceSubscribeClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -85,12 +77,11 @@ func (x *pluginServiceSubscribeClient) Recv() (*StreamMessage, error) { } func (c *pluginServiceClient) Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], PluginService_Poll_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], "/grpc.PluginService/Poll", opts...) if err != nil { return nil, err } - x := &pluginServicePollClient{ClientStream: stream} + x := &pluginServicePollClient{stream} return x, nil } @@ -162,7 +153,7 @@ func _PluginService_Register_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: PluginService_Register_FullMethodName, + FullMethod: "/grpc.PluginService/Register", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(PluginServiceServer).Register(ctx, req.(*PluginRequest)) @@ -175,7 +166,7 @@ func _PluginService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{ServerStream: stream}) + return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{stream}) } type PluginService_SubscribeServer interface { @@ -192,7 +183,7 @@ func (x *pluginServiceSubscribeServer) Send(m *StreamMessage) error { } func _PluginService_Poll_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(PluginServiceServer).Poll(&pluginServicePollServer{ServerStream: stream}) + return srv.(PluginServiceServer).Poll(&pluginServicePollServer{stream}) } type PluginService_PollServer interface { diff --git a/grpc/proto/services/metrics_service_v2.proto b/grpc/proto/services/metrics_service_v2.proto index 4dd0618c9..ea63c1c65 100644 --- a/grpc/proto/services/metrics_service_v2.proto +++ b/grpc/proto/services/metrics_service_v2.proto @@ -10,12 +10,7 @@ message Metric { string name = 4; float value = 5; string help = 6; - string labels = 7; - MetricProcess process = 8; -} - -message MetricProcess { - int32 pid = 1; + bytes labels = 7; } message MetricsServiceV2ConnectRequest { diff --git a/grpc/request.pb.go b/grpc/request.pb.go index 345df3a03..1dc5d2622 100644 --- a/grpc/request.pb.go +++ b/grpc/request.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/request.proto package grpc diff --git a/grpc/response.pb.go b/grpc/response.pb.go index f3aa76142..88908a722 100644 --- a/grpc/response.pb.go +++ b/grpc/response.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/response.proto package grpc diff --git a/grpc/response_code.pb.go b/grpc/response_code.pb.go index 817353ce8..60b05167a 100644 --- a/grpc/response_code.pb.go +++ b/grpc/response_code.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/response_code.proto package grpc diff --git a/grpc/stream_message.pb.go b/grpc/stream_message.pb.go index 6960039a4..a1158803b 100644 --- a/grpc/stream_message.pb.go +++ b/grpc/stream_message.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/stream_message.proto package grpc diff --git a/grpc/stream_message_code.pb.go b/grpc/stream_message_code.pb.go index d2296acb2..a961e58ca 100644 --- a/grpc/stream_message_code.pb.go +++ b/grpc/stream_message_code.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/stream_message_code.proto package grpc diff --git a/grpc/stream_message_data_task.pb.go b/grpc/stream_message_data_task.pb.go index dbbd97c67..5533cb28c 100644 --- a/grpc/stream_message_data_task.pb.go +++ b/grpc/stream_message_data_task.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: entity/stream_message_data_task.proto package grpc diff --git a/grpc/task.pb.go b/grpc/task.pb.go index cf12e9e79..d15cfbf6c 100644 --- a/grpc/task.pb.go +++ b/grpc/task.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: models/task.proto package grpc diff --git a/grpc/task_service.pb.go b/grpc/task_service.pb.go index a8c2258d9..5d03cbb33 100644 --- a/grpc/task_service.pb.go +++ b/grpc/task_service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v3.12.4 +// protoc v5.27.2 // source: services/task_service.proto package grpc diff --git a/grpc/task_service_grpc.pb.go b/grpc/task_service_grpc.pb.go index a4b40dc31..d71adef33 100644 --- a/grpc/task_service_grpc.pb.go +++ b/grpc/task_service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v5.27.2 // source: services/task_service.proto package grpc @@ -15,14 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 - -const ( - TaskService_Subscribe_FullMethodName = "/grpc.TaskService/Subscribe" - TaskService_Fetch_FullMethodName = "/grpc.TaskService/Fetch" - TaskService_SendNotification_FullMethodName = "/grpc.TaskService/SendNotification" -) +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 // TaskServiceClient is the client API for TaskService service. // @@ -42,12 +36,11 @@ func NewTaskServiceClient(cc grpc.ClientConnInterface) TaskServiceClient { } func (c *taskServiceClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], TaskService_Subscribe_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], "/grpc.TaskService/Subscribe", opts...) if err != nil { return nil, err } - x := &taskServiceSubscribeClient{ClientStream: stream} + x := &taskServiceSubscribeClient{stream} return x, nil } @@ -77,9 +70,8 @@ func (x *taskServiceSubscribeClient) CloseAndRecv() (*Response, error) { } func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, TaskService_Fetch_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.TaskService/Fetch", in, out, opts...) if err != nil { return nil, err } @@ -87,9 +79,8 @@ func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc } func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, TaskService_SendNotification_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, "/grpc.TaskService/SendNotification", in, out, opts...) if err != nil { return nil, err } @@ -133,7 +124,7 @@ func RegisterTaskServiceServer(s grpc.ServiceRegistrar, srv TaskServiceServer) { } func _TaskService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{ServerStream: stream}) + return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{stream}) } type TaskService_SubscribeServer interface { @@ -168,7 +159,7 @@ func _TaskService_Fetch_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TaskService_Fetch_FullMethodName, + FullMethod: "/grpc.TaskService/Fetch", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TaskServiceServer).Fetch(ctx, req.(*Request)) @@ -186,7 +177,7 @@ func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TaskService_SendNotification_FullMethodName, + FullMethod: "/grpc.TaskService/SendNotification", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TaskServiceServer).SendNotification(ctx, req.(*Request)) From 88362068353f6a43ed30941a573fa05d346f3d98 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 26 Jun 2024 23:47:27 +0800 Subject: [PATCH 026/106] fix: missing notification settings controller --- core/controllers/router_v2.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 6b8f91aaf..02dfb3b50 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -75,8 +75,6 @@ func InitRoutes(app *gin.Engine) (err error) { }...)) RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) - RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.NotificationSettingV2]()) - //RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]([]Action{ { Method: http.MethodGet, @@ -84,7 +82,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetProjectList, }, }...)) - RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]()) RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]([]Action{ { Method: http.MethodPost, From 4830d5d546a746690572c779cf0b1a5587a5ba19 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 27 Jun 2024 23:52:01 +0800 Subject: [PATCH 027/106] fix: duplicated start grpc client --- core/grpc/client/client_v2.go | 31 +++++++++++++++++------------- core/grpc/server/node_server_v2.go | 2 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/core/grpc/client/client_v2.go b/core/grpc/client/client_v2.go index 1e49733f5..a5f32ee0b 100644 --- a/core/grpc/client/client_v2.go +++ b/core/grpc/client/client_v2.go @@ -36,6 +36,7 @@ type GrpcClientV2 struct { stream grpc2.NodeService_SubscribeClient msgCh chan *grpc2.StreamMessage err error + once sync.Once // clients NodeClient grpc2.NodeServiceClient @@ -46,23 +47,27 @@ type GrpcClientV2 struct { } func (c *GrpcClientV2) Start() (err error) { - // connect - if err := c.connect(); err != nil { - return err - } + c.once.Do(func() { + // connect + err = c.connect() + if err != nil { + return + } - // register rpc services - c.Register() + // register rpc services + c.Register() - // subscribe - if err := c.subscribe(); err != nil { - return err - } + // subscribe + err = c.subscribe() + if err != nil { + return + } - // handle stream message - go c.handleStreamMessage() + // handle stream message + go c.handleStreamMessage() + }) - return nil + return err } func (c *GrpcClientV2) Stop() (err error) { diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index 6c69b14bc..051198ff4 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -69,7 +69,7 @@ func (svr NodeServerV2) Register(ctx context.Context, req *grpc.Request) (res *g if err != nil { return HandleError(err) } - log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", nodeKey, node.Id.Hex()) + log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", nodeKey, nodeDb.Id.Hex()) } } else if errors2.Is(err, mongo.ErrNoDocuments) { // register new From 68888107a09b058b8a04ae0ec7d8a01d444ad034 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 27 Jun 2024 23:59:10 +0800 Subject: [PATCH 028/106] chore: updated deps --- bin/gen-ver.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 bin/gen-ver.sh diff --git a/bin/gen-ver.sh b/bin/gen-ver.sh new file mode 100755 index 000000000..ab810167a --- /dev/null +++ b/bin/gen-ver.sh @@ -0,0 +1,4 @@ +#!/bin/bash +COMMIT_HASH=$(git rev-parse HEAD) +TIMESTAMP=$(date +%Y%m%d%H%M%S) +echo "v0.0.0-$TIMESTAMP-$COMMIT_HASH" \ No newline at end of file From 0b3fbcfed8d388e6e298cbc759fbaf0fa1d6b1b1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 28 Jun 2024 18:52:49 +0800 Subject: [PATCH 029/106] feat: added multi-spider in git https://github.com/crawlab-team/crawlab/issues/1485 --- core/controllers/spider_v2.go | 9 +++------ core/go.mod | 2 +- go.work.sum | 31 +++++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 5887b58d6..333c31765 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -560,7 +560,8 @@ func PostSpiderSaveFiles(c *gin.Context) { HandleErrorForbidden(c, err) return } - PostBaseFileSaveFiles(rootPath, c) + targetDirectory := c.PostForm("targetDirectory") + PostBaseFileSaveFiles(filepath.Join(rootPath, targetDirectory), c) } func PostSpiderSaveDir(c *gin.Context) { @@ -723,10 +724,6 @@ func getSpiderFsSvc(s *models.SpiderV2) (svc interfaces.FsServiceV2, err error) return fsSvc, nil } -func GetSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) { - return getSpiderFsSvcById(id) -} - func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) { s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) if err != nil { @@ -803,5 +800,5 @@ func getSpiderRootPath(c *gin.Context) (rootPath string, err error) { return id.Hex(), nil } - return filepath.Join(s.GitId.Hex(), rootPath), nil + return filepath.Join(s.GitId.Hex(), s.GitRootPath), nil } diff --git a/core/go.mod b/core/go.mod index 57637c28e..9658175fa 100644 --- a/core/go.mod +++ b/core/go.mod @@ -25,7 +25,7 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 github.com/gavv/httpexpect/v2 v2.16.0 - github.com/gin-gonic/gin v1.9.1 + github.com/gin-gonic/gin v1.10.0 github.com/go-git/go-git/v5 v5.12.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 diff --git a/go.work.sum b/go.work.sum index 45fa9773c..9590de37b 100644 --- a/go.work.sum +++ b/go.work.sum @@ -538,6 +538,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -549,6 +553,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWs github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= @@ -562,7 +570,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= @@ -583,7 +590,11 @@ github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CA github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= @@ -593,6 +604,8 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -712,16 +725,24 @@ github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+ github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1 h1:0pHpWtx9vcvC0xGZqEQlQdfSQs7WRlAjuPvk3fOZDCo= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lyft/protoc-gen-star v0.5.3 h1:zSGLzsUew8RT+ZKPHc3jnf8XLaVyHzTcAFBzHtCNR20= github.com/lyft/protoc-gen-star/v2 v2.0.3 h1:/3+/2sWyXeMLzKd1bX+ixWKgEMsULrIivpDsuaF441o= github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= @@ -801,6 +822,8 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9 github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -843,7 +866,6 @@ go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFu go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= @@ -861,6 +883,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -924,6 +948,7 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -1051,6 +1076,8 @@ modernc.org/ql v1.4.0 h1:CqLAho+y4N8JwvqT7NJsYsp7YPwiRv6RE2n0n1ksSCU= modernc.org/sortutil v1.1.0 h1:oP3U4uM+NT/qBQcbg/K2iqAX0Nx7B1b6YZtq3Gk/PjM= modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= modernc.org/zappy v1.0.3 h1:Tr+P3kclDSrvC6zYBW2hWmOmu5SjG6PtvCt3RCjRmss= +nullprogram.com/x/optparse v1.0.0 h1:xGFgVi5ZaWOnYdac2foDT3vg0ZZC9ErXFV57mr4OHrI= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= From 8ebfda7377ed73e5819c9e992e22da5712ffbbc7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 29 Jun 2024 15:29:24 +0800 Subject: [PATCH 030/106] feat: added multi-spider git https://github.com/crawlab-team/crawlab/issues/1485 --- core/controllers/spider_v2.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 333c31765..889c6c141 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -108,8 +108,12 @@ func getSpiderListWithStats(c *gin.Context) { // ids var ids []primitive.ObjectID + var gitIds []primitive.ObjectID for _, s := range spiders { ids = append(ids, s.Id) + if !s.GitId.IsZero() { + gitIds = append(gitIds, s.GitId) + } } // total count @@ -182,6 +186,22 @@ func getSpiderListWithStats(c *gin.Context) { } } + // git list + var gits []models.GitV2 + if len(gitIds) > 0 && utils.IsPro() { + gits, err = service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // cache git list to dict + dictGit := map[primitive.ObjectID]models.GitV2{} + for _, g := range gits { + dictGit[g.Id] = g + } + // iterate list again var data []models.SpiderV2 for _, s := range spiders { @@ -197,6 +217,14 @@ func getSpiderListWithStats(c *gin.Context) { } } + // git + if !s.GitId.IsZero() && utils.IsPro() { + g, ok := dictGit[s.GitId] + if ok { + s.Git = &g + } + } + // add to list data = append(data, s) } From 1eb81b3f43cc522013395b33554fdc4c6dcb62ab Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 29 Jun 2024 22:29:34 +0800 Subject: [PATCH 031/106] feat: finalized git progress for clone, pull, push https://github.com/crawlab-team/crawlab/issues/1485 --- core/controllers/spider_v2.go | 11 +++++++ core/controllers/ws_writer.go | 57 +++++++++++++++++++++++++++++++++++ core/go.mod | 23 +++++++------- core/go.sum | 51 +++++++++++++++++-------------- core/models/models/git_v2.go | 1 + 5 files changed, 109 insertions(+), 34 deletions(-) create mode 100644 core/controllers/ws_writer.go diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 889c6c141..7523d5407 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -61,6 +61,17 @@ func GetSpiderById(c *gin.Context) { } } + // git + if utils.IsPro() && !s.GitId.IsZero() { + s.Git, err = service.NewModelServiceV2[models.GitV2]().GetById(s.GitId) + if err != nil { + if !errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorInternalServerError(c, err) + return + } + } + } + HandleSuccessWithData(c, s) } diff --git a/core/controllers/ws_writer.go b/core/controllers/ws_writer.go new file mode 100644 index 000000000..798b162c1 --- /dev/null +++ b/core/controllers/ws_writer.go @@ -0,0 +1,57 @@ +package controllers + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/trace" + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "io" + http2 "net/http" +) + +type WsWriter struct { + io.Writer + io.Closer + conn *websocket.Conn +} + +func (w *WsWriter) Write(data []byte) (n int, err error) { + log.Infof("websocket write: %s", string(data)) + err = w.conn.WriteMessage(websocket.TextMessage, data) + if err != nil { + return 0, err + } + return len(data), nil +} + +func (w *WsWriter) Close() (err error) { + return w.conn.Close() +} + +func (w *WsWriter) CloseWithText(text string) { + _ = w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, text)) +} + +func (w *WsWriter) CloseWithError(err error) { + _ = w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error())) +} + +func NewWsWriter(c *gin.Context) (writer *WsWriter, err error) { + upgrader := websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http2.Request) bool { + return true + }, + } + + conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) + if err != nil { + log.Errorf("websocket open connection error: %v", err) + trace.PrintError(err) + } + + return &WsWriter{ + conn: conn, + }, nil +} diff --git a/core/go.mod b/core/go.mod index 9658175fa..09b0fb105 100644 --- a/core/go.mod +++ b/core/go.mod @@ -29,6 +29,7 @@ require ( github.com/go-git/go-git/v5 v5.12.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.4.2 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-uuid v1.0.3 github.com/imroc/req v0.3.0 @@ -67,16 +68,18 @@ require ( github.com/andybalholm/brotli v1.0.4 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aokoli/goutils v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/crawlab-team/goseaweedfs v0.6.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect @@ -85,18 +88,16 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -116,12 +117,12 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect @@ -153,7 +154,7 @@ require ( github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.34.0 // indirect github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 // indirect @@ -177,7 +178,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/goleak v1.1.11 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/arch v0.3.0 // indirect + golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.17.0 // indirect diff --git a/core/go.sum b/core/go.sum index 15e4f90ca..29b5d26e1 100644 --- a/core/go.sum +++ b/core/go.sum @@ -109,9 +109,10 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -119,9 +120,6 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -131,6 +129,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -149,6 +151,7 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= +github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -192,15 +195,15 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -234,8 +237,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -470,8 +473,9 @@ github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -485,8 +489,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -517,8 +521,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= @@ -685,7 +689,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -709,8 +712,8 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -779,6 +782,7 @@ go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -806,8 +810,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1361,6 +1365,7 @@ modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go index 6dc92f117..730c2c89d 100644 --- a/core/models/models/git_v2.go +++ b/core/models/models/git_v2.go @@ -19,6 +19,7 @@ type GitV2 struct { Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` Refs []vcs.GitRef `json:"refs" bson:"refs"` RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` // settings AutoPull bool `json:"auto_pull" bson:"auto_pull"` From 1ea0a352c4cc0d88abf74feb848b9d87f4f3cb71 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 29 Jun 2024 22:30:56 +0800 Subject: [PATCH 032/106] chore: updated deps --- core/go.mod | 2 +- core/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/go.mod b/core/go.mod index 09b0fb105..f574b5e60 100644 --- a/core/go.mod +++ b/core/go.mod @@ -29,7 +29,7 @@ require ( github.com/go-git/go-git/v5 v5.12.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.4.2 + github.com/gorilla/websocket v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-uuid v1.0.3 github.com/imroc/req v0.3.0 diff --git a/core/go.sum b/core/go.sum index 29b5d26e1..bd3fb4266 100644 --- a/core/go.sum +++ b/core/go.sum @@ -345,6 +345,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= From 68307d250e3245384bf0b79d3f478ff7f2c2d9e4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 30 Jun 2024 16:10:51 +0800 Subject: [PATCH 033/106] feat: added git diff https://github.com/crawlab-team/crawlab/issues/1485 --- core/controllers/base_file_v2.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/controllers/base_file_v2.go b/core/controllers/base_file_v2.go index 38987be79..5ea63d9a6 100644 --- a/core/controllers/base_file_v2.go +++ b/core/controllers/base_file_v2.go @@ -291,6 +291,10 @@ func PostBaseFileExport(rootPath string, c *gin.Context) { c.File(zipFilePath) } +func GetBaseFileFsSvc(rootPath string) (svc interfaces.FsServiceV2, err error) { + return getBaseFileFsSvc(rootPath) +} + func getBaseFileFsSvc(rootPath string) (svc interfaces.FsServiceV2, err error) { workspacePath := viper.GetString("workspace") fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, rootPath)) From 023ba275664a189d59cafec75abee654939379a7 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 1 Jul 2024 15:59:20 +0800 Subject: [PATCH 034/106] fix: unable to sync directories to work nodes --- core/apps/server.go | 149 ---------- core/apps/server_test.go | 29 -- core/apps/server_v2.go | 4 +- core/apps/utils.go | 3 +- .../server/model_base_service_v2_server.go | 1 + core/grpc/server/node_server.go | 5 +- core/grpc/server/node_server_v2.go | 95 +++---- core/grpc/server/server.go | 265 ------------------ core/grpc/server/task_server_v2.go | 2 +- core/grpc/test/auth_token_test.go | 96 ------- core/grpc/test/base.go | 77 ----- core/models/client/model_service_v2.go | 2 +- core/models/service/base_service_v2.go | 4 +- core/node/config/config_service.go | 18 +- core/node/service/master_service_v2.go | 33 ++- core/node/service/worker_service.go | 238 ---------------- core/node/service/worker_service_v2.go | 63 +++-- core/node/test/base.go | 206 -------------- core/node/test/base_test.go | 67 ----- core/notification/service_v2.go | 13 +- core/schedule/service_v2.go | 9 +- core/spider/admin/service_v2.go | 14 +- core/system/service_v2.go | 11 +- core/task/handler/runner_v2.go | 107 ++++--- core/task/handler/service_v2.go | 17 +- core/task/scheduler/service_v2.go | 29 +- core/utils/file.go | 13 +- grpc/bin/compile.sh | 24 +- grpc/bin/compile_all.sh | 37 +++ grpc/dependencies_service_v2_grpc.pb.go | 31 +- grpc/message_service_grpc.pb.go | 17 +- grpc/metrics_service_v2_grpc.pb.go | 17 +- grpc/model_base_service_grpc.pb.go | 81 ++++-- grpc/model_base_service_v2_grpc.pb.go | 93 ++++-- grpc/model_delegate_grpc.pb.go | 15 +- grpc/node_service.pb.go | 246 +++++++++++++--- grpc/node_service_grpc.pb.go | 95 +++---- grpc/plugin_service_grpc.pb.go | 31 +- grpc/proto/services/node_service.proto | 16 +- grpc/task_service_grpc.pb.go | 29 +- 40 files changed, 765 insertions(+), 1537 deletions(-) delete mode 100644 core/apps/server.go delete mode 100644 core/apps/server_test.go delete mode 100644 core/grpc/server/server.go delete mode 100644 core/grpc/test/auth_token_test.go delete mode 100644 core/grpc/test/base.go delete mode 100644 core/node/service/worker_service.go delete mode 100644 core/node/test/base.go delete mode 100644 core/node/test/base_test.go mode change 100755 => 100644 grpc/bin/compile.sh create mode 100644 grpc/bin/compile_all.sh diff --git a/core/apps/server.go b/core/apps/server.go deleted file mode 100644 index bf13256cf..000000000 --- a/core/apps/server.go +++ /dev/null @@ -1,149 +0,0 @@ -package apps - -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/node/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/spf13/viper" - "net/http" - _ "net/http/pprof" -) - -func init() { - injectModules() -} - -type Server struct { - // settings - grpcAddress interfaces.Address - - // dependencies - interfaces.WithConfigPath - nodeSvc interfaces.NodeService - - // modules - api *Api - dck *Docker - - // internals - quit chan int -} - -func (app *Server) SetGrpcAddress(address interfaces.Address) { - app.grpcAddress = address -} - -func (app *Server) GetApi() (api ApiApp) { - return app.api -} - -func (app *Server) GetNodeService() (svc interfaces.NodeService) { - return app.nodeSvc -} - -func (app *Server) Init() { - // log node info - app.logNodeInfo() - - if utils.IsMaster() { - - // initialize controllers - if err := controllers.InitControllers(); err != nil { - panic(err) - } - } - - // pprof - app.initPprof() -} - -func (app *Server) Start() { - if utils.IsMaster() { - // start docker app - if utils.IsDocker() { - go app.dck.Start() - } - - // start api - go app.api.Start() - } - - // start node service - go app.nodeSvc.Start() -} - -func (app *Server) Wait() { - <-app.quit -} - -func (app *Server) Stop() { - app.api.Stop() - app.quit <- 1 -} - -func (app *Server) logNodeInfo() { - log.Infof("current node type: %s", utils.GetNodeType()) - if utils.IsDocker() { - log.Infof("running in docker container") - } -} - -func (app *Server) initPprof() { - if viper.GetBool("pprof") { - go func() { - fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil)) - }() - } -} - -func NewServer() (app NodeApp) { - // server - svr := &Server{ - WithConfigPath: config.NewConfigPathService(), - quit: make(chan int, 1), - } - - // service options - var svcOpts []service.Option - if svr.grpcAddress != nil { - svcOpts = append(svcOpts, service.WithAddress(svr.grpcAddress)) - } - - // master modules - if utils.IsMaster() { - // api - svr.api = GetApi() - - // docker - if utils.IsDocker() { - svr.dck = GetDocker(svr) - } - } - - // node service - var err error - if utils.IsMaster() { - svr.nodeSvc, err = service.NewMasterService(svcOpts...) - } else { - svr.nodeSvc, err = service.NewWorkerService(svcOpts...) - } - if err != nil { - panic(err) - } - - return svr -} - -var server NodeApp - -func GetServer() NodeApp { - if server != nil { - return server - } - server = NewServer() - return server -} diff --git a/core/apps/server_test.go b/core/apps/server_test.go deleted file mode 100644 index a2db6e7f9..000000000 --- a/core/apps/server_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package apps - -import ( - "fmt" - "github.com/imroc/req" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "os" - "testing" - "time" -) - -func init() { - _ = os.Setenv("CRAWLAB_DEMO", "false") -} - -func TestServer_Start(t *testing.T) { - svr := GetServer() - - // start - go Start(svr) - time.Sleep(1 * time.Second) - - res, err := req.Get(fmt.Sprintf("http://localhost:%s/system-info", viper.GetString("server.port"))) - require.Nil(t, err) - resStr, err := res.ToString() - require.Nil(t, err) - require.Contains(t, resStr, "success") -} diff --git a/core/apps/server_v2.go b/core/apps/server_v2.go index fdc3ee0a6..80a67dfad 100644 --- a/core/apps/server_v2.go +++ b/core/apps/server_v2.go @@ -104,9 +104,9 @@ func NewServerV2() (app NodeApp) { // node service var err error if utils.IsMaster() { - svr.nodeSvc, err = service.NewMasterServiceV2() + svr.nodeSvc, err = service.GetMasterServiceV2() } else { - svr.nodeSvc, err = service.NewWorkerServiceV2() + svr.nodeSvc, err = service.GetWorkerServiceV2() } if err != nil { panic(err) diff --git a/core/apps/utils.go b/core/apps/utils.go index cd10edab3..f0d279e3d 100644 --- a/core/apps/utils.go +++ b/core/apps/utils.go @@ -64,7 +64,6 @@ var injectors = []interface{}{ modelsclient.NewEnvironmentServiceDelegate, grpcclient.NewClient, grpcclient.NewPool, - grpcserver.GetServer, grpcserver.NewModelDelegateServer, grpcserver.NewModelBaseServiceServer, grpcserver.NewNodeServer, @@ -75,7 +74,7 @@ var injectors = []interface{}{ schedule.GetScheduleService, admin.GetSpiderAdminService, stats.GetStatsService, - nodeconfig.NewNodeConfigService, + nodeconfig.GetNodeConfigService, taskstats.GetTaskStatsService, color.NewService, scheduler.GetTaskSchedulerService, diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 3050e118d..5761e0605 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -34,6 +34,7 @@ var ( *new(models.ScheduleV2), *new(models.SettingV2), *new(models.SpiderV2), + *new(models.SpiderStatV2), *new(models.TaskQueueItemV2), *new(models.TaskStatV2), *new(models.TaskV2), diff --git a/core/grpc/server/node_server.go b/core/grpc/server/node_server.go index c548ea7f7..ce1c99291 100644 --- a/core/grpc/server/node_server.go +++ b/core/grpc/server/node_server.go @@ -193,10 +193,7 @@ func NewNodeServer() (res *NodeServer, err error) { if err != nil { return nil, err } - svr.cfgSvc, err = nodeconfig.NewNodeConfigService() - if err != nil { - return nil, err - } + svr.cfgSvc = nodeconfig.GetNodeConfigService() return svr, nil } diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index 051198ff4..7df8d9979 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -2,7 +2,6 @@ package server import ( "context" - "encoding/json" "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" @@ -16,6 +15,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "sync" "time" ) @@ -30,78 +30,61 @@ type NodeServerV2 struct { } // Register from handler/worker to master -func (svr NodeServerV2) Register(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { +func (svr NodeServerV2) Register(ctx context.Context, req *grpc.NodeServiceRegisterRequest) (res *grpc.Response, err error) { // unmarshall data - var node models.NodeV2 - if req.Data != nil { - if err := json.Unmarshal(req.Data, &node); err != nil { - return HandleError(err) - } - - if node.IsMaster { - // error: cannot register master node - return HandleError(errors.ErrorGrpcNotAllowed) - } + if req.IsMaster { + // error: cannot register master node + return HandleError(errors.ErrorGrpcNotAllowed) } // node key - var nodeKey string - if req.NodeKey != "" { - nodeKey = req.NodeKey - } else { - nodeKey = node.Key - } - if nodeKey == "" { + if req.Key == "" { return HandleError(errors.ErrorModelMissingRequiredData) } // find in db - nodeDb, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + var node *models.NodeV2 + node, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.Key}, nil) if err == nil { - if node.IsMaster { - // error: cannot register master node - return HandleError(errors.ErrorGrpcNotAllowed) - } else { - // register existing - nodeDb.Status = constants.NodeStatusRegistered - nodeDb.Active = true - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(nodeDb.Id, *nodeDb) - if err != nil { - return HandleError(err) - } - log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", nodeKey, nodeDb.Id.Hex()) - } - } else if errors2.Is(err, mongo.ErrNoDocuments) { - // register new - node.Key = nodeKey + // register existing node.Status = constants.NodeStatusRegistered node.Active = true node.ActiveAt = time.Now() - node.Enabled = true - if node.Name == "" { - node.Name = nodeKey + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + if err != nil { + return HandleError(err) + } + log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", req.Key, node.Id.Hex()) + } else if errors2.Is(err, mongo.ErrNoDocuments) { + // register new + node = &models.NodeV2{ + Key: req.Key, + Status: constants.NodeStatusRegistered, + Active: true, + ActiveAt: time.Now(), + Enabled: true, } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - node.Id, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(node) + node.Id, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(*node) if err != nil { return HandleError(err) } - log.Infof("[NodeServerV2] added worker[%s] in db. id: %s", nodeKey, node.Id.Hex()) + log.Infof("[NodeServerV2] added worker[%s] in db. id: %s", req.Key, node.Id.Hex()) } else { // error return HandleError(err) } - log.Infof("[NodeServerV2] master registered worker[%s]", req.GetNodeKey()) + log.Infof("[NodeServerV2] master registered worker[%s]", req.Key) return HandleSuccessWithData(node) } // SendHeartbeat from worker to master -func (svr NodeServerV2) SendHeartbeat(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { +func (svr NodeServerV2) SendHeartbeat(ctx context.Context, req *grpc.NodeServiceSendHeartbeatRequest) (res *grpc.Response, err error) { // find in db - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.Key}, nil) if err != nil { if errors2.Is(err, mongo.ErrNoDocuments) { return HandleError(errors.ErrorNodeNotExists) @@ -126,11 +109,6 @@ func (svr NodeServerV2) SendHeartbeat(ctx context.Context, req *grpc.Request) (r return HandleSuccessWithData(node) } -// Ping from worker to master -func (svr NodeServerV2) Ping(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return HandleSuccess() -} - func (svr NodeServerV2) Subscribe(request *grpc.Request, stream grpc.NodeService_SubscribeServer) (err error) { log.Infof("[NodeServerV2] master received subscribe request from node[%s]", request.NodeKey) @@ -177,13 +155,22 @@ func (svr NodeServerV2) Unsubscribe(ctx context.Context, req *grpc.Request) (res }, nil } +var nodeSvrV2 *NodeServerV2 +var nodeSvrV2Once = new(sync.Once) + func NewNodeServerV2() (res *NodeServerV2, err error) { - // node server - svr := &NodeServerV2{} - svr.cfgSvc, err = nodeconfig.NewNodeConfigService() + if nodeSvrV2 != nil { + return nodeSvrV2, nil + } + nodeSvrV2Once.Do(func() { + nodeSvrV2 = &NodeServerV2{} + nodeSvrV2.cfgSvc = nodeconfig.GetNodeConfigService() + if err != nil { + log.Errorf("[NodeServerV2] error: %s", err.Error()) + } + }) if err != nil { return nil, err } - - return svr, nil + return nodeSvrV2, nil } diff --git a/core/grpc/server/server.go b/core/grpc/server/server.go deleted file mode 100644 index 5639e6416..000000000 --- a/core/grpc/server/server.go +++ /dev/null @@ -1,265 +0,0 @@ -package server - -import ( - "encoding/json" - "fmt" - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/grpc/middlewares" - "github.com/crawlab-team/crawlab/core/interfaces" - grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" - "github.com/grpc-ecosystem/go-grpc-middleware/recovery" - "github.com/spf13/viper" - "go/types" - "google.golang.org/grpc" - "net" - "sync" -) - -var subs = sync.Map{} - -type Server struct { - // dependencies - nodeCfgSvc interfaces.NodeConfigService - nodeSvr *NodeServer - taskSvr *TaskServer - messageSvr *MessageServer - modelDelegateSvr *ModelDelegateServer - modelBaseServiceSvr *ModelBaseServiceServer - - // settings - cfgPath string - address interfaces.Address - - // internals - svr *grpc.Server - l net.Listener - stopped bool -} - -func (svr *Server) Init() (err error) { - // register - if err := svr.Register(); err != nil { - return err - } - - return nil -} - -func (svr *Server) Start() (err error) { - // grpc server binding address - address := svr.address.String() - - // listener - svr.l, err = net.Listen("tcp", address) - if err != nil { - _ = trace.TraceError(err) - return errors.ErrorGrpcServerFailedToListen - } - log.Infof("grpc server listens to %s", address) - - // start grpc server - go func() { - if err := svr.svr.Serve(svr.l); err != nil { - if err == grpc.ErrServerStopped { - return - } - trace.PrintError(err) - log.Error(errors.ErrorGrpcServerFailedToServe.Error()) - } - }() - - return nil -} - -func (svr *Server) Stop() (err error) { - // skip if listener is nil - if svr.l == nil { - return nil - } - - // graceful stop - log.Infof("grpc server stopping...") - svr.svr.Stop() - - // close listener - log.Infof("grpc server closing listener...") - _ = svr.l.Close() - - // mark as stopped - svr.stopped = true - - // log - log.Infof("grpc server stopped") - - return nil -} - -func (svr *Server) Register() (err error) { - grpc2.RegisterModelDelegateServer(svr.svr, *svr.modelDelegateSvr) // model delegate - grpc2.RegisterModelBaseServiceServer(svr.svr, *svr.modelBaseServiceSvr) // model base service - grpc2.RegisterNodeServiceServer(svr.svr, *svr.nodeSvr) // node service - grpc2.RegisterTaskServiceServer(svr.svr, *svr.taskSvr) // task service - grpc2.RegisterMessageServiceServer(svr.svr, *svr.messageSvr) // message service - - return nil -} - -func (svr *Server) SetAddress(address interfaces.Address) { - svr.address = address -} - -func (svr *Server) GetConfigPath() (path string) { - return svr.cfgPath -} - -func (svr *Server) SetConfigPath(path string) { - svr.cfgPath = path -} - -func (svr *Server) GetSubscribe(key string) (sub interfaces.GrpcSubscribe, err error) { - res, ok := subs.Load(key) - if !ok { - return nil, trace.TraceError(errors.ErrorGrpcStreamNotFound) - } - sub, ok = res.(interfaces.GrpcSubscribe) - if !ok { - return nil, trace.TraceError(errors.ErrorGrpcInvalidType) - } - return sub, nil -} - -func (svr *Server) SetSubscribe(key string, sub interfaces.GrpcSubscribe) { - subs.Store(key, sub) -} - -func (svr *Server) DeleteSubscribe(key string) { - subs.Delete(key) -} - -func (svr *Server) SendStreamMessage(key string, code grpc2.StreamMessageCode) (err error) { - return svr.SendStreamMessageWithData(key, code, nil) -} - -func (svr *Server) SendStreamMessageWithData(key string, code grpc2.StreamMessageCode, d interface{}) (err error) { - var data []byte - switch d.(type) { - case types.Nil: - // do nothing - case []byte: - data = d.([]byte) - default: - var err error - data, err = json.Marshal(d) - if err != nil { - panic(err) - } - } - sub, err := svr.GetSubscribe(key) - if err != nil { - return err - } - msg := &grpc2.StreamMessage{ - Code: code, - Key: svr.nodeCfgSvc.GetNodeKey(), - Data: data, - } - return sub.GetStream().Send(msg) -} - -func (svr *Server) IsStopped() (res bool) { - return svr.stopped -} - -func (svr *Server) recoveryHandlerFunc(p interface{}) (err error) { - err = errors.NewError(errors.ErrorPrefixGrpc, fmt.Sprintf("%v", p)) - trace.PrintError(err) - return err -} - -func NewServer() (svr2 interfaces.GrpcServer, err error) { - // server - svr := &Server{ - cfgPath: config2.GetConfigPath(), - address: entity.NewAddress(&entity.AddressOptions{ - Host: constants.DefaultGrpcServerHost, - Port: constants.DefaultGrpcServerPort, - }), - } - - if viper.GetString("grpc.server.address") != "" { - svr.address, err = entity.NewAddressFromString(viper.GetString("grpc.server.address")) - if err != nil { - return nil, err - } - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - nodeCfgSvc interfaces.NodeConfigService, - modelDelegateSvr *ModelDelegateServer, - modelBaseServiceSvr *ModelBaseServiceServer, - nodeSvr *NodeServer, - taskSvr *TaskServer, - messageSvr *MessageServer, - ) { - // dependencies - svr.nodeCfgSvc = nodeCfgSvc - svr.modelDelegateSvr = modelDelegateSvr - svr.modelBaseServiceSvr = modelBaseServiceSvr - svr.nodeSvr = nodeSvr - svr.taskSvr = taskSvr - svr.messageSvr = messageSvr - - // server - svr.nodeSvr.server = svr - svr.taskSvr.server = svr - svr.messageSvr.server = svr - }); err != nil { - return nil, err - } - - // recovery options - recoveryOpts := []grpc_recovery.Option{ - grpc_recovery.WithRecoveryHandler(svr.recoveryHandlerFunc), - } - - // grpc server - svr.svr = grpc.NewServer( - grpc_middleware.WithUnaryServerChain( - grpc_recovery.UnaryServerInterceptor(recoveryOpts...), - grpc_auth.UnaryServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), - ), - grpc_middleware.WithStreamServerChain( - grpc_recovery.StreamServerInterceptor(recoveryOpts...), - grpc_auth.StreamServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), - ), - ) - - // initialize - if err := svr.Init(); err != nil { - return nil, err - } - - return svr, nil -} - -var _server interfaces.GrpcServer - -func GetServer() (svr interfaces.GrpcServer, err error) { - if _server != nil { - return _server, nil - } - _server, err = NewServer() - if err != nil { - return nil, err - } - return _server, nil -} diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index f1c5585e1..62b6ab328 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -110,7 +110,7 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo } func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { - svc := notification.GetServiceV2() + svc := notification.GetNotificationServiceV2() var t = new(models.TaskV2) if err := json.Unmarshal(request.Data, t); err != nil { return nil, trace.TraceError(err) diff --git a/core/grpc/test/auth_token_test.go b/core/grpc/test/auth_token_test.go deleted file mode 100644 index fa09e976a..000000000 --- a/core/grpc/test/auth_token_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package test - -import ( - "context" - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/node/config" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/stretchr/testify/require" - "io/ioutil" - "os" - "path" - "testing" -) - -func TestAuthToken(t *testing.T) { - var err error - - // auth key - authKey := "test-auth-key" - - // auth key (invalid) - authKeyInvalid := "test-auth-key-invalid" - - // tmp dir - tmpDir := os.TempDir() - - // master config - masterConfigPath := path.Join(tmpDir, "config-master.json") - masterConfig := config.Config{ - Key: "master", - IsMaster: true, - AuthKey: authKey, - } - masterConfigData, err := json.Marshal(&masterConfig) - require.Nil(t, err) - err = ioutil.WriteFile(masterConfigPath, masterConfigData, os.FileMode(0777)) - - // worker config - workerConfigPath := path.Join(tmpDir, "config-worker.json") - workerConfig := config.Config{ - Key: "worker", - IsMaster: false, - AuthKey: authKey, - } - workerConfigData, err := json.Marshal(&workerConfig) - require.Nil(t, err) - err = ioutil.WriteFile(workerConfigPath, workerConfigData, os.FileMode(0777)) - - // worker config (invalid) - workerInvalidConfigPath := path.Join(tmpDir, "worker-invalid") - workerInvalidConfig := config.Config{ - Key: "worker", - IsMaster: false, - AuthKey: authKeyInvalid, - } - workerInvalidConfigData, err := json.Marshal(&workerInvalidConfig) - require.Nil(t, err) - err = ioutil.WriteFile(workerInvalidConfigPath, workerInvalidConfigData, os.FileMode(0777)) - - // server - svr, err := server.NewServer( - server.WithConfigPath(masterConfigPath), - server.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: "0.0.0.0", - Port: "9999", - })), - ) - require.Nil(t, err) - err = svr.Start() - require.Nil(t, err) - - // client - c, err := client.GetClient(workerConfigPath, client.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: "localhost", - Port: "9999", - }))) - require.Nil(t, err) - err = c.Start() - require.Nil(t, err) - _, err = c.GetNodeClient().Ping(context.Background(), &grpc.Request{NodeKey: workerConfig.Key}) - require.Nil(t, err) - - // client (invalid) - ci, err := client.GetClient(workerInvalidConfigPath, client.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: "localhost", - Port: "9999", - }))) - require.Nil(t, err) - err = ci.Start() - require.Nil(t, err) - _, err = ci.GetNodeClient().Ping(context.Background(), &grpc.Request{NodeKey: workerInvalidConfig.Key}) - require.NotNil(t, err) -} diff --git a/core/grpc/test/base.go b/core/grpc/test/base.go deleted file mode 100644 index e8e3923ea..000000000 --- a/core/grpc/test/base.go +++ /dev/null @@ -1,77 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/node/test" - "testing" - "time" -) - -type Test struct { - Server interfaces.GrpcServer - Client interfaces.GrpcClient - - MasterNodeInfo *entity.NodeInfo - WorkerNodeInfo *entity.NodeInfo -} - -func (t *Test) Setup(t2 *testing.T) { - test.T.Cleanup() - t2.Cleanup(t.Cleanup) - if !T.Client.IsStarted() { - _ = T.Client.Start() - } else if T.Client.IsClosed() { - _ = T.Client.Restart() - } -} - -func (t *Test) Cleanup() { - _ = t.Client.Stop() - _ = t.Server.Stop() - test.T.Cleanup() - - // wait to avoid caching - time.Sleep(200 * time.Millisecond) -} - -var T *Test - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // server - t.Server, err = server.NewServer( - server.WithConfigPath(test.T.MasterSvc.GetConfigPath()), - server.WithAddress(test.T.MasterSvc.GetAddress()), - ) - if err != nil { - return nil, err - } - if err := t.Server.Start(); err != nil { - return nil, err - } - - // client - t.Client, err = client.GetClient(test.T.WorkerSvc.GetConfigPath()) - if err != nil { - return nil, err - } - - // master node info - t.MasterNodeInfo = &entity.NodeInfo{ - Key: "master", - IsMaster: true, - } - - // worker node info - t.WorkerNodeInfo = &entity.NodeInfo{ - Key: "worker", - IsMaster: false, - } - - return t, nil -} diff --git a/core/models/client/model_service_v2.go b/core/models/client/model_service_v2.go index d8921ab5e..8ad827d8c 100644 --- a/core/models/client/model_service_v2.go +++ b/core/models/client/model_service_v2.go @@ -331,7 +331,7 @@ func NewModelServiceV2[T any]() *ModelServiceV2[T] { typeName := t.Name() if _, exists := onceMap[typeName]; !exists { - onceMap[typeName] = &sync.Once{} + onceMap[typeName] = new(sync.Once) } var instance *ModelServiceV2[T] diff --git a/core/models/service/base_service_v2.go b/core/models/service/base_service_v2.go index 71dc0f15c..9ebe4c91c 100644 --- a/core/models/service/base_service_v2.go +++ b/core/models/service/base_service_v2.go @@ -146,7 +146,7 @@ func NewModelServiceV2[T any]() *ModelServiceV2[T] { defer mu.Unlock() if _, exists := onceMap[typeName]; !exists { - onceMap[typeName] = &sync.Once{} + onceMap[typeName] = new(sync.Once) } var instance *ModelServiceV2[T] @@ -166,7 +166,7 @@ func NewModelServiceV2WithColName[T any](colName string) *ModelServiceV2[T] { defer mu.Unlock() if _, exists := onceColNameMap[colName]; !exists { - onceColNameMap[colName] = &sync.Once{} + onceColNameMap[colName] = new(sync.Once) } var instance *ModelServiceV2[T] diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go index c91c58bf3..9e772c6af 100644 --- a/core/node/config/config_service.go +++ b/core/node/config/config_service.go @@ -9,6 +9,7 @@ import ( "github.com/crawlab-team/crawlab/trace" "os" "path/filepath" + "sync" ) type Service struct { @@ -93,7 +94,7 @@ func (svc *Service) SetConfigPath(path string) { svc.path = path } -func NewNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { +func newNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { // cfg cfg := NewConfig(nil) @@ -115,17 +116,18 @@ func NewNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { } var _service interfaces.NodeConfigService +var _serviceOnce = new(sync.Once) func GetNodeConfigService() interfaces.NodeConfigService { if _service != nil { return _service } - - var err error - _service, err = NewNodeConfigService() - if err != nil { - panic(err) - } - + _serviceOnce.Do(func() { + var err error + _service, err = newNodeConfigService() + if err != nil { + panic(err) + } + }) return _service } diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 6c168c106..cbf2249d6 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -6,7 +6,6 @@ import ( "github.com/cenkalti/backoff/v4" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/common" @@ -20,7 +19,7 @@ import ( "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -320,7 +319,7 @@ func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models.NodeV2) (err return nil } -func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { +func newMasterServiceV2() (res *MasterServiceV2, err error) { // master service svc := &MasterServiceV2{ cfgPath: config2.GetConfigPath(), @@ -334,14 +333,8 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { serverOpts = append(serverOpts, server.WithAddress(svc.address)) } - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - ) { - svc.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } + // node config service + svc.cfgSvc = config.GetNodeConfigService() // grpc server svc.server, err = server.GetGrpcServerV2() @@ -368,7 +361,7 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { } // notification service - svc.notificationSvc = notification.GetServiceV2() + svc.notificationSvc = notification.GetNotificationServiceV2() // spider admin service svc.spiderAdminSvc, err = admin.GetSpiderAdminServiceV2() @@ -377,7 +370,7 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { } // system service - svc.systemSvc = system.GetServiceV2() + svc.systemSvc = system.GetSystemServiceV2() // init if err := svc.Init(); err != nil { @@ -386,3 +379,17 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { return svc, nil } + +var masterServiceV2 *MasterServiceV2 +var masterServiceV2Once = new(sync.Once) + +func GetMasterServiceV2() (res *MasterServiceV2, err error) { + masterServiceV2Once.Do(func() { + masterServiceV2, err = newMasterServiceV2() + if err != nil { + log.Errorf("failed to get master service: %v", err) + } + }) + return masterServiceV2, err + +} diff --git a/core/node/service/worker_service.go b/core/node/service/worker_service.go deleted file mode 100644 index cf91625ab..000000000 --- a/core/node/service/worker_service.go +++ /dev/null @@ -1,238 +0,0 @@ -package service - -import ( - "context" - "encoding/json" - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" - "time" -) - -type WorkerService struct { - // dependencies - cfgSvc interfaces.NodeConfigService - client interfaces.GrpcClient - handlerSvc interfaces.TaskHandlerService - - // settings - cfgPath string - address interfaces.Address - heartbeatInterval time.Duration - - // internals - n interfaces.Node - s grpc.NodeService_SubscribeClient -} - -func (svc *WorkerService) Init() (err error) { - // do nothing - return nil -} - -func (svc *WorkerService) Start() { - // start grpc client - if err := svc.client.Start(); err != nil { - panic(err) - } - - // register to master - svc.Register() - - // start receiving stream messages - go svc.Recv() - - // start sending heartbeat to master - go svc.ReportStatus() - - // start handler - go svc.handlerSvc.Start() - - // wait for quit signal - svc.Wait() - - // stop - svc.Stop() -} - -func (svc *WorkerService) Wait() { - utils.DefaultWait() -} - -func (svc *WorkerService) Stop() { - _ = svc.client.Stop() - log.Infof("worker[%s] service has stopped", svc.cfgSvc.GetNodeKey()) -} - -func (svc *WorkerService) Register() { - ctx, cancel := svc.client.Context() - defer cancel() - req := svc.client.NewRequest(svc.GetConfigService().GetBasicNodeInfo()) - res, err := svc.client.GetNodeClient().Register(ctx, req) - if err != nil { - panic(err) - } - if err := json.Unmarshal(res.Data, svc.n); err != nil { - panic(err) - } - log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.GetId().Hex()) - return -} - -func (svc *WorkerService) Recv() { - msgCh := svc.client.GetMessageChannel() - for { - // return if client is closed - if svc.client.IsClosed() { - return - } - - // receive message from channel - msg := <-msgCh - - // handle message - if err := svc.handleStreamMessage(msg); err != nil { - continue - } - } -} - -func (svc *WorkerService) handleStreamMessage(msg *grpc.StreamMessage) (err error) { - log.Debugf("[WorkerService] handle msg: %v", msg) - switch msg.Code { - case grpc.StreamMessageCode_PING: - if _, err := svc.client.GetNodeClient().SendHeartbeat(context.Background(), svc.client.NewRequest(svc.cfgSvc.GetBasicNodeInfo())); err != nil { - return trace.TraceError(err) - } - case grpc.StreamMessageCode_RUN_TASK: - var t models.Task - if err := json.Unmarshal(msg.Data, &t); err != nil { - return trace.TraceError(err) - } - if err := svc.handlerSvc.Run(t.Id); err != nil { - return trace.TraceError(err) - } - case grpc.StreamMessageCode_CANCEL_TASK: - var t models.Task - if err := json.Unmarshal(msg.Data, &t); err != nil { - return trace.TraceError(err) - } - if err := svc.handlerSvc.Cancel(t.Id); err != nil { - return trace.TraceError(err) - } - } - - return nil -} - -func (svc *WorkerService) ReportStatus() { - for { - // return if client is closed - if svc.client.IsClosed() { - return - } - - // report status - svc.reportStatus() - - // sleep - time.Sleep(svc.heartbeatInterval) - } -} - -func (svc *WorkerService) GetConfigService() (cfgSvc interfaces.NodeConfigService) { - return svc.cfgSvc -} - -func (svc *WorkerService) GetConfigPath() (path string) { - return svc.cfgPath -} - -func (svc *WorkerService) SetConfigPath(path string) { - svc.cfgPath = path -} - -func (svc *WorkerService) GetAddress() (address interfaces.Address) { - return svc.address -} - -func (svc *WorkerService) SetAddress(address interfaces.Address) { - svc.address = address -} - -func (svc *WorkerService) SetHeartbeatInterval(duration time.Duration) { - svc.heartbeatInterval = duration -} - -func (svc *WorkerService) reportStatus() { - ctx, cancel := context.WithTimeout(context.Background(), svc.heartbeatInterval) - defer cancel() - _, err := svc.client.GetNodeClient().SendHeartbeat(ctx, &grpc.Request{ - NodeKey: svc.cfgSvc.GetNodeKey(), - }) - if err != nil { - trace.PrintError(err) - } -} - -func NewWorkerService(opts ...Option) (res *WorkerService, err error) { - svc := &WorkerService{ - cfgPath: config2.GetConfigPath(), - heartbeatInterval: 15 * time.Second, - n: &models.Node{}, - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // dependency options - var clientOpts []client.Option - if svc.address != nil { - clientOpts = append(clientOpts, client.WithAddress(svc.address)) - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - client interfaces.GrpcClient, - taskHandlerSvc interfaces.TaskHandlerService, - ) { - svc.cfgSvc = cfgSvc - svc.client = client - svc.handlerSvc = taskHandlerSvc - }); err != nil { - return nil, err - } - - // init - if err := svc.Init(); err != nil { - return nil, err - } - - return svc, nil -} - -func ProvideWorkerService(path string, opts ...Option) func() (interfaces.NodeWorkerService, error) { - // path - if path == "" || path == config2.GetConfigPath() { - if viper.GetString("config.path") != "" { - path = viper.GetString("config.path") - } else { - path = config2.GetConfigPath() - } - } - opts = append(opts, WithConfigPath(path)) - - return func() (interfaces.NodeWorkerService, error) { - return NewWorkerService(opts...) - } -} diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index e83405233..da4047581 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -5,14 +5,17 @@ import ( "encoding/json" "github.com/apex/log" config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" + client2 "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" + "go.mongodb.org/mongo-driver/bson" + "sync" "time" ) @@ -28,7 +31,7 @@ type WorkerServiceV2 struct { heartbeatInterval time.Duration // internals - n interfaces.Node + n *models.NodeV2 s grpc.NodeService_SubscribeClient } @@ -74,15 +77,21 @@ func (svc *WorkerServiceV2) Stop() { func (svc *WorkerServiceV2) Register() { ctx, cancel := svc.client.Context() defer cancel() - req := svc.client.NewRequest(svc.GetConfigService().GetBasicNodeInfo()) - res, err := svc.client.NodeClient.Register(ctx, req) + _, err := svc.client.NodeClient.Register(ctx, &grpc.NodeServiceRegisterRequest{ + Key: svc.cfgSvc.GetNodeKey(), + Name: svc.cfgSvc.GetNodeName(), + IsMaster: svc.cfgSvc.IsMaster(), + AuthKey: svc.cfgSvc.GetAuthKey(), + MaxRunners: int32(svc.cfgSvc.GetMaxRunners()), + }) if err != nil { panic(err) } - if err := json.Unmarshal(res.Data, svc.n); err != nil { + svc.n, err = client2.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) + if err != nil { panic(err) } - log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.GetId().Hex()) + log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.Id.Hex()) return } @@ -108,7 +117,10 @@ func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err er log.Debugf("[WorkerServiceV2] handle msg: %v", msg) switch msg.Code { case grpc.StreamMessageCode_PING: - if _, err := svc.client.NodeClient.SendHeartbeat(context.Background(), svc.client.NewRequest(svc.cfgSvc.GetBasicNodeInfo())); err != nil { + _, err := svc.client.NodeClient.SendHeartbeat(context.Background(), &grpc.NodeServiceSendHeartbeatRequest{ + Key: svc.cfgSvc.GetNodeKey(), + }) + if err != nil { return trace.TraceError(err) } case grpc.StreamMessageCode_RUN_TASK: @@ -133,9 +145,11 @@ func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err er } func (svc *WorkerServiceV2) ReportStatus() { + ticker := time.NewTicker(svc.heartbeatInterval) for { // return if client is closed if svc.client.IsClosed() { + ticker.Stop() return } @@ -143,7 +157,7 @@ func (svc *WorkerServiceV2) ReportStatus() { svc.reportStatus() // sleep - time.Sleep(svc.heartbeatInterval) + <-ticker.C } } @@ -174,19 +188,21 @@ func (svc *WorkerServiceV2) SetHeartbeatInterval(duration time.Duration) { func (svc *WorkerServiceV2) reportStatus() { ctx, cancel := context.WithTimeout(context.Background(), svc.heartbeatInterval) defer cancel() - _, err := svc.client.NodeClient.SendHeartbeat(ctx, &grpc.Request{ - NodeKey: svc.cfgSvc.GetNodeKey(), + _, err := svc.client.NodeClient.SendHeartbeat(ctx, &grpc.NodeServiceSendHeartbeatRequest{ + Key: svc.cfgSvc.GetNodeKey(), }) if err != nil { trace.PrintError(err) } } -func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { +var workerServiceV2 *WorkerServiceV2 +var workerServiceV2Once = new(sync.Once) + +func newWorkerServiceV2() (res *WorkerServiceV2, err error) { svc := &WorkerServiceV2{ cfgPath: config2.GetConfigPath(), heartbeatInterval: 15 * time.Second, - n: &models.Node{}, } // dependency options @@ -195,14 +211,8 @@ func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { clientOpts = append(clientOpts, client.WithAddress(svc.address)) } - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - ) { - svc.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } + // node config service + svc.cfgSvc = nodeconfig.GetNodeConfigService() // grpc client svc.client = client.GetGrpcClientV2() @@ -214,9 +224,20 @@ func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { } // init - if err := svc.Init(); err != nil { + err = svc.Init() + if err != nil { return nil, err } return svc, nil } + +func GetWorkerServiceV2() (res *WorkerServiceV2, err error) { + workerServiceV2Once.Do(func() { + workerServiceV2, err = newWorkerServiceV2() + if err != nil { + log.Errorf("failed to get worker service: %v", err) + } + }) + return workerServiceV2, err +} diff --git a/core/node/test/base.go b/core/node/test/base.go deleted file mode 100644 index 223c6a926..000000000 --- a/core/node/test/base.go +++ /dev/null @@ -1,206 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - service2 "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/node/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/spf13/viper" - "go.uber.org/dig" - "io/ioutil" - "os" - "path" - "testing" - "time" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -var T *Test - -type Test struct { - DefaultSvc interfaces.NodeMasterService - MasterSvc interfaces.NodeMasterService - WorkerSvc interfaces.NodeWorkerService - MasterSvcMonitor interfaces.NodeMasterService - WorkerSvcMonitor interfaces.NodeWorkerService - ModelSvc service2.ModelService -} - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // recreate config directory path - _ = os.RemoveAll(viper.GetString("metadata")) - _ = os.MkdirAll(viper.GetString("metadata"), os.FileMode(0766)) - - // master config and settings - masterNodeConfigName := "config-master.json" - masterNodeConfigPath := path.Join(viper.GetString("metadata"), masterNodeConfigName) - if err := ioutil.WriteFile(masterNodeConfigPath, []byte("{\"key\":\"master\",\"is_master\":true}"), os.FileMode(0766)); err != nil { - return nil, err - } - masterHost := "0.0.0.0" - masterPort := "9667" - - // worker config and settings - workerNodeConfigName := "config-worker.json" - workerNodeConfigPath := path.Join(viper.GetString("metadata"), workerNodeConfigName) - if err = ioutil.WriteFile(workerNodeConfigPath, []byte("{\"key\":\"worker\",\"is_master\":false}"), os.FileMode(0766)); err != nil { - return nil, err - } - workerHost := "localhost" - workerPort := masterPort - - // master for monitor config and settings - masterNodeMonitorConfigName := "config-master-monitor.json" - masterNodeMonitorConfigPath := path.Join(viper.GetString("metadata"), masterNodeMonitorConfigName) - if err := ioutil.WriteFile(masterNodeMonitorConfigPath, []byte("{\"key\":\"master-monitor\",\"is_master\":true}"), os.FileMode(0766)); err != nil { - return nil, err - } - masterMonitorHost := masterHost - masterMonitorPort := "9668" - - // worker for monitor config and settings - workerNodeMonitorConfigName := "config-worker-monitor.json" - workerNodeMonitorConfigPath := path.Join(viper.GetString("metadata"), workerNodeMonitorConfigName) - if err := ioutil.WriteFile(workerNodeMonitorConfigPath, []byte("{\"key\":\"worker-monitor\",\"is_master\":false}"), os.FileMode(0766)); err != nil { - return nil, err - } - workerMonitorHost := workerHost - workerMonitorPort := masterMonitorPort - - // dependency injection - c := dig.New() - if err := c.Provide(service.ProvideMasterService( - masterNodeConfigPath, - service.WithMonitorInterval(3*time.Second), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: masterHost, - Port: masterPort, - })), - )); err != nil { - return nil, err - } - if err := c.Provide(service.ProvideWorkerService( - workerNodeConfigPath, - service.WithHeartbeatInterval(1*time.Second), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: workerHost, - Port: workerPort, - })), - )); err != nil { - return nil, err - } - if err := c.Provide(service2.NewService); err != nil { - return nil, err - } - if err := c.Invoke(func(masterSvc interfaces.NodeMasterService, workerSvc interfaces.NodeWorkerService, modelSvc service2.ModelService) { - t.MasterSvc = masterSvc - t.WorkerSvc = workerSvc - t.ModelSvc = modelSvc - }); err != nil { - return nil, err - } - - // default service - t.DefaultSvc, err = service.NewMasterService() - if err != nil { - return nil, err - } - - // master and worker for monitor - t.MasterSvcMonitor, err = service.NewMasterService( - service.WithConfigPath(masterNodeMonitorConfigPath), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: masterMonitorHost, - Port: masterMonitorPort, - })), - service.WithMonitorInterval(3*time.Second), - service.WithStopOnError(), - ) - if err != nil { - return nil, err - } - t.WorkerSvcMonitor, err = service.NewWorkerService( - service.WithConfigPath(workerNodeMonitorConfigPath), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: workerMonitorHost, - Port: workerMonitorPort, - })), - service.WithHeartbeatInterval(1*time.Second), - service.WithStopOnError(), - ) - if err != nil { - return nil, err - } - - // removed all data in db - _ = t.ModelSvc.DropAll() - - // visualize dependencies - if err := utils.VisualizeContainer(c); err != nil { - return nil, err - } - - return t, nil -} - -func (t *Test) Setup(t2 *testing.T) { - if err := t.ModelSvc.DropAll(); err != nil { - panic(err) - } - _ = os.RemoveAll(viper.GetString("metadata")) - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - if err := t.ModelSvc.DropAll(); err != nil { - panic(err) - } - _ = os.RemoveAll(viper.GetString("metadata")) -} - -func (t *Test) StartMasterWorker() { - startMasterWorker() -} - -func (t *Test) StopMasterWorker() { - stopMasterWorker() -} - -func startMasterWorker() { - go T.MasterSvc.Start() - time.Sleep(1 * time.Second) - go T.WorkerSvc.Start() - time.Sleep(1 * time.Second) -} - -func stopMasterWorker() { - go T.MasterSvc.Stop() - time.Sleep(1 * time.Second) - go T.WorkerSvc.Stop() - time.Sleep(1 * time.Second) -} - -func startMasterWorkerMonitor() { - go T.MasterSvcMonitor.Start() - time.Sleep(1 * time.Second) - go T.WorkerSvcMonitor.Start() - time.Sleep(1 * time.Second) -} - -func stopMasterWorkerMonitor() { - go T.MasterSvcMonitor.Stop() - time.Sleep(1 * time.Second) - go T.WorkerSvcMonitor.Stop() - time.Sleep(1 * time.Second) -} diff --git a/core/node/test/base_test.go b/core/node/test/base_test.go deleted file mode 100644 index 70832586c..000000000 --- a/core/node/test/base_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func TestNodeServices_Master_Worker(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - startMasterWorker() - - // validate master - masterNodeKey := T.MasterSvc.GetConfigService().GetNodeKey() - masterNode, err := T.ModelSvc.GetNodeByKey(masterNodeKey, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, masterNode.Status) - require.Equal(t, masterNodeKey, masterNode.Key) - require.True(t, masterNode.IsMaster) - - // validate worker - workerNodeKey := T.WorkerSvc.GetConfigService().GetNodeKey() - workerNode, err := T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, workerNode.Status) - require.Equal(t, workerNodeKey, workerNode.Key) - require.False(t, workerNode.IsMaster) - - stopMasterWorker() -} - -func TestNodeServices_Default(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - - go T.DefaultSvc.Start() - time.Sleep(1 * time.Second) - - // validate default - defaultNodeKey := T.DefaultSvc.GetConfigService().GetNodeKey() - defaultNode, err := T.ModelSvc.GetNodeByKey(defaultNodeKey, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, defaultNode.Status) - require.Equal(t, defaultNodeKey, defaultNode.Key) - require.True(t, defaultNode.IsMaster) - - T.DefaultSvc.Stop() - time.Sleep(1 * time.Second) -} - -func TestNodeServices_Monitor(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - startMasterWorkerMonitor() - time.Sleep(3 * time.Second) - - // stop worker - T.WorkerSvcMonitor.Stop() - time.Sleep(5 * time.Second) - - // validate - require.True(t, T.MasterSvcMonitor.GetServer().IsStopped()) - - stopMasterWorkerMonitor() -} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 5a00b7c61..0d8a4c889 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -11,6 +11,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "sync" ) type ServiceV2 struct { @@ -341,7 +342,7 @@ func (svc *ServiceV2) _toggleSettingFunc(value bool) func(id primitive.ObjectID) } } -func NewServiceV2() *ServiceV2 { +func newNotificationServiceV2() *ServiceV2 { // service svc := &ServiceV2{} @@ -349,10 +350,14 @@ func NewServiceV2() *ServiceV2 { } var _serviceV2 *ServiceV2 +var _serviceV2Once = new(sync.Once) -func GetServiceV2() *ServiceV2 { - if _serviceV2 == nil { - _serviceV2 = NewServiceV2() +func GetNotificationServiceV2() *ServiceV2 { + if _serviceV2 != nil { + return _serviceV2 } + _serviceV2Once.Do(func() { + _serviceV2 = newNotificationServiceV2() + }) return _serviceV2 } diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go index e7c4f51d3..1d2d09da1 100644 --- a/core/schedule/service_v2.go +++ b/core/schedule/service_v2.go @@ -1,6 +1,7 @@ package schedule import ( + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" @@ -270,12 +271,18 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { } var svcV2 *ServiceV2 +var svcV2Once = new(sync.Once) func GetScheduleServiceV2() (res *ServiceV2, err error) { if svcV2 != nil { return svcV2, nil } - svcV2, err = NewScheduleServiceV2() + svcV2Once.Do(func() { + svcV2, err = NewScheduleServiceV2() + if err != nil { + log.Errorf("failed to get schedule service: %v", err) + } + }) if err != nil { return nil, err } diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index bd6d1e19f..4ead1ec84 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -1,6 +1,7 @@ package admin import ( + log2 "github.com/apex/log" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" @@ -13,6 +14,7 @@ import ( "github.com/robfig/cron/v3" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" + "sync" "time" ) @@ -160,7 +162,7 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) } } -func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { +func newSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { svc := &ServiceV2{ nodeCfgSvc: config.GetNodeConfigService(), cfgPath: config2.GetConfigPath(), @@ -182,16 +184,20 @@ func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { } var svcV2 *ServiceV2 +var svcV2Once = new(sync.Once) func GetSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { if svcV2 != nil { return svcV2, nil } - - svcV2, err = NewSpiderAdminServiceV2() + svcV2Once.Do(func() { + svcV2, err = newSpiderAdminServiceV2() + if err != nil { + log2.Errorf("[GetSpiderAdminServiceV2] error: %v", err) + } + }) if err != nil { return nil, err } - return svcV2, nil } diff --git a/core/system/service_v2.go b/core/system/service_v2.go index e61513ed7..a893c9d97 100644 --- a/core/system/service_v2.go +++ b/core/system/service_v2.go @@ -4,6 +4,7 @@ import ( "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" "go.mongodb.org/mongo-driver/bson" + "sync" ) type ServiceV2 struct { @@ -46,7 +47,7 @@ func (svc *ServiceV2) initData() (err error) { return nil } -func NewServiceV2() *ServiceV2 { +func newSystemServiceV2() *ServiceV2 { // service svc := &ServiceV2{} @@ -58,10 +59,14 @@ func NewServiceV2() *ServiceV2 { } var _serviceV2 *ServiceV2 +var _serviceV2Once = new(sync.Once) -func GetServiceV2() *ServiceV2 { +func GetSystemServiceV2() *ServiceV2 { if _serviceV2 == nil { - _serviceV2 = NewServiceV2() + _serviceV2 = newSystemServiceV2() } + _serviceV2Once.Do(func() { + _serviceV2 = newSystemServiceV2() + }) return _serviceV2 } diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 341ff408c..46789bbfd 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -4,14 +4,14 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "github.com/apex/log" "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" fs2 "github.com/crawlab-team/crawlab/core/fs" + client2 "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" @@ -21,7 +21,6 @@ import ( grpc "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" - "github.com/sirupsen/logrus" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -54,7 +53,7 @@ type RunnerV2 struct { err error // standard process error envs []models.Env // environment variables cwd string // working directory - c interfaces.GrpcClient // grpc client + c *client2.GrpcClientV2 // grpc client sub grpc.TaskService_SubscribeClient // grpc task service stream client // log internals @@ -181,7 +180,7 @@ func (r *RunnerV2) Cancel() (err error) { // make sure the process does not exist op := func() error { if exists, _ := process.PidExists(int32(r.pid)); exists { - return errors.ErrorTaskProcessStillExists + return errors.New(fmt.Sprintf("task process %d still exists", r.pid)) } return nil } @@ -189,7 +188,8 @@ func (r *RunnerV2) Cancel() (err error) { defer cancel() b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) if err := backoff.Retry(op, b); err != nil { - return trace.TraceError(errors.ErrorTaskUnableToCancel) + log.Errorf("Error canceling task %s: %v", r.tid, err) + return trace.TraceError(err) } return nil @@ -362,7 +362,7 @@ func (r *RunnerV2) syncFiles() (err error) { masterFilesMap[file.Path] = file } - // create worker directory if not exists + // create working directory if not exists if _, err := os.Stat(r.cwd); os.IsNotExist(err) { if err := os.MkdirAll(r.cwd, os.ModePerm); err != nil { log.Errorf("Error creating worker directory: %v", err) @@ -377,10 +377,6 @@ func (r *RunnerV2) syncFiles() (err error) { return trace.TraceError(err) } - // set up wait group and error channel - var wg sync.WaitGroup - errCh := make(chan error, 1) - // delete files that are deleted on master node for path, workerFile := range workerFiles { if _, exists := masterFilesMap[path]; !exists { @@ -392,50 +388,85 @@ func (r *RunnerV2) syncFiles() (err error) { } } + // set up wait group and error channel + var wg sync.WaitGroup + pool := make(chan struct{}, 10) + // download files that are new or modified on master node for path, masterFile := range masterFilesMap { workerFile, exists := workerFiles[path] if !exists || masterFile.Hash != workerFile.Hash { wg.Add(1) - go func(path string, masterFile entity.FsFileInfo) { + + // acquire token + pool <- struct{}{} + + // start goroutine to synchronize file or directory + go func(path string, masterFile *entity.FsFileInfo) { defer wg.Done() - logrus.Infof("File needs to be synchronized: %s", path) - err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(r.cwd, path)) - if err != nil { - logrus.Errorf("Error downloading file: %v", err) - select { - case errCh <- err: - default: + + if masterFile.IsDir { + log.Infof("Directory needs to be synchronized: %s", path) + _err := os.MkdirAll(filepath.Join(r.cwd, path), masterFile.Mode) + if _err != nil { + log.Errorf("Error creating directory: %v", _err) + err = errors.Join(err, _err) + } + } else { + log.Infof("File needs to be synchronized: %s", path) + _err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(r.cwd, path), masterFile) + if _err != nil { + log.Errorf("Error downloading file: %v", _err) + err = errors.Join(err, _err) } } - }(path, masterFile) + + // release token + <-pool + + }(path, &masterFile) } } + // wait for all files and directories to be synchronized wg.Wait() - close(errCh) - if err := <-errCh; err != nil { - return err - } - return nil + return err } -func (r *RunnerV2) downloadFile(url string, filePath string) error { +func (r *RunnerV2) downloadFile(url string, filePath string, fileInfo *entity.FsFileInfo) error { + // get file response resp, err := http.Get(url) if err != nil { + log.Errorf("Error getting file response: %v", err) return err } defer resp.Body.Close() - out, err := os.Create(filePath) + // create directory if not exists + dirPath := filepath.Dir(filePath) + utils.Exists(dirPath) + err = os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + log.Errorf("Error creating directory: %v", err) + return err + } + + // create local file + out, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode) if err != nil { + log.Errorf("Error creating file: %v", err) return err } defer out.Close() + // copy file content to local file _, err = io.Copy(out, resp.Body) - return err + if err != nil { + log.Errorf("Error copying file: %v", err) + return err + } + return nil } // wait for process to finish and send task signal (constants.TaskSignal) @@ -443,7 +474,8 @@ func (r *RunnerV2) downloadFile(url string, filePath string) error { func (r *RunnerV2) wait() { // wait for process to finish if err := r.cmd.Wait(); err != nil { - exitError, ok := err.(*exec.ExitError) + var exitError *exec.ExitError + ok := errors.As(err, &exitError) if !ok { r.ch <- constants.TaskSignalError return @@ -505,7 +537,7 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { } func (r *RunnerV2) initSub() (err error) { - r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) + r.sub, err = r.c.TaskClient.Subscribe(context.Background()) if err != nil { return trace.TraceError(err) } @@ -577,7 +609,7 @@ func (r *RunnerV2) sendNotification() { NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), Data: data, } - _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) + _, err = r.c.TaskClient.SendNotification(context.Background(), req) if err != nil { trace.PrintError(err) return @@ -617,7 +649,8 @@ func (r *RunnerV2) _updateSpiderStat(status string) { }, } default: - trace.PrintError(errors.ErrorTaskInvalidType) + log.Errorf("Invalid task status: %s", status) + trace.PrintError(errors.New("invalid task status")) return } @@ -679,14 +712,8 @@ func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err e // task fs service r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) - // dependency injection - if err := container.GetContainer().Invoke(func( - c interfaces.GrpcClient, - ) { - r.c = c - }); err != nil { - return nil, trace.TraceError(err) - } + // grpc client + r.c = client2.GetGrpcClientV2() // initialize task runner if err := r.Init(); err != nil { diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index 50afc7813..e6edb2b22 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -74,9 +74,10 @@ func (svc *ServiceV2) Cancel(taskId primitive.ObjectID) (err error) { } func (svc *ServiceV2) Fetch() { + ticker := time.NewTicker(svc.fetchInterval) for { // wait - time.Sleep(svc.fetchInterval) + <-ticker.C // current node n, err := svc.GetCurrentNode() @@ -96,6 +97,7 @@ func (svc *ServiceV2) Fetch() { // stop if svc.stopped { + ticker.Stop() return } @@ -115,10 +117,11 @@ func (svc *ServiceV2) Fetch() { if err := svc.run(tid); err != nil { trace.PrintError(err) t, err := svc.GetTaskById(tid) - if err == nil && t.Status != constants.TaskStatusCancelled { + if err != nil && t.Status != constants.TaskStatusCancelled { t.Error = err.Error() t.Status = constants.TaskStatusError t.SetUpdated(t.CreatedBy) + _ = client.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) continue } continue @@ -388,7 +391,7 @@ func (svc *ServiceV2) run(taskId primitive.ObjectID) (err error) { return nil } -func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { +func newTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { // service svc := &ServiceV2{ exitWatchDuration: 60 * time.Second, @@ -413,12 +416,18 @@ func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { } var _serviceV2 *ServiceV2 +var _serviceV2Once = new(sync.Once) func GetTaskHandlerServiceV2() (svr *ServiceV2, err error) { if _serviceV2 != nil { return _serviceV2, nil } - _serviceV2, err = NewTaskHandlerServiceV2() + _serviceV2Once.Do(func() { + _serviceV2, err = newTaskHandlerServiceV2() + if err != nil { + log.Errorf("failed to create task handler service: %v", err) + } + }) if err != nil { return nil, err } diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index fbf8ad406..740bf3e3c 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -2,12 +2,15 @@ package scheduler import ( errors2 "errors" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/models" "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" @@ -20,8 +23,8 @@ import ( type ServiceV2 struct { // dependencies nodeCfgSvc interfaces.NodeConfigService - svr interfaces.GrpcServer - handlerSvc interfaces.TaskHandlerService + svr *server.GrpcServerV2 + handlerSvc *handler.ServiceV2 // settings interval time.Duration @@ -233,17 +236,15 @@ func NewTaskSchedulerServiceV2() (svc2 *ServiceV2, err error) { svc := &ServiceV2{ interval: 5 * time.Second, } - - // dependency injection - if err := container.GetContainer().Invoke(func( - nodeCfgSvc interfaces.NodeConfigService, - svr interfaces.GrpcServer, - handlerSvc interfaces.TaskHandlerService, - ) { - svc.nodeCfgSvc = nodeCfgSvc - svc.svr = svr - svc.handlerSvc = handlerSvc - }); err != nil { + svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() + svc.svr, err = server.GetGrpcServerV2() + if err != nil { + log.Errorf("failed to get grpc server: %v", err) + return nil, err + } + svc.handlerSvc, err = handler.GetTaskHandlerServiceV2() + if err != nil { + log.Errorf("failed to get task handler service: %v", err) return nil, err } diff --git a/core/utils/file.go b/core/utils/file.go index 216a129a9..572730ec0 100644 --- a/core/utils/file.go +++ b/core/utils/file.go @@ -348,13 +348,13 @@ func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) { if err != nil { return err } - if info.IsDir() { - return nil - } - hash, err := GetFileHash(path) - if err != nil { - return err + var hash string + if !info.IsDir() { + hash, err = GetFileHash(path) + if err != nil { + return err + } } relPath, err := filepath.Rel(dir, path) @@ -367,6 +367,7 @@ func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) { Path: relPath, FullPath: path, Extension: filepath.Ext(path), + IsDir: info.IsDir(), FileSize: info.Size(), ModTime: info.ModTime(), Mode: info.Mode(), diff --git a/grpc/bin/compile.sh b/grpc/bin/compile.sh old mode 100755 new mode 100644 index b0d5c65fe..d69da1033 --- a/grpc/bin/compile.sh +++ b/grpc/bin/compile.sh @@ -8,30 +8,8 @@ else fi base_path=$(cd $BASE_DIR/..; pwd) -cd $base_path && \ - rm -rf dist | true - -cd $base_path && \ - mkdir -p dist/python | true && \ - mkdir -p dist/js | true && \ - mkdir -p dist/ts | true && \ - mkdir -p dist/java | true && \ - mkdir -p dist/csharp | true && \ - mkdir -p dist/php | true && \ - mkdir -p dist/ruby | true - cd $base_path && \ protoc -I ./proto \ --go_out=. \ --go-grpc_out=. \ - --python_out=dist/python \ - --js_out=dist/js \ - --java_out=dist/java \ - --csharp_out=dist/csharp \ - ./proto/**/*.proto - -# python -cd $base_path && \ - python3 -m grpc_tools.protoc -I ./proto \ - --grpc_python_out=dist/python \ - ./proto/**/*.proto + ./proto/**/*.proto \ No newline at end of file diff --git a/grpc/bin/compile_all.sh b/grpc/bin/compile_all.sh new file mode 100644 index 000000000..b0d5c65fe --- /dev/null +++ b/grpc/bin/compile_all.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +if [ -L $0 ] +then + BASE_DIR=`dirname $(readlink $0)` +else + BASE_DIR=`dirname $0` +fi +base_path=$(cd $BASE_DIR/..; pwd) + +cd $base_path && \ + rm -rf dist | true + +cd $base_path && \ + mkdir -p dist/python | true && \ + mkdir -p dist/js | true && \ + mkdir -p dist/ts | true && \ + mkdir -p dist/java | true && \ + mkdir -p dist/csharp | true && \ + mkdir -p dist/php | true && \ + mkdir -p dist/ruby | true + +cd $base_path && \ + protoc -I ./proto \ + --go_out=. \ + --go-grpc_out=. \ + --python_out=dist/python \ + --js_out=dist/js \ + --java_out=dist/java \ + --csharp_out=dist/csharp \ + ./proto/**/*.proto + +# python +cd $base_path && \ + python3 -m grpc_tools.protoc -I ./proto \ + --grpc_python_out=dist/python \ + ./proto/**/*.proto diff --git a/grpc/dependencies_service_v2_grpc.pb.go b/grpc/dependencies_service_v2_grpc.pb.go index 90aa999b3..6a3fc1a39 100644 --- a/grpc/dependencies_service_v2_grpc.pb.go +++ b/grpc/dependencies_service_v2_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/dependencies_service_v2.proto @@ -15,8 +15,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + DependenciesServiceV2_Connect_FullMethodName = "/grpc.DependenciesServiceV2/Connect" + DependenciesServiceV2_Sync_FullMethodName = "/grpc.DependenciesServiceV2/Sync" + DependenciesServiceV2_UpdateTaskLog_FullMethodName = "/grpc.DependenciesServiceV2/UpdateTaskLog" +) // DependenciesServiceV2Client is the client API for DependenciesServiceV2 service. // @@ -36,11 +42,12 @@ func NewDependenciesServiceV2Client(cc grpc.ClientConnInterface) DependenciesSer } func (c *dependenciesServiceV2Client) Connect(ctx context.Context, in *DependenciesServiceV2ConnectRequest, opts ...grpc.CallOption) (DependenciesServiceV2_ConnectClient, error) { - stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], "/grpc.DependenciesServiceV2/Connect", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[0], DependenciesServiceV2_Connect_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &dependenciesServiceV2ConnectClient{stream} + x := &dependenciesServiceV2ConnectClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -68,8 +75,9 @@ func (x *dependenciesServiceV2ConnectClient) Recv() (*DependenciesServiceV2Conne } func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.DependenciesServiceV2/Sync", in, out, opts...) + err := c.cc.Invoke(ctx, DependenciesServiceV2_Sync_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -77,11 +85,12 @@ func (c *dependenciesServiceV2Client) Sync(ctx context.Context, in *Dependencies } func (c *dependenciesServiceV2Client) UpdateTaskLog(ctx context.Context, opts ...grpc.CallOption) (DependenciesServiceV2_UpdateTaskLogClient, error) { - stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], "/grpc.DependenciesServiceV2/UpdateTaskLog", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &DependenciesServiceV2_ServiceDesc.Streams[1], DependenciesServiceV2_UpdateTaskLog_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &dependenciesServiceV2UpdateTaskLogClient{stream} + x := &dependenciesServiceV2UpdateTaskLogClient{ClientStream: stream} return x, nil } @@ -151,7 +160,7 @@ func _DependenciesServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerS if err := stream.RecvMsg(m); err != nil { return err } - return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{stream}) + return srv.(DependenciesServiceV2Server).Connect(m, &dependenciesServiceV2ConnectServer{ServerStream: stream}) } type DependenciesServiceV2_ConnectServer interface { @@ -177,7 +186,7 @@ func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.DependenciesServiceV2/Sync", + FullMethod: DependenciesServiceV2_Sync_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(DependenciesServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) @@ -186,7 +195,7 @@ func _DependenciesServiceV2_Sync_Handler(srv interface{}, ctx context.Context, d } func _DependenciesServiceV2_UpdateTaskLog_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{stream}) + return srv.(DependenciesServiceV2Server).UpdateTaskLog(&dependenciesServiceV2UpdateTaskLogServer{ServerStream: stream}) } type DependenciesServiceV2_UpdateTaskLogServer interface { diff --git a/grpc/message_service_grpc.pb.go b/grpc/message_service_grpc.pb.go index ab488b00a..077a582b8 100644 --- a/grpc/message_service_grpc.pb.go +++ b/grpc/message_service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/message_service.proto @@ -15,8 +15,12 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + MessageService_Connect_FullMethodName = "/grpc.MessageService/Connect" +) // MessageServiceClient is the client API for MessageService service. // @@ -34,11 +38,12 @@ func NewMessageServiceClient(cc grpc.ClientConnInterface) MessageServiceClient { } func (c *messageServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) { - stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], "/grpc.MessageService/Connect", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], MessageService_Connect_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &messageServiceConnectClient{stream} + x := &messageServiceConnectClient{ClientStream: stream} return x, nil } @@ -93,7 +98,7 @@ func RegisterMessageServiceServer(s grpc.ServiceRegistrar, srv MessageServiceSer } func _MessageService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{stream}) + return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{ServerStream: stream}) } type MessageService_ConnectServer interface { diff --git a/grpc/metrics_service_v2_grpc.pb.go b/grpc/metrics_service_v2_grpc.pb.go index 23fe5c83b..c65e0263d 100644 --- a/grpc/metrics_service_v2_grpc.pb.go +++ b/grpc/metrics_service_v2_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/metrics_service_v2.proto @@ -15,8 +15,12 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + MetricsServiceV2_Connect_FullMethodName = "/grpc.MetricsServiceV2/Connect" +) // MetricsServiceV2Client is the client API for MetricsServiceV2 service. // @@ -34,11 +38,12 @@ func NewMetricsServiceV2Client(cc grpc.ClientConnInterface) MetricsServiceV2Clie } func (c *metricsServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (MetricsServiceV2_ConnectClient, error) { - stream, err := c.cc.NewStream(ctx, &MetricsServiceV2_ServiceDesc.Streams[0], "/grpc.MetricsServiceV2/Connect", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &MetricsServiceV2_ServiceDesc.Streams[0], MetricsServiceV2_Connect_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &metricsServiceV2ConnectClient{stream} + x := &metricsServiceV2ConnectClient{ClientStream: stream} return x, nil } @@ -93,7 +98,7 @@ func RegisterMetricsServiceV2Server(s grpc.ServiceRegistrar, srv MetricsServiceV } func _MetricsServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(MetricsServiceV2Server).Connect(&metricsServiceV2ConnectServer{stream}) + return srv.(MetricsServiceV2Server).Connect(&metricsServiceV2ConnectServer{ServerStream: stream}) } type MetricsServiceV2_ConnectServer interface { diff --git a/grpc/model_base_service_grpc.pb.go b/grpc/model_base_service_grpc.pb.go index efca79065..b5c75d4c7 100644 --- a/grpc/model_base_service_grpc.pb.go +++ b/grpc/model_base_service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/model_base_service.proto @@ -15,8 +15,23 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ModelBaseService_GetById_FullMethodName = "/grpc.ModelBaseService/GetById" + ModelBaseService_Get_FullMethodName = "/grpc.ModelBaseService/Get" + ModelBaseService_GetList_FullMethodName = "/grpc.ModelBaseService/GetList" + ModelBaseService_DeleteById_FullMethodName = "/grpc.ModelBaseService/DeleteById" + ModelBaseService_Delete_FullMethodName = "/grpc.ModelBaseService/Delete" + ModelBaseService_DeleteList_FullMethodName = "/grpc.ModelBaseService/DeleteList" + ModelBaseService_ForceDeleteList_FullMethodName = "/grpc.ModelBaseService/ForceDeleteList" + ModelBaseService_UpdateById_FullMethodName = "/grpc.ModelBaseService/UpdateById" + ModelBaseService_Update_FullMethodName = "/grpc.ModelBaseService/Update" + ModelBaseService_UpdateDoc_FullMethodName = "/grpc.ModelBaseService/UpdateDoc" + ModelBaseService_Insert_FullMethodName = "/grpc.ModelBaseService/Insert" + ModelBaseService_Count_FullMethodName = "/grpc.ModelBaseService/Count" +) // ModelBaseServiceClient is the client API for ModelBaseService service. // @@ -45,8 +60,9 @@ func NewModelBaseServiceClient(cc grpc.ClientConnInterface) ModelBaseServiceClie } func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_GetById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -54,8 +70,9 @@ func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts } func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Get", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Get_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -63,8 +80,9 @@ func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...g } func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetList", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_GetList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -72,8 +90,9 @@ func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts } func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_DeleteById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -81,8 +100,9 @@ func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Delete", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Delete_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -90,8 +110,9 @@ func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteList", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_DeleteList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -99,8 +120,9 @@ func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/ForceDeleteList", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_ForceDeleteList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -108,8 +130,9 @@ func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Reques } func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_UpdateById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -117,8 +140,9 @@ func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, op } func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Update", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Update_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -126,8 +150,9 @@ func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateDoc", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_UpdateDoc_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -135,8 +160,9 @@ func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opt } func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Insert", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Insert_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -144,8 +170,9 @@ func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts . } func (c *modelBaseServiceClient) Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Count", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseService_Count_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -234,7 +261,7 @@ func _ModelBaseService_GetById_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/GetById", + FullMethod: ModelBaseService_GetById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).GetById(ctx, req.(*Request)) @@ -252,7 +279,7 @@ func _ModelBaseService_Get_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Get", + FullMethod: ModelBaseService_Get_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Get(ctx, req.(*Request)) @@ -270,7 +297,7 @@ func _ModelBaseService_GetList_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/GetList", + FullMethod: ModelBaseService_GetList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).GetList(ctx, req.(*Request)) @@ -288,7 +315,7 @@ func _ModelBaseService_DeleteById_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/DeleteById", + FullMethod: ModelBaseService_DeleteById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).DeleteById(ctx, req.(*Request)) @@ -306,7 +333,7 @@ func _ModelBaseService_Delete_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Delete", + FullMethod: ModelBaseService_Delete_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Delete(ctx, req.(*Request)) @@ -324,7 +351,7 @@ func _ModelBaseService_DeleteList_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/DeleteList", + FullMethod: ModelBaseService_DeleteList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).DeleteList(ctx, req.(*Request)) @@ -342,7 +369,7 @@ func _ModelBaseService_ForceDeleteList_Handler(srv interface{}, ctx context.Cont } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/ForceDeleteList", + FullMethod: ModelBaseService_ForceDeleteList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, req.(*Request)) @@ -360,7 +387,7 @@ func _ModelBaseService_UpdateById_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/UpdateById", + FullMethod: ModelBaseService_UpdateById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).UpdateById(ctx, req.(*Request)) @@ -378,7 +405,7 @@ func _ModelBaseService_Update_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Update", + FullMethod: ModelBaseService_Update_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Update(ctx, req.(*Request)) @@ -396,7 +423,7 @@ func _ModelBaseService_UpdateDoc_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/UpdateDoc", + FullMethod: ModelBaseService_UpdateDoc_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).UpdateDoc(ctx, req.(*Request)) @@ -414,7 +441,7 @@ func _ModelBaseService_Insert_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Insert", + FullMethod: ModelBaseService_Insert_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Insert(ctx, req.(*Request)) @@ -432,7 +459,7 @@ func _ModelBaseService_Count_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseService/Count", + FullMethod: ModelBaseService_Count_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceServer).Count(ctx, req.(*Request)) diff --git a/grpc/model_base_service_v2_grpc.pb.go b/grpc/model_base_service_v2_grpc.pb.go index 203049d72..4bc81df09 100644 --- a/grpc/model_base_service_v2_grpc.pb.go +++ b/grpc/model_base_service_v2_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/model_base_service_v2.proto @@ -15,8 +15,25 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ModelBaseServiceV2_GetById_FullMethodName = "/grpc.ModelBaseServiceV2/GetById" + ModelBaseServiceV2_GetOne_FullMethodName = "/grpc.ModelBaseServiceV2/GetOne" + ModelBaseServiceV2_GetMany_FullMethodName = "/grpc.ModelBaseServiceV2/GetMany" + ModelBaseServiceV2_DeleteById_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteById" + ModelBaseServiceV2_DeleteOne_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteOne" + ModelBaseServiceV2_DeleteMany_FullMethodName = "/grpc.ModelBaseServiceV2/DeleteMany" + ModelBaseServiceV2_UpdateById_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateById" + ModelBaseServiceV2_UpdateOne_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateOne" + ModelBaseServiceV2_UpdateMany_FullMethodName = "/grpc.ModelBaseServiceV2/UpdateMany" + ModelBaseServiceV2_ReplaceById_FullMethodName = "/grpc.ModelBaseServiceV2/ReplaceById" + ModelBaseServiceV2_ReplaceOne_FullMethodName = "/grpc.ModelBaseServiceV2/ReplaceOne" + ModelBaseServiceV2_InsertOne_FullMethodName = "/grpc.ModelBaseServiceV2/InsertOne" + ModelBaseServiceV2_InsertMany_FullMethodName = "/grpc.ModelBaseServiceV2/InsertMany" + ModelBaseServiceV2_Count_FullMethodName = "/grpc.ModelBaseServiceV2/Count" +) // ModelBaseServiceV2Client is the client API for ModelBaseServiceV2 service. // @@ -47,8 +64,9 @@ func NewModelBaseServiceV2Client(cc grpc.ClientConnInterface) ModelBaseServiceV2 } func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -56,8 +74,9 @@ func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelService } func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -65,8 +84,9 @@ func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV } func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_GetMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -74,8 +94,9 @@ func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelService } func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -83,8 +104,9 @@ func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -92,8 +114,9 @@ func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_DeleteMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -101,8 +124,9 @@ func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -110,8 +134,9 @@ func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -119,8 +144,9 @@ func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_UpdateMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -128,8 +154,9 @@ func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceById", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_ReplaceById_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -137,8 +164,9 @@ func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelSer } func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_ReplaceOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -146,8 +174,9 @@ func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertOne", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_InsertOne_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -155,8 +184,9 @@ func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServi } func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertMany", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_InsertMany_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -164,8 +194,9 @@ func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServ } func (c *modelBaseServiceV2Client) Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/Count", in, out, opts...) + err := c.cc.Invoke(ctx, ModelBaseServiceV2_Count_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -262,7 +293,7 @@ func _ModelBaseServiceV2_GetById_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/GetById", + FullMethod: ModelBaseServiceV2_GetById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetById(ctx, req.(*ModelServiceV2GetByIdRequest)) @@ -280,7 +311,7 @@ func _ModelBaseServiceV2_GetOne_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/GetOne", + FullMethod: ModelBaseServiceV2_GetOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetOne(ctx, req.(*ModelServiceV2GetOneRequest)) @@ -298,7 +329,7 @@ func _ModelBaseServiceV2_GetMany_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/GetMany", + FullMethod: ModelBaseServiceV2_GetMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).GetMany(ctx, req.(*ModelServiceV2GetManyRequest)) @@ -316,7 +347,7 @@ func _ModelBaseServiceV2_DeleteById_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/DeleteById", + FullMethod: ModelBaseServiceV2_DeleteById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteById(ctx, req.(*ModelServiceV2DeleteByIdRequest)) @@ -334,7 +365,7 @@ func _ModelBaseServiceV2_DeleteOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/DeleteOne", + FullMethod: ModelBaseServiceV2_DeleteOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, req.(*ModelServiceV2DeleteOneRequest)) @@ -352,7 +383,7 @@ func _ModelBaseServiceV2_DeleteMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/DeleteMany", + FullMethod: ModelBaseServiceV2_DeleteMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, req.(*ModelServiceV2DeleteManyRequest)) @@ -370,7 +401,7 @@ func _ModelBaseServiceV2_UpdateById_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/UpdateById", + FullMethod: ModelBaseServiceV2_UpdateById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateById(ctx, req.(*ModelServiceV2UpdateByIdRequest)) @@ -388,7 +419,7 @@ func _ModelBaseServiceV2_UpdateOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/UpdateOne", + FullMethod: ModelBaseServiceV2_UpdateOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, req.(*ModelServiceV2UpdateOneRequest)) @@ -406,7 +437,7 @@ func _ModelBaseServiceV2_UpdateMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/UpdateMany", + FullMethod: ModelBaseServiceV2_UpdateMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, req.(*ModelServiceV2UpdateManyRequest)) @@ -424,7 +455,7 @@ func _ModelBaseServiceV2_ReplaceById_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/ReplaceById", + FullMethod: ModelBaseServiceV2_ReplaceById_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, req.(*ModelServiceV2ReplaceByIdRequest)) @@ -442,7 +473,7 @@ func _ModelBaseServiceV2_ReplaceOne_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/ReplaceOne", + FullMethod: ModelBaseServiceV2_ReplaceOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, req.(*ModelServiceV2ReplaceOneRequest)) @@ -460,7 +491,7 @@ func _ModelBaseServiceV2_InsertOne_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/InsertOne", + FullMethod: ModelBaseServiceV2_InsertOne_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).InsertOne(ctx, req.(*ModelServiceV2InsertOneRequest)) @@ -478,7 +509,7 @@ func _ModelBaseServiceV2_InsertMany_Handler(srv interface{}, ctx context.Context } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/InsertMany", + FullMethod: ModelBaseServiceV2_InsertMany_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).InsertMany(ctx, req.(*ModelServiceV2InsertManyRequest)) @@ -496,7 +527,7 @@ func _ModelBaseServiceV2_Count_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelBaseServiceV2/Count", + FullMethod: ModelBaseServiceV2_Count_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelBaseServiceV2Server).Count(ctx, req.(*ModelServiceV2CountRequest)) diff --git a/grpc/model_delegate_grpc.pb.go b/grpc/model_delegate_grpc.pb.go index c8da4cc34..60acc5df2 100644 --- a/grpc/model_delegate_grpc.pb.go +++ b/grpc/model_delegate_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/model_delegate.proto @@ -15,8 +15,12 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + ModelDelegate_Do_FullMethodName = "/grpc.ModelDelegate/Do" +) // ModelDelegateClient is the client API for ModelDelegate service. // @@ -34,8 +38,9 @@ func NewModelDelegateClient(cc grpc.ClientConnInterface) ModelDelegateClient { } func (c *modelDelegateClient) Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.ModelDelegate/Do", in, out, opts...) + err := c.cc.Invoke(ctx, ModelDelegate_Do_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -80,7 +85,7 @@ func _ModelDelegate_Do_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.ModelDelegate/Do", + FullMethod: ModelDelegate_Do_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ModelDelegateServer).Do(ctx, req.(*Request)) diff --git a/grpc/node_service.pb.go b/grpc/node_service.pb.go index b5efe6c38..2cd3949ac 100644 --- a/grpc/node_service.pb.go +++ b/grpc/node_service.pb.go @@ -10,6 +10,7 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" + sync "sync" ) const ( @@ -19,6 +20,132 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type NodeServiceRegisterRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + IsMaster bool `protobuf:"varint,3,opt,name=isMaster,proto3" json:"isMaster,omitempty"` + AuthKey string `protobuf:"bytes,4,opt,name=authKey,proto3" json:"authKey,omitempty"` + MaxRunners int32 `protobuf:"varint,5,opt,name=maxRunners,proto3" json:"maxRunners,omitempty"` +} + +func (x *NodeServiceRegisterRequest) Reset() { + *x = NodeServiceRegisterRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_node_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeServiceRegisterRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeServiceRegisterRequest) ProtoMessage() {} + +func (x *NodeServiceRegisterRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_node_service_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeServiceRegisterRequest.ProtoReflect.Descriptor instead. +func (*NodeServiceRegisterRequest) Descriptor() ([]byte, []int) { + return file_services_node_service_proto_rawDescGZIP(), []int{0} +} + +func (x *NodeServiceRegisterRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *NodeServiceRegisterRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *NodeServiceRegisterRequest) GetIsMaster() bool { + if x != nil { + return x.IsMaster + } + return false +} + +func (x *NodeServiceRegisterRequest) GetAuthKey() string { + if x != nil { + return x.AuthKey + } + return "" +} + +func (x *NodeServiceRegisterRequest) GetMaxRunners() int32 { + if x != nil { + return x.MaxRunners + } + return 0 +} + +type NodeServiceSendHeartbeatRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` +} + +func (x *NodeServiceSendHeartbeatRequest) Reset() { + *x = NodeServiceSendHeartbeatRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_node_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeServiceSendHeartbeatRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeServiceSendHeartbeatRequest) ProtoMessage() {} + +func (x *NodeServiceSendHeartbeatRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_node_service_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeServiceSendHeartbeatRequest.ProtoReflect.Descriptor instead. +func (*NodeServiceSendHeartbeatRequest) Descriptor() ([]byte, []int) { + return file_services_node_service_proto_rawDescGZIP(), []int{1} +} + +func (x *NodeServiceSendHeartbeatRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + var File_services_node_service_proto protoreflect.FileDescriptor var file_services_node_service_proto_rawDesc = []byte{ @@ -28,44 +155,70 @@ var file_services_node_service_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xfa, 0x01, - 0x0a, 0x0b, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2b, 0x0a, - 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x0d, 0x53, 0x65, - 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x98, 0x01, + 0x0a, 0x1a, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x12, 0x18, + 0x0a, 0x07, 0x61, 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x61, 0x75, 0x74, 0x68, 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x61, 0x78, 0x52, + 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, + 0x78, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x73, 0x22, 0x33, 0x0a, 0x1f, 0x4e, 0x6f, 0x64, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, + 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x32, 0xfc, 0x01, + 0x0a, 0x0b, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3e, 0x0a, + 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, + 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x25, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, + 0x72, 0x69, 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x2e, 0x0a, 0x0b, + 0x55, 0x6e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x27, 0x0a, 0x04, - 0x50, 0x69, 0x6e, 0x67, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, - 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x2e, 0x0a, 0x0b, 0x55, 0x6e, - 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, - 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, + 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_services_node_service_proto_rawDescOnce sync.Once + file_services_node_service_proto_rawDescData = file_services_node_service_proto_rawDesc +) + +func file_services_node_service_proto_rawDescGZIP() []byte { + file_services_node_service_proto_rawDescOnce.Do(func() { + file_services_node_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_services_node_service_proto_rawDescData) + }) + return file_services_node_service_proto_rawDescData } +var file_services_node_service_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_services_node_service_proto_goTypes = []any{ - (*Request)(nil), // 0: grpc.Request - (*Response)(nil), // 1: grpc.Response - (*StreamMessage)(nil), // 2: grpc.StreamMessage + (*NodeServiceRegisterRequest)(nil), // 0: grpc.NodeServiceRegisterRequest + (*NodeServiceSendHeartbeatRequest)(nil), // 1: grpc.NodeServiceSendHeartbeatRequest + (*Request)(nil), // 2: grpc.Request + (*Response)(nil), // 3: grpc.Response + (*StreamMessage)(nil), // 4: grpc.StreamMessage } var file_services_node_service_proto_depIdxs = []int32{ - 0, // 0: grpc.NodeService.Register:input_type -> grpc.Request - 0, // 1: grpc.NodeService.SendHeartbeat:input_type -> grpc.Request - 0, // 2: grpc.NodeService.Ping:input_type -> grpc.Request - 0, // 3: grpc.NodeService.Subscribe:input_type -> grpc.Request - 0, // 4: grpc.NodeService.Unsubscribe:input_type -> grpc.Request - 1, // 5: grpc.NodeService.Register:output_type -> grpc.Response - 1, // 6: grpc.NodeService.SendHeartbeat:output_type -> grpc.Response - 1, // 7: grpc.NodeService.Ping:output_type -> grpc.Response - 2, // 8: grpc.NodeService.Subscribe:output_type -> grpc.StreamMessage - 1, // 9: grpc.NodeService.Unsubscribe:output_type -> grpc.Response - 5, // [5:10] is the sub-list for method output_type - 0, // [0:5] is the sub-list for method input_type + 0, // 0: grpc.NodeService.Register:input_type -> grpc.NodeServiceRegisterRequest + 1, // 1: grpc.NodeService.SendHeartbeat:input_type -> grpc.NodeServiceSendHeartbeatRequest + 2, // 2: grpc.NodeService.Subscribe:input_type -> grpc.Request + 2, // 3: grpc.NodeService.Unsubscribe:input_type -> grpc.Request + 3, // 4: grpc.NodeService.Register:output_type -> grpc.Response + 3, // 5: grpc.NodeService.SendHeartbeat:output_type -> grpc.Response + 4, // 6: grpc.NodeService.Subscribe:output_type -> grpc.StreamMessage + 3, // 7: grpc.NodeService.Unsubscribe:output_type -> grpc.Response + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -79,18 +232,45 @@ func file_services_node_service_proto_init() { file_entity_request_proto_init() file_entity_response_proto_init() file_entity_stream_message_proto_init() + if !protoimpl.UnsafeEnabled { + file_services_node_service_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*NodeServiceRegisterRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_services_node_service_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*NodeServiceSendHeartbeatRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_services_node_service_proto_rawDesc, NumEnums: 0, - NumMessages: 0, + NumMessages: 2, NumExtensions: 0, NumServices: 1, }, GoTypes: file_services_node_service_proto_goTypes, DependencyIndexes: file_services_node_service_proto_depIdxs, + MessageInfos: file_services_node_service_proto_msgTypes, }.Build() File_services_node_service_proto = out.File file_services_node_service_proto_rawDesc = nil diff --git a/grpc/node_service_grpc.pb.go b/grpc/node_service_grpc.pb.go index dbff3b281..db21128b4 100644 --- a/grpc/node_service_grpc.pb.go +++ b/grpc/node_service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/node_service.proto @@ -15,16 +15,22 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + NodeService_Register_FullMethodName = "/grpc.NodeService/Register" + NodeService_SendHeartbeat_FullMethodName = "/grpc.NodeService/SendHeartbeat" + NodeService_Subscribe_FullMethodName = "/grpc.NodeService/Subscribe" + NodeService_Unsubscribe_FullMethodName = "/grpc.NodeService/Unsubscribe" +) // NodeServiceClient is the client API for NodeService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type NodeServiceClient interface { - Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) - SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) - Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Register(ctx context.Context, in *NodeServiceRegisterRequest, opts ...grpc.CallOption) (*Response, error) + SendHeartbeat(ctx context.Context, in *NodeServiceSendHeartbeatRequest, opts ...grpc.CallOption) (*Response, error) Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) } @@ -37,27 +43,20 @@ func NewNodeServiceClient(cc grpc.ClientConnInterface) NodeServiceClient { return &nodeServiceClient{cc} } -func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { - out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/Register", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { +func (c *nodeServiceClient) Register(ctx context.Context, in *NodeServiceRegisterRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/SendHeartbeat", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_Register_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } -func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { +func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *NodeServiceSendHeartbeatRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/Ping", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_SendHeartbeat_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -65,11 +64,12 @@ func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc. } func (c *nodeServiceClient) Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], "/grpc.NodeService/Subscribe", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], NodeService_Subscribe_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &nodeServiceSubscribeClient{stream} + x := &nodeServiceSubscribeClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -97,8 +97,9 @@ func (x *nodeServiceSubscribeClient) Recv() (*StreamMessage, error) { } func (c *nodeServiceClient) Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.NodeService/Unsubscribe", in, out, opts...) + err := c.cc.Invoke(ctx, NodeService_Unsubscribe_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -109,9 +110,8 @@ func (c *nodeServiceClient) Unsubscribe(ctx context.Context, in *Request, opts . // All implementations must embed UnimplementedNodeServiceServer // for forward compatibility type NodeServiceServer interface { - Register(context.Context, *Request) (*Response, error) - SendHeartbeat(context.Context, *Request) (*Response, error) - Ping(context.Context, *Request) (*Response, error) + Register(context.Context, *NodeServiceRegisterRequest) (*Response, error) + SendHeartbeat(context.Context, *NodeServiceSendHeartbeatRequest) (*Response, error) Subscribe(*Request, NodeService_SubscribeServer) error Unsubscribe(context.Context, *Request) (*Response, error) mustEmbedUnimplementedNodeServiceServer() @@ -121,15 +121,12 @@ type NodeServiceServer interface { type UnimplementedNodeServiceServer struct { } -func (UnimplementedNodeServiceServer) Register(context.Context, *Request) (*Response, error) { +func (UnimplementedNodeServiceServer) Register(context.Context, *NodeServiceRegisterRequest) (*Response, error) { return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") } -func (UnimplementedNodeServiceServer) SendHeartbeat(context.Context, *Request) (*Response, error) { +func (UnimplementedNodeServiceServer) SendHeartbeat(context.Context, *NodeServiceSendHeartbeatRequest) (*Response, error) { return nil, status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented") } -func (UnimplementedNodeServiceServer) Ping(context.Context, *Request) (*Response, error) { - return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") -} func (UnimplementedNodeServiceServer) Subscribe(*Request, NodeService_SubscribeServer) error { return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") } @@ -150,7 +147,7 @@ func RegisterNodeServiceServer(s grpc.ServiceRegistrar, srv NodeServiceServer) { } func _NodeService_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Request) + in := new(NodeServiceRegisterRequest) if err := dec(in); err != nil { return nil, err } @@ -159,16 +156,16 @@ func _NodeService_Register_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/Register", + FullMethod: NodeService_Register_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NodeServiceServer).Register(ctx, req.(*Request)) + return srv.(NodeServiceServer).Register(ctx, req.(*NodeServiceRegisterRequest)) } return interceptor(ctx, in, info, handler) } func _NodeService_SendHeartbeat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Request) + in := new(NodeServiceSendHeartbeatRequest) if err := dec(in); err != nil { return nil, err } @@ -177,28 +174,10 @@ func _NodeService_SendHeartbeat_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/SendHeartbeat", + FullMethod: NodeService_SendHeartbeat_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NodeServiceServer).SendHeartbeat(ctx, req.(*Request)) - } - return interceptor(ctx, in, info, handler) -} - -func _NodeService_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Request) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(NodeServiceServer).Ping(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.NodeService/Ping", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NodeServiceServer).Ping(ctx, req.(*Request)) + return srv.(NodeServiceServer).SendHeartbeat(ctx, req.(*NodeServiceSendHeartbeatRequest)) } return interceptor(ctx, in, info, handler) } @@ -208,7 +187,7 @@ func _NodeService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) e if err := stream.RecvMsg(m); err != nil { return err } - return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{stream}) + return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{ServerStream: stream}) } type NodeService_SubscribeServer interface { @@ -234,7 +213,7 @@ func _NodeService_Unsubscribe_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.NodeService/Unsubscribe", + FullMethod: NodeService_Unsubscribe_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NodeServiceServer).Unsubscribe(ctx, req.(*Request)) @@ -257,10 +236,6 @@ var NodeService_ServiceDesc = grpc.ServiceDesc{ MethodName: "SendHeartbeat", Handler: _NodeService_SendHeartbeat_Handler, }, - { - MethodName: "Ping", - Handler: _NodeService_Ping_Handler, - }, { MethodName: "Unsubscribe", Handler: _NodeService_Unsubscribe_Handler, diff --git a/grpc/plugin_service_grpc.pb.go b/grpc/plugin_service_grpc.pb.go index 5ddd329da..45f021f3a 100644 --- a/grpc/plugin_service_grpc.pb.go +++ b/grpc/plugin_service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/plugin_service.proto @@ -15,8 +15,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + PluginService_Register_FullMethodName = "/grpc.PluginService/Register" + PluginService_Subscribe_FullMethodName = "/grpc.PluginService/Subscribe" + PluginService_Poll_FullMethodName = "/grpc.PluginService/Poll" +) // PluginServiceClient is the client API for PluginService service. // @@ -36,8 +42,9 @@ func NewPluginServiceClient(cc grpc.ClientConnInterface) PluginServiceClient { } func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.PluginService/Register", in, out, opts...) + err := c.cc.Invoke(ctx, PluginService_Register_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -45,11 +52,12 @@ func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, o } func (c *pluginServiceClient) Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], "/grpc.PluginService/Subscribe", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], PluginService_Subscribe_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &pluginServiceSubscribeClient{stream} + x := &pluginServiceSubscribeClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -77,11 +85,12 @@ func (x *pluginServiceSubscribeClient) Recv() (*StreamMessage, error) { } func (c *pluginServiceClient) Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) { - stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], "/grpc.PluginService/Poll", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], PluginService_Poll_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &pluginServicePollClient{stream} + x := &pluginServicePollClient{ClientStream: stream} return x, nil } @@ -153,7 +162,7 @@ func _PluginService_Register_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.PluginService/Register", + FullMethod: PluginService_Register_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(PluginServiceServer).Register(ctx, req.(*PluginRequest)) @@ -166,7 +175,7 @@ func _PluginService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{stream}) + return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{ServerStream: stream}) } type PluginService_SubscribeServer interface { @@ -183,7 +192,7 @@ func (x *pluginServiceSubscribeServer) Send(m *StreamMessage) error { } func _PluginService_Poll_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(PluginServiceServer).Poll(&pluginServicePollServer{stream}) + return srv.(PluginServiceServer).Poll(&pluginServicePollServer{ServerStream: stream}) } type PluginService_PollServer interface { diff --git a/grpc/proto/services/node_service.proto b/grpc/proto/services/node_service.proto index 4b43d083b..6b1f23f96 100644 --- a/grpc/proto/services/node_service.proto +++ b/grpc/proto/services/node_service.proto @@ -7,10 +7,20 @@ import "entity/stream_message.proto"; package grpc; option go_package = ".;grpc"; +message NodeServiceRegisterRequest { + string key = 1; + string name = 2; + bool isMaster = 3; + string authKey = 4; + int32 maxRunners = 5; +} +message NodeServiceSendHeartbeatRequest { + string key = 1; +} + service NodeService { - rpc Register(Request) returns (Response){}; - rpc SendHeartbeat(Request) returns (Response){}; - rpc Ping(Request) returns (Response){}; + rpc Register(NodeServiceRegisterRequest) returns (Response){}; + rpc SendHeartbeat(NodeServiceSendHeartbeatRequest) returns (Response){}; rpc Subscribe(Request) returns (stream StreamMessage){}; rpc Unsubscribe(Request) returns (Response){}; } diff --git a/grpc/task_service_grpc.pb.go b/grpc/task_service_grpc.pb.go index d71adef33..6cb274dd0 100644 --- a/grpc/task_service_grpc.pb.go +++ b/grpc/task_service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.27.2 // source: services/task_service.proto @@ -15,8 +15,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + TaskService_Subscribe_FullMethodName = "/grpc.TaskService/Subscribe" + TaskService_Fetch_FullMethodName = "/grpc.TaskService/Fetch" + TaskService_SendNotification_FullMethodName = "/grpc.TaskService/SendNotification" +) // TaskServiceClient is the client API for TaskService service. // @@ -36,11 +42,12 @@ func NewTaskServiceClient(cc grpc.ClientConnInterface) TaskServiceClient { } func (c *taskServiceClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) { - stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], "/grpc.TaskService/Subscribe", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], TaskService_Subscribe_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &taskServiceSubscribeClient{stream} + x := &taskServiceSubscribeClient{ClientStream: stream} return x, nil } @@ -70,8 +77,9 @@ func (x *taskServiceSubscribeClient) CloseAndRecv() (*Response, error) { } func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.TaskService/Fetch", in, out, opts...) + err := c.cc.Invoke(ctx, TaskService_Fetch_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -79,8 +87,9 @@ func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc } func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) - err := c.cc.Invoke(ctx, "/grpc.TaskService/SendNotification", in, out, opts...) + err := c.cc.Invoke(ctx, TaskService_SendNotification_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -124,7 +133,7 @@ func RegisterTaskServiceServer(s grpc.ServiceRegistrar, srv TaskServiceServer) { } func _TaskService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{stream}) + return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{ServerStream: stream}) } type TaskService_SubscribeServer interface { @@ -159,7 +168,7 @@ func _TaskService_Fetch_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.TaskService/Fetch", + FullMethod: TaskService_Fetch_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TaskServiceServer).Fetch(ctx, req.(*Request)) @@ -177,7 +186,7 @@ func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/grpc.TaskService/SendNotification", + FullMethod: TaskService_SendNotification_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TaskServiceServer).SendNotification(ctx, req.(*Request)) From de016348885b406d5ab5ac3e4b5e96b9fe904b7b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 1 Jul 2024 18:32:33 +0800 Subject: [PATCH 035/106] ci: added qodana code check --- .github/workflows/qodana_code_quality.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/qodana_code_quality.yml diff --git a/.github/workflows/qodana_code_quality.yml b/.github/workflows/qodana_code_quality.yml new file mode 100644 index 000000000..ebfb9343b --- /dev/null +++ b/.github/workflows/qodana_code_quality.yml @@ -0,0 +1,21 @@ +name: Qodana +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + - develop + +jobs: + qodana: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: 'Qodana Scan' + uses: JetBrains/qodana-action@v2024.1 + env: + QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} \ No newline at end of file From 7ca98f91420570a3d7c22815a1ef5c10f774d15a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 2 Jul 2024 09:48:40 +0800 Subject: [PATCH 036/106] chore: updated deps --- core/go.mod | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/go.mod b/core/go.mod index f574b5e60..2df5691c5 100644 --- a/core/go.mod +++ b/core/go.mod @@ -15,12 +15,12 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 - github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/db v0.0.0-20240701075920-023ba275664a + github.com/crawlab-team/crawlab/fs v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/grpc v0.0.0-20240614111723-e5b20af9a40b - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 - github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 - github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240701075920-023ba275664a + github.com/crawlab-team/crawlab/trace v0.0.0-20240701075920-023ba275664a + github.com/crawlab-team/crawlab/vcs v0.0.0-20240701075920-023ba275664a github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 From 3385caf972587c909d01ac2fe91d7cb60b9fc395 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 2 Jul 2024 09:51:47 +0800 Subject: [PATCH 037/106] chore: updated deps --- core/go.mod | 17 ++++++++------- core/go.sum | 29 ++++++++----------------- db/go.mod | 8 +++---- fs/go.mod | 2 +- fs/go.sum | 3 +-- go.work.sum | 49 ++++++++++++++++-------------------------- grpc/go.mod | 1 - grpc/go.sum | 2 -- template-parser/go.mod | 8 +++---- template-parser/go.sum | 12 ++++------- vcs/go.mod | 1 + vcs/go.sum | 2 +- 12 files changed, 53 insertions(+), 81 deletions(-) diff --git a/core/go.mod b/core/go.mod index 2df5691c5..fa31d6da7 100644 --- a/core/go.mod +++ b/core/go.mod @@ -14,7 +14,7 @@ replace ( require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 - github.com/cenkalti/backoff/v4 v4.1.0 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/crawlab-team/crawlab/db v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/fs v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/grpc v0.0.0-20240614111723-e5b20af9a40b @@ -32,7 +32,7 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-uuid v1.0.3 - github.com/imroc/req v0.3.0 + github.com/imroc/req v0.3.2 github.com/matcornic/hermes/v2 v2.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 @@ -46,7 +46,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.9.1 github.com/upper/db/v4 v4.6.0 - go.mongodb.org/mongo-driver v1.15.0 + go.mongodb.org/mongo-driver v1.15.1 go.uber.org/dig v1.10.0 google.golang.org/grpc v1.64.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df @@ -62,11 +62,11 @@ require ( github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect - github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/PuerkitoBio/goquery v1.9.2 // indirect github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect github.com/ajg/form v1.5.1 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/aokoli/goutils v1.0.1 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -116,7 +116,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.4 // indirect @@ -131,6 +131,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/olekukonko/tablewriter v0.0.1 // indirect + github.com/onsi/gomega v1.30.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -180,7 +181,7 @@ require ( go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect diff --git a/core/go.sum b/core/go.sum index bd3fb4266..9bfd5e460 100644 --- a/core/go.sum +++ b/core/go.sum @@ -68,8 +68,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= -github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= -github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= @@ -80,11 +79,10 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -113,8 +111,7 @@ github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= -github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -343,8 +340,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= @@ -397,8 +392,7 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= -github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= +github.com/imroc/req v0.3.2 h1:M/JkeU6RPmX+WYvT2vaaOL0K+q8ufL5LxwvJc4xeB4o= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -472,8 +466,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -562,8 +555,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -770,8 +762,7 @@ github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyG go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -850,8 +841,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -927,7 +917,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= diff --git a/db/go.mod b/db/go.mod index d48c39dd7..c0f417aaf 100644 --- a/db/go.mod +++ b/db/go.mod @@ -6,13 +6,13 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 - github.com/cenkalti/backoff/v4 v4.1.0 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - go.mongodb.org/mongo-driver v1.15.0 + go.mongodb.org/mongo-driver v1.15.1 ) require ( @@ -22,7 +22,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/lib/pq v1.10.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect @@ -47,7 +47,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect diff --git a/fs/go.mod b/fs/go.mod index 0cf9826b6..2474624d4 100644 --- a/fs/go.mod +++ b/fs/go.mod @@ -6,7 +6,7 @@ replace github.com/crawlab-team/crawlab/trace => ../trace require ( github.com/apex/log v1.9.0 - github.com/cenkalti/backoff/v4 v4.1.0 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/crawlab-team/goseaweedfs v0.6.3 github.com/emirpasic/gods v1.18.1 diff --git a/fs/go.sum b/fs/go.sum index 00dede36a..dfb2527d4 100644 --- a/fs/go.sum +++ b/fs/go.sum @@ -5,8 +5,7 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= -github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/go.work.sum b/go.work.sum index 9590de37b..e74231d68 100644 --- a/go.work.sum +++ b/go.work.sum @@ -514,8 +514,11 @@ github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dX github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE= github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= @@ -538,10 +541,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= -github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= -github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= -github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -553,10 +553,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWs github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= @@ -570,7 +566,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -590,11 +585,7 @@ github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CA github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= @@ -604,8 +595,6 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= -github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -617,13 +606,14 @@ github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9 h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= @@ -709,6 +699,7 @@ github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHL github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= +github.com/imroc/req v0.3.2/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= @@ -716,7 +707,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= @@ -724,25 +714,20 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/knz/go-libedit v1.10.1 h1:0pHpWtx9vcvC0xGZqEQlQdfSQs7WRlAjuPvk3fOZDCo= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lyft/protoc-gen-star v0.5.3 h1:zSGLzsUew8RT+ZKPHc3jnf8XLaVyHzTcAFBzHtCNR20= github.com/lyft/protoc-gen-star/v2 v2.0.3 h1:/3+/2sWyXeMLzKd1bX+ixWKgEMsULrIivpDsuaF441o= github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= @@ -765,6 +750,9 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= @@ -822,8 +810,6 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9 github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -847,6 +833,7 @@ go.etcd.io/etcd/client/v2 v2.305.12 h1:0m4ovXYo1CHaA/Mp3X/Fak5sRNIWf01wk/X1/G3sG go.etcd.io/etcd/client/v2 v2.305.12/go.mod h1:aQ/yhsxMu+Oht1FOupSr60oBvcS9cKXHrzBpDsPTf9E= go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= +go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -866,7 +853,6 @@ go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFu go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= @@ -883,8 +869,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= -golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -900,13 +884,16 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -934,6 +921,7 @@ golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -945,6 +933,7 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -955,6 +944,7 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= @@ -1077,7 +1067,6 @@ modernc.org/sortutil v1.1.0 h1:oP3U4uM+NT/qBQcbg/K2iqAX0Nx7B1b6YZtq3Gk/PjM= modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= modernc.org/zappy v1.0.3 h1:Tr+P3kclDSrvC6zYBW2hWmOmu5SjG6PtvCt3RCjRmss= nullprogram.com/x/optparse v1.0.0 h1:xGFgVi5ZaWOnYdac2foDT3vg0ZZC9ErXFV57mr4OHrI= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= diff --git a/grpc/go.mod b/grpc/go.mod index c9c005c9a..4af7ffc81 100644 --- a/grpc/go.mod +++ b/grpc/go.mod @@ -3,7 +3,6 @@ module github.com/crawlab-team/crawlab/grpc go 1.22 require ( - github.com/golang/protobuf v1.5.4 google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 ) diff --git a/grpc/go.sum b/grpc/go.sum index 664480a81..3de846abb 100644 --- a/grpc/go.sum +++ b/grpc/go.sum @@ -1,5 +1,3 @@ -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= diff --git a/template-parser/go.mod b/template-parser/go.mod index 5602ba475..3829a2a10 100644 --- a/template-parser/go.mod +++ b/template-parser/go.mod @@ -8,17 +8,17 @@ require ( github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 github.com/stretchr/testify v1.9.0 - go.mongodb.org/mongo-driver v1.15.0 + go.mongodb.org/mongo-driver v1.15.1 ) require ( github.com/apex/log v1.9.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect @@ -40,7 +40,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect diff --git a/template-parser/go.sum b/template-parser/go.sum index f0fac8a58..f96dcfeb2 100644 --- a/template-parser/go.sum +++ b/template-parser/go.sum @@ -5,8 +5,7 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= -github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -28,8 +27,7 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -108,8 +106,7 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -119,8 +116,7 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/vcs/go.mod b/vcs/go.mod index 5f5060e6e..9d6fe9e7d 100644 --- a/vcs/go.mod +++ b/vcs/go.mod @@ -22,6 +22,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/onsi/gomega v1.30.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/vcs/go.sum b/vcs/go.sum index e4c471386..a041cdf5a 100644 --- a/vcs/go.sum +++ b/vcs/go.sum @@ -57,7 +57,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From ffcb44ff9eca1835fedb4ebe8683d2b337d2c047 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 2 Jul 2024 10:22:32 +0800 Subject: [PATCH 038/106] chore: updated deps --- core/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/go.mod b/core/go.mod index fa31d6da7..aa42b786b 100644 --- a/core/go.mod +++ b/core/go.mod @@ -17,7 +17,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/crawlab-team/crawlab/db v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/fs v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/grpc v0.0.0-20240614111723-e5b20af9a40b + github.com/crawlab-team/crawlab/grpc v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/template-parser v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/trace v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/vcs v0.0.0-20240701075920-023ba275664a From 7eb8c08e0efc7cbeac18ca730235ce583452c80c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 2 Jul 2024 17:59:40 +0800 Subject: [PATCH 039/106] feat: added metric service --- core/grpc/server/metrics_server_v2.go | 98 ++--- .../server/model_base_service_v2_server.go | 1 + core/models/common/index_service_v2.go | 20 + core/models/models/metric_v2.go | 23 ++ grpc/metrics_service_v2.pb.go | 373 +++++++----------- grpc/metrics_service_v2_grpc.pb.go | 86 ++-- grpc/proto/services/metrics_service_v2.proto | 42 +- 7 files changed, 273 insertions(+), 370 deletions(-) create mode 100644 core/models/models/metric_v2.go diff --git a/core/grpc/server/metrics_server_v2.go b/core/grpc/server/metrics_server_v2.go index c694b2ab9..fc51ad01a 100644 --- a/core/grpc/server/metrics_server_v2.go +++ b/core/grpc/server/metrics_server_v2.go @@ -1,86 +1,66 @@ package server import ( - "errors" + "context" "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/grpc" - "io" + "go.mongodb.org/mongo-driver/bson" "sync" + "time" ) type MetricsServerV2 struct { grpc.UnimplementedMetricsServiceV2Server - mu *sync.Mutex - streams map[string]*grpc.MetricsServiceV2_ConnectServer - channels map[string]chan []*grpc.Metric } -func (svr MetricsServerV2) Connect(stream grpc.MetricsServiceV2_ConnectServer) (err error) { - // receive first message - req, err := stream.Recv() +func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2SendRequest) (res *grpc.Response, err error) { + log.Info("[MetricsServerV2] received metric from node: " + req.NodeKey) + n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) if err != nil { - log.Errorf("[MetricsServerV2] receive error: %v", err) - return err + log.Errorf("[MetricsServerV2] error getting node: %v", err) + return HandleError(err) } - - // save stream and channel - svr.mu.Lock() - svr.streams[req.NodeKey] = &stream - svr.channels[req.NodeKey] = make(chan []*grpc.Metric) - svr.mu.Unlock() - - log.Info("[MetricsServerV2] connected: " + req.NodeKey) - - for { - // receive metrics - req, err = stream.Recv() - if errors.Is(err, io.EOF) { - log.Errorf("[MetricsServerV2] receive EOF: %v", err) - return - } - - // send metrics to channel - svr.channels[req.NodeKey] <- req.Metrics - - // keep this scope alive because once this scope exits - the stream is closed - select { - case <-stream.Context().Done(): - log.Info("[MetricsServerV2] disconnected: " + req.NodeKey) - delete(svr.streams, req.NodeKey) - delete(svr.channels, req.NodeKey) - return nil - } + metric := models.MetricV2{ + Type: req.Type, + NodeId: n.Id, + CpuUsagePercent: req.CpuUsagePercent, + TotalMemory: req.TotalMemory, + AvailableMemory: req.AvailableMemory, + UsedMemory: req.UsedMemory, + UsedMemoryPercent: req.UsedMemoryPercent, + TotalDisk: req.TotalDisk, + AvailableDisk: req.AvailableDisk, + UsedDisk: req.UsedDisk, + UsedDiskPercent: req.UsedDiskPercent, + DiskReadBytesRate: req.DiskReadBytesRate, + DiskWriteBytesRate: req.DiskWriteBytesRate, + NetworkBytesSentRate: req.NetworkBytesSentRate, + NetworkBytesRecvRate: req.NetworkBytesRecvRate, } + metric.CreatedAt = time.Unix(req.Timestamp, 0) + _, err = service.NewModelServiceV2[models.MetricV2]().InsertOne(metric) + if err != nil { + log.Errorf("[MetricsServerV2] error inserting metric: %v", err) + return HandleError(err) + } + return HandleSuccess() } -func (svr MetricsServerV2) GetStream(nodeKey string) (stream *grpc.MetricsServiceV2_ConnectServer, ok bool) { - svr.mu.Lock() - defer svr.mu.Unlock() - stream, ok = svr.streams[nodeKey] - return stream, ok -} - -func (svr MetricsServerV2) GetChannel(nodeKey string) (ch chan []*grpc.Metric, ok bool) { - svr.mu.Lock() - defer svr.mu.Unlock() - ch, ok = svr.channels[nodeKey] - return ch, ok -} - -func NewMetricsServerV2() *MetricsServerV2 { - return &MetricsServerV2{ - mu: new(sync.Mutex), - streams: make(map[string]*grpc.MetricsServiceV2_ConnectServer), - channels: make(map[string]chan []*grpc.Metric), - } +func newMetricsServerV2() *MetricsServerV2 { + return &MetricsServerV2{} } var metricsServerV2 *MetricsServerV2 +var metricsServerV2Once = &sync.Once{} func GetMetricsServerV2() *MetricsServerV2 { if metricsServerV2 != nil { return metricsServerV2 } - metricsServerV2 = NewMetricsServerV2() + metricsServerV2Once.Do(func() { + metricsServerV2 = newMetricsServerV2() + }) return metricsServerV2 } diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 5761e0605..f777da61a 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -25,6 +25,7 @@ var ( *new(models.DependencyTaskV2), *new(models.EnvironmentV2), *new(models.GitV2), + *new(models.MetricV2), *new(models.NodeV2), *new(models.NotificationSettingV2), *new(models.PermissionV2), diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index c46ec3dc5..e87ec7f35 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -157,4 +157,24 @@ func CreateIndexesV2() { Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24), }, }) + + // metrics + mongo.GetMongoCol(service.GetCollectionNameByInstance(models.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"created_ts", -1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30), + }, + { + Keys: bson.D{ + {"node_id", 1}, + }, + }, + { + Keys: bson.D{ + {"type", 1}, + }, + }, + }) } diff --git a/core/models/models/metric_v2.go b/core/models/models/metric_v2.go new file mode 100644 index 000000000..d7dbf0aac --- /dev/null +++ b/core/models/models/metric_v2.go @@ -0,0 +1,23 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type MetricV2 struct { + any `collection:"metrics"` + BaseModelV2[MetricV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` +} diff --git a/grpc/metrics_service_v2.pb.go b/grpc/metrics_service_v2.pb.go index 62a37ecf4..20705ba69 100644 --- a/grpc/metrics_service_v2.pb.go +++ b/grpc/metrics_service_v2.pb.go @@ -20,65 +20,31 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type MetricsServiceV2Code int32 - -const ( - MetricsServiceV2Code_SYNC_METRICS MetricsServiceV2Code = 0 -) - -// Enum value maps for MetricsServiceV2Code. -var ( - MetricsServiceV2Code_name = map[int32]string{ - 0: "SYNC_METRICS", - } - MetricsServiceV2Code_value = map[string]int32{ - "SYNC_METRICS": 0, - } -) - -func (x MetricsServiceV2Code) Enum() *MetricsServiceV2Code { - p := new(MetricsServiceV2Code) - *p = x - return p -} - -func (x MetricsServiceV2Code) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (MetricsServiceV2Code) Descriptor() protoreflect.EnumDescriptor { - return file_services_metrics_service_v2_proto_enumTypes[0].Descriptor() -} - -func (MetricsServiceV2Code) Type() protoreflect.EnumType { - return &file_services_metrics_service_v2_proto_enumTypes[0] -} - -func (x MetricsServiceV2Code) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use MetricsServiceV2Code.Descriptor instead. -func (MetricsServiceV2Code) EnumDescriptor() ([]byte, []int) { - return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{0} -} - -type Metric struct { +type MetricsServiceV2SendRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` - Subsystem string `protobuf:"bytes,2,opt,name=subsystem,proto3" json:"subsystem,omitempty"` - Module string `protobuf:"bytes,3,opt,name=module,proto3" json:"module,omitempty"` - Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` - Value float32 `protobuf:"fixed32,5,opt,name=value,proto3" json:"value,omitempty"` - Help string `protobuf:"bytes,6,opt,name=help,proto3" json:"help,omitempty"` - Labels []byte `protobuf:"bytes,7,opt,name=labels,proto3" json:"labels,omitempty"` -} - -func (x *Metric) Reset() { - *x = Metric{} + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + NodeKey string `protobuf:"bytes,2,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + CpuUsagePercent float32 `protobuf:"fixed32,4,opt,name=cpu_usage_percent,json=cpuUsagePercent,proto3" json:"cpu_usage_percent,omitempty"` + TotalMemory uint64 `protobuf:"varint,5,opt,name=total_memory,json=totalMemory,proto3" json:"total_memory,omitempty"` + AvailableMemory uint64 `protobuf:"varint,6,opt,name=available_memory,json=availableMemory,proto3" json:"available_memory,omitempty"` + UsedMemory uint64 `protobuf:"varint,7,opt,name=used_memory,json=usedMemory,proto3" json:"used_memory,omitempty"` + UsedMemoryPercent float32 `protobuf:"fixed32,8,opt,name=used_memory_percent,json=usedMemoryPercent,proto3" json:"used_memory_percent,omitempty"` + TotalDisk uint64 `protobuf:"varint,9,opt,name=total_disk,json=totalDisk,proto3" json:"total_disk,omitempty"` + AvailableDisk uint64 `protobuf:"varint,10,opt,name=available_disk,json=availableDisk,proto3" json:"available_disk,omitempty"` + UsedDisk uint64 `protobuf:"varint,11,opt,name=used_disk,json=usedDisk,proto3" json:"used_disk,omitempty"` + UsedDiskPercent float32 `protobuf:"fixed32,12,opt,name=used_disk_percent,json=usedDiskPercent,proto3" json:"used_disk_percent,omitempty"` + DiskReadBytesRate float32 `protobuf:"fixed32,15,opt,name=disk_read_bytes_rate,json=diskReadBytesRate,proto3" json:"disk_read_bytes_rate,omitempty"` + DiskWriteBytesRate float32 `protobuf:"fixed32,16,opt,name=disk_write_bytes_rate,json=diskWriteBytesRate,proto3" json:"disk_write_bytes_rate,omitempty"` + NetworkBytesSentRate float32 `protobuf:"fixed32,17,opt,name=network_bytes_sent_rate,json=networkBytesSentRate,proto3" json:"network_bytes_sent_rate,omitempty"` + NetworkBytesRecvRate float32 `protobuf:"fixed32,18,opt,name=network_bytes_recv_rate,json=networkBytesRecvRate,proto3" json:"network_bytes_recv_rate,omitempty"` +} + +func (x *MetricsServiceV2SendRequest) Reset() { + *x = MetricsServiceV2SendRequest{} if protoimpl.UnsafeEnabled { mi := &file_services_metrics_service_v2_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -86,13 +52,13 @@ func (x *Metric) Reset() { } } -func (x *Metric) String() string { +func (x *MetricsServiceV2SendRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Metric) ProtoMessage() {} +func (*MetricsServiceV2SendRequest) ProtoMessage() {} -func (x *Metric) ProtoReflect() protoreflect.Message { +func (x *MetricsServiceV2SendRequest) ProtoReflect() protoreflect.Message { mi := &file_services_metrics_service_v2_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -104,160 +70,121 @@ func (x *Metric) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Metric.ProtoReflect.Descriptor instead. -func (*Metric) Descriptor() ([]byte, []int) { +// Deprecated: Use MetricsServiceV2SendRequest.ProtoReflect.Descriptor instead. +func (*MetricsServiceV2SendRequest) Descriptor() ([]byte, []int) { return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{0} } -func (x *Metric) GetNamespace() string { +func (x *MetricsServiceV2SendRequest) GetType() string { if x != nil { - return x.Namespace + return x.Type } return "" } -func (x *Metric) GetSubsystem() string { +func (x *MetricsServiceV2SendRequest) GetNodeKey() string { if x != nil { - return x.Subsystem + return x.NodeKey } return "" } -func (x *Metric) GetModule() string { +func (x *MetricsServiceV2SendRequest) GetTimestamp() int64 { if x != nil { - return x.Module + return x.Timestamp } - return "" + return 0 } -func (x *Metric) GetName() string { +func (x *MetricsServiceV2SendRequest) GetCpuUsagePercent() float32 { if x != nil { - return x.Name + return x.CpuUsagePercent } - return "" + return 0 } -func (x *Metric) GetValue() float32 { +func (x *MetricsServiceV2SendRequest) GetTotalMemory() uint64 { if x != nil { - return x.Value + return x.TotalMemory } return 0 } -func (x *Metric) GetHelp() string { +func (x *MetricsServiceV2SendRequest) GetAvailableMemory() uint64 { if x != nil { - return x.Help + return x.AvailableMemory } - return "" + return 0 } -func (x *Metric) GetLabels() []byte { +func (x *MetricsServiceV2SendRequest) GetUsedMemory() uint64 { if x != nil { - return x.Labels + return x.UsedMemory } - return nil -} - -type MetricsServiceV2ConnectRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` - Metrics []*Metric `protobuf:"bytes,2,rep,name=metrics,proto3" json:"metrics,omitempty"` + return 0 } -func (x *MetricsServiceV2ConnectRequest) Reset() { - *x = MetricsServiceV2ConnectRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_services_metrics_service_v2_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *MetricsServiceV2SendRequest) GetUsedMemoryPercent() float32 { + if x != nil { + return x.UsedMemoryPercent } + return 0 } -func (x *MetricsServiceV2ConnectRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MetricsServiceV2ConnectRequest) ProtoMessage() {} - -func (x *MetricsServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { - mi := &file_services_metrics_service_v2_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *MetricsServiceV2SendRequest) GetTotalDisk() uint64 { + if x != nil { + return x.TotalDisk } - return mi.MessageOf(x) -} - -// Deprecated: Use MetricsServiceV2ConnectRequest.ProtoReflect.Descriptor instead. -func (*MetricsServiceV2ConnectRequest) Descriptor() ([]byte, []int) { - return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{1} + return 0 } -func (x *MetricsServiceV2ConnectRequest) GetNodeKey() string { +func (x *MetricsServiceV2SendRequest) GetAvailableDisk() uint64 { if x != nil { - return x.NodeKey + return x.AvailableDisk } - return "" + return 0 } -func (x *MetricsServiceV2ConnectRequest) GetMetrics() []*Metric { +func (x *MetricsServiceV2SendRequest) GetUsedDisk() uint64 { if x != nil { - return x.Metrics + return x.UsedDisk } - return nil -} - -type MetricsServiceV2ConnectResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Code MetricsServiceV2Code `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.MetricsServiceV2Code" json:"code,omitempty"` + return 0 } -func (x *MetricsServiceV2ConnectResponse) Reset() { - *x = MetricsServiceV2ConnectResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_services_metrics_service_v2_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *MetricsServiceV2SendRequest) GetUsedDiskPercent() float32 { + if x != nil { + return x.UsedDiskPercent } + return 0 } -func (x *MetricsServiceV2ConnectResponse) String() string { - return protoimpl.X.MessageStringOf(x) +func (x *MetricsServiceV2SendRequest) GetDiskReadBytesRate() float32 { + if x != nil { + return x.DiskReadBytesRate + } + return 0 } -func (*MetricsServiceV2ConnectResponse) ProtoMessage() {} - -func (x *MetricsServiceV2ConnectResponse) ProtoReflect() protoreflect.Message { - mi := &file_services_metrics_service_v2_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *MetricsServiceV2SendRequest) GetDiskWriteBytesRate() float32 { + if x != nil { + return x.DiskWriteBytesRate } - return mi.MessageOf(x) + return 0 } -// Deprecated: Use MetricsServiceV2ConnectResponse.ProtoReflect.Descriptor instead. -func (*MetricsServiceV2ConnectResponse) Descriptor() ([]byte, []int) { - return file_services_metrics_service_v2_proto_rawDescGZIP(), []int{2} +func (x *MetricsServiceV2SendRequest) GetNetworkBytesSentRate() float32 { + if x != nil { + return x.NetworkBytesSentRate + } + return 0 } -func (x *MetricsServiceV2ConnectResponse) GetCode() MetricsServiceV2Code { +func (x *MetricsServiceV2SendRequest) GetNetworkBytesRecvRate() float32 { if x != nil { - return x.Code + return x.NetworkBytesRecvRate } - return MetricsServiceV2Code_SYNC_METRICS + return 0 } var File_services_metrics_service_v2_proto protoreflect.FileDescriptor @@ -265,41 +192,56 @@ var File_services_metrics_service_v2_proto protoreflect.FileDescriptor var file_services_metrics_service_v2_proto_rawDesc = []byte{ 0x0a, 0x21, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0xb2, 0x01, 0x0a, 0x06, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x22, 0x63, - 0x0a, 0x1e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x07, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x67, - 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x22, 0x51, 0x0a, 0x1f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, 0x65, - 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x2a, 0x28, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x10, - 0x0a, 0x0c, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x53, 0x10, 0x00, - 0x32, 0x70, 0x0a, 0x10, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x56, 0x32, 0x12, 0x5c, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, - 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, - 0x30, 0x01, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x96, 0x05, 0x0a, 0x1b, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x2a, 0x0a, + 0x11, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, + 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0f, 0x63, 0x70, 0x75, 0x55, 0x73, 0x61, + 0x67, 0x65, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x74, + 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x29, 0x0a, 0x10, + 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x64, 0x5f, + 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x75, 0x73, + 0x65, 0x64, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x2e, 0x0a, 0x13, 0x75, 0x73, 0x65, 0x64, + 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, 0x75, 0x73, 0x65, 0x64, 0x4d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x76, 0x61, 0x69, 0x6c, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0d, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x1b, + 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x2a, 0x0a, 0x11, 0x75, + 0x73, 0x65, 0x64, 0x5f, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, + 0x18, 0x0c, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0f, 0x75, 0x73, 0x65, 0x64, 0x44, 0x69, 0x73, 0x6b, + 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x14, 0x64, 0x69, 0x73, 0x6b, 0x5f, + 0x72, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, + 0x0f, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, 0x64, 0x69, 0x73, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x52, 0x61, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x15, 0x64, 0x69, 0x73, 0x6b, + 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x72, 0x61, 0x74, + 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, 0x02, 0x52, 0x12, 0x64, 0x69, 0x73, 0x6b, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x52, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x17, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x6e, + 0x74, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x02, 0x52, 0x14, 0x6e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x65, 0x6e, 0x74, 0x52, 0x61, + 0x74, 0x65, 0x12, 0x35, 0x0a, 0x17, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x5f, 0x72, 0x65, 0x63, 0x76, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x12, 0x20, + 0x01, 0x28, 0x02, 0x52, 0x14, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x52, 0x65, 0x63, 0x76, 0x52, 0x61, 0x74, 0x65, 0x32, 0x4f, 0x0a, 0x10, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x12, 0x3b, 0x0a, + 0x04, 0x53, 0x65, 0x6e, 0x64, 0x12, 0x21, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x65, 0x6e, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -314,24 +256,19 @@ func file_services_metrics_service_v2_proto_rawDescGZIP() []byte { return file_services_metrics_service_v2_proto_rawDescData } -var file_services_metrics_service_v2_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_services_metrics_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_services_metrics_service_v2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_services_metrics_service_v2_proto_goTypes = []any{ - (MetricsServiceV2Code)(0), // 0: grpc.MetricsServiceV2Code - (*Metric)(nil), // 1: grpc.Metric - (*MetricsServiceV2ConnectRequest)(nil), // 2: grpc.MetricsServiceV2ConnectRequest - (*MetricsServiceV2ConnectResponse)(nil), // 3: grpc.MetricsServiceV2ConnectResponse + (*MetricsServiceV2SendRequest)(nil), // 0: grpc.MetricsServiceV2SendRequest + (*Response)(nil), // 1: grpc.Response } var file_services_metrics_service_v2_proto_depIdxs = []int32{ - 1, // 0: grpc.MetricsServiceV2ConnectRequest.metrics:type_name -> grpc.Metric - 0, // 1: grpc.MetricsServiceV2ConnectResponse.code:type_name -> grpc.MetricsServiceV2Code - 2, // 2: grpc.MetricsServiceV2.Connect:input_type -> grpc.MetricsServiceV2ConnectRequest - 3, // 3: grpc.MetricsServiceV2.Connect:output_type -> grpc.MetricsServiceV2ConnectResponse - 3, // [3:4] is the sub-list for method output_type - 2, // [2:3] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 0, // 0: grpc.MetricsServiceV2.Send:input_type -> grpc.MetricsServiceV2SendRequest + 1, // 1: grpc.MetricsServiceV2.Send:output_type -> grpc.Response + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name } func init() { file_services_metrics_service_v2_proto_init() } @@ -339,33 +276,10 @@ func file_services_metrics_service_v2_proto_init() { if File_services_metrics_service_v2_proto != nil { return } + file_entity_response_proto_init() if !protoimpl.UnsafeEnabled { file_services_metrics_service_v2_proto_msgTypes[0].Exporter = func(v any, i int) any { - switch v := v.(*Metric); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_services_metrics_service_v2_proto_msgTypes[1].Exporter = func(v any, i int) any { - switch v := v.(*MetricsServiceV2ConnectRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_services_metrics_service_v2_proto_msgTypes[2].Exporter = func(v any, i int) any { - switch v := v.(*MetricsServiceV2ConnectResponse); i { + switch v := v.(*MetricsServiceV2SendRequest); i { case 0: return &v.state case 1: @@ -382,14 +296,13 @@ func file_services_metrics_service_v2_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_services_metrics_service_v2_proto_rawDesc, - NumEnums: 1, - NumMessages: 3, + NumEnums: 0, + NumMessages: 1, NumExtensions: 0, NumServices: 1, }, GoTypes: file_services_metrics_service_v2_proto_goTypes, DependencyIndexes: file_services_metrics_service_v2_proto_depIdxs, - EnumInfos: file_services_metrics_service_v2_proto_enumTypes, MessageInfos: file_services_metrics_service_v2_proto_msgTypes, }.Build() File_services_metrics_service_v2_proto = out.File diff --git a/grpc/metrics_service_v2_grpc.pb.go b/grpc/metrics_service_v2_grpc.pb.go index c65e0263d..0b2bbf98c 100644 --- a/grpc/metrics_service_v2_grpc.pb.go +++ b/grpc/metrics_service_v2_grpc.pb.go @@ -19,14 +19,14 @@ import ( const _ = grpc.SupportPackageIsVersion8 const ( - MetricsServiceV2_Connect_FullMethodName = "/grpc.MetricsServiceV2/Connect" + MetricsServiceV2_Send_FullMethodName = "/grpc.MetricsServiceV2/Send" ) // MetricsServiceV2Client is the client API for MetricsServiceV2 service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type MetricsServiceV2Client interface { - Connect(ctx context.Context, opts ...grpc.CallOption) (MetricsServiceV2_ConnectClient, error) + Send(ctx context.Context, in *MetricsServiceV2SendRequest, opts ...grpc.CallOption) (*Response, error) } type metricsServiceV2Client struct { @@ -37,43 +37,21 @@ func NewMetricsServiceV2Client(cc grpc.ClientConnInterface) MetricsServiceV2Clie return &metricsServiceV2Client{cc} } -func (c *metricsServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (MetricsServiceV2_ConnectClient, error) { +func (c *metricsServiceV2Client) Send(ctx context.Context, in *MetricsServiceV2SendRequest, opts ...grpc.CallOption) (*Response, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &MetricsServiceV2_ServiceDesc.Streams[0], MetricsServiceV2_Connect_FullMethodName, cOpts...) + out := new(Response) + err := c.cc.Invoke(ctx, MetricsServiceV2_Send_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } - x := &metricsServiceV2ConnectClient{ClientStream: stream} - return x, nil -} - -type MetricsServiceV2_ConnectClient interface { - Send(*MetricsServiceV2ConnectRequest) error - Recv() (*MetricsServiceV2ConnectResponse, error) - grpc.ClientStream -} - -type metricsServiceV2ConnectClient struct { - grpc.ClientStream -} - -func (x *metricsServiceV2ConnectClient) Send(m *MetricsServiceV2ConnectRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *metricsServiceV2ConnectClient) Recv() (*MetricsServiceV2ConnectResponse, error) { - m := new(MetricsServiceV2ConnectResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil + return out, nil } // MetricsServiceV2Server is the server API for MetricsServiceV2 service. // All implementations must embed UnimplementedMetricsServiceV2Server // for forward compatibility type MetricsServiceV2Server interface { - Connect(MetricsServiceV2_ConnectServer) error + Send(context.Context, *MetricsServiceV2SendRequest) (*Response, error) mustEmbedUnimplementedMetricsServiceV2Server() } @@ -81,8 +59,8 @@ type MetricsServiceV2Server interface { type UnimplementedMetricsServiceV2Server struct { } -func (UnimplementedMetricsServiceV2Server) Connect(MetricsServiceV2_ConnectServer) error { - return status.Errorf(codes.Unimplemented, "method Connect not implemented") +func (UnimplementedMetricsServiceV2Server) Send(context.Context, *MetricsServiceV2SendRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Send not implemented") } func (UnimplementedMetricsServiceV2Server) mustEmbedUnimplementedMetricsServiceV2Server() {} @@ -97,30 +75,22 @@ func RegisterMetricsServiceV2Server(s grpc.ServiceRegistrar, srv MetricsServiceV s.RegisterService(&MetricsServiceV2_ServiceDesc, srv) } -func _MetricsServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(MetricsServiceV2Server).Connect(&metricsServiceV2ConnectServer{ServerStream: stream}) -} - -type MetricsServiceV2_ConnectServer interface { - Send(*MetricsServiceV2ConnectResponse) error - Recv() (*MetricsServiceV2ConnectRequest, error) - grpc.ServerStream -} - -type metricsServiceV2ConnectServer struct { - grpc.ServerStream -} - -func (x *metricsServiceV2ConnectServer) Send(m *MetricsServiceV2ConnectResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *metricsServiceV2ConnectServer) Recv() (*MetricsServiceV2ConnectRequest, error) { - m := new(MetricsServiceV2ConnectRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { +func _MetricsServiceV2_Send_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MetricsServiceV2SendRequest) + if err := dec(in); err != nil { return nil, err } - return m, nil + if interceptor == nil { + return srv.(MetricsServiceV2Server).Send(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MetricsServiceV2_Send_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetricsServiceV2Server).Send(ctx, req.(*MetricsServiceV2SendRequest)) + } + return interceptor(ctx, in, info, handler) } // MetricsServiceV2_ServiceDesc is the grpc.ServiceDesc for MetricsServiceV2 service. @@ -129,14 +99,12 @@ func (x *metricsServiceV2ConnectServer) Recv() (*MetricsServiceV2ConnectRequest, var MetricsServiceV2_ServiceDesc = grpc.ServiceDesc{ ServiceName: "grpc.MetricsServiceV2", HandlerType: (*MetricsServiceV2Server)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ + Methods: []grpc.MethodDesc{ { - StreamName: "Connect", - Handler: _MetricsServiceV2_Connect_Handler, - ServerStreams: true, - ClientStreams: true, + MethodName: "Send", + Handler: _MetricsServiceV2_Send_Handler, }, }, + Streams: []grpc.StreamDesc{}, Metadata: "services/metrics_service_v2.proto", } diff --git a/grpc/proto/services/metrics_service_v2.proto b/grpc/proto/services/metrics_service_v2.proto index ea63c1c65..6ed7ac25b 100644 --- a/grpc/proto/services/metrics_service_v2.proto +++ b/grpc/proto/services/metrics_service_v2.proto @@ -1,31 +1,29 @@ syntax = "proto3"; +import "entity/response.proto"; + package grpc; option go_package = ".;grpc"; -message Metric { - string namespace = 1; - string subsystem = 2; - string module = 3; - string name = 4; - float value = 5; - string help = 6; - bytes labels = 7; -} - -message MetricsServiceV2ConnectRequest { - string node_key = 1; - repeated Metric metrics = 2; -} - -enum MetricsServiceV2Code { - SYNC_METRICS = 0; -} - -message MetricsServiceV2ConnectResponse { - MetricsServiceV2Code code = 1; +message MetricsServiceV2SendRequest { + string type = 1; + string node_key = 2; + int64 timestamp = 3; + float cpu_usage_percent = 4; + uint64 total_memory = 5; + uint64 available_memory = 6; + uint64 used_memory = 7; + float used_memory_percent = 8; + uint64 total_disk = 9; + uint64 available_disk = 10; + uint64 used_disk = 11; + float used_disk_percent = 12; + float disk_read_bytes_rate = 15; + float disk_write_bytes_rate = 16; + float network_bytes_sent_rate = 17; + float network_bytes_recv_rate = 18; } service MetricsServiceV2 { - rpc Connect(stream MetricsServiceV2ConnectRequest) returns (stream MetricsServiceV2ConnectResponse){}; + rpc Send(MetricsServiceV2SendRequest) returns (Response){}; } From 5fbcbe0b9f70609fbd658f650e491b8a8d6a0fcf Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 2 Jul 2024 23:54:31 +0800 Subject: [PATCH 040/106] feat: added monitoring for node --- go.work.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.work.sum b/go.work.sum index e74231d68..b6bae2763 100644 --- a/go.work.sum +++ b/go.work.sum @@ -607,6 +607,7 @@ github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -707,6 +708,7 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= From 40f37e85ef5d62ea8c5945e7a593cd7207fca3d6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 3 Jul 2024 14:57:33 +0800 Subject: [PATCH 041/106] fix: missing name and max runners when registering nodes --- core/grpc/server/node_server_v2.go | 12 +++++++----- core/node/config/config_service.go | 3 --- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index 7df8d9979..a45e222e4 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -58,11 +58,13 @@ func (svr NodeServerV2) Register(ctx context.Context, req *grpc.NodeServiceRegis } else if errors2.Is(err, mongo.ErrNoDocuments) { // register new node = &models.NodeV2{ - Key: req.Key, - Status: constants.NodeStatusRegistered, - Active: true, - ActiveAt: time.Now(), - Enabled: true, + Key: req.Key, + Name: req.Name, + Status: constants.NodeStatusRegistered, + Active: true, + ActiveAt: time.Now(), + Enabled: true, + MaxRunners: int(req.MaxRunners), } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go index 9e772c6af..9a4cee4fc 100644 --- a/core/node/config/config_service.go +++ b/core/node/config/config_service.go @@ -119,9 +119,6 @@ var _service interfaces.NodeConfigService var _serviceOnce = new(sync.Once) func GetNodeConfigService() interfaces.NodeConfigService { - if _service != nil { - return _service - } _serviceOnce.Do(func() { var err error _service, err = newNodeConfigService() From f4025a982d4093a39253b4b4942e693f2c892e02 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 4 Jul 2024 13:01:54 +0800 Subject: [PATCH 042/106] feat: updated node monitoring tab --- core/utils/time.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/core/utils/time.go b/core/utils/time.go index 689e2c284..1ddcbdcfa 100644 --- a/core/utils/time.go +++ b/core/utils/time.go @@ -1,6 +1,11 @@ package utils import ( + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/trace" + "regexp" + "strconv" "time" ) @@ -16,3 +21,53 @@ func GetLocalTimeString(t time.Time) string { t = GetLocalTime(t) return GetTimeString(t) } + +func GetTimeUnitParts(timeUnit string) (num int, unit string, err error) { + re := regexp.MustCompile(`^(\d+)([a-zA-Z])$`) + groups := re.FindStringSubmatch(timeUnit) + if len(groups) < 3 { + err = errors.New("failed to parse duration text") + log.Errorf("failed to parse duration text: %v", err) + trace.PrintError(err) + return 0, "", err + } + num, err = strconv.Atoi(groups[1]) + if err != nil { + log.Errorf("failed to convert string to int: %v", err) + trace.PrintError(err) + return 0, "", err + } + unit = groups[2] + return num, unit, nil +} + +func GetTimeDuration(num string, unit string) (d time.Duration, err error) { + numInt, err := strconv.Atoi(num) + if err != nil { + log.Errorf("failed to convert string to int: %v", err) + trace.PrintError(err) + return d, err + } + switch unit { + case "s": + d = time.Duration(numInt) * time.Second + case "m": + d = time.Duration(numInt) * time.Minute + case "h": + d = time.Duration(numInt) * time.Hour + case "d": + d = time.Duration(numInt) * 24 * time.Hour + case "w": + d = time.Duration(numInt) * 7 * 24 * time.Hour + case "M": + d = time.Duration(numInt) * 30 * 24 * time.Hour + case "y": + d = time.Duration(numInt) * 365 * 24 * time.Hour + default: + err = errors.New("invalid time unit") + log.Errorf("invalid time unit: %v", unit) + trace.PrintError(err) + return d, err + } + return d, nil +} From 059711a533f5447f10939a06fff519be47b2cb4b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 5 Jul 2024 12:57:09 +0800 Subject: [PATCH 043/106] fix: unable to sync files from spiders created from sub-folder of git repo --- core/task/handler/runner_v2.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 46789bbfd..885b1c018 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -18,7 +18,7 @@ import ( service2 "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" "github.com/spf13/viper" @@ -414,7 +414,7 @@ func (r *RunnerV2) syncFiles() (err error) { } } else { log.Infof("File needs to be synchronized: %s", path) - _err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(r.cwd, path), masterFile) + _err := r.downloadFile(masterURL+"/download?path="+filepath.Join(workingDir, path), filepath.Join(r.cwd, path), masterFile) if _err != nil { log.Errorf("Error downloading file: %v", _err) err = errors.Join(err, _err) @@ -441,6 +441,10 @@ func (r *RunnerV2) downloadFile(url string, filePath string, fileInfo *entity.Fs log.Errorf("Error getting file response: %v", err) return err } + if resp.StatusCode != http.StatusOK { + log.Errorf("Error downloading file: %s", resp.Status) + return errors.New(resp.Status) + } defer resp.Body.Close() // create directory if not exists From 4da89762287d7a0273b3f6339b7b2dd5fe8000e4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 6 Jul 2024 14:15:14 +0800 Subject: [PATCH 044/106] refactor: code cleanup --- Jenkinsfile | 56 -- core/apps/api.go | 126 --- core/cmd/server.go | 1 - core/config/default_config.go | 5 +- core/controllers/data_collection.go | 103 -- core/controllers/data_source.go | 148 --- core/controllers/demo.go | 73 -- core/controllers/export.go | 127 --- core/controllers/filer.go | 130 --- core/controllers/filter.go | 100 -- core/controllers/git.go | 3 - core/controllers/init.go | 42 - core/controllers/login.go | 64 -- core/controllers/node.go | 94 -- core/controllers/notification.go | 158 --- core/controllers/permission.go | 3 - core/controllers/project.go | 103 -- core/controllers/result.go | 150 --- core/controllers/role.go | 3 - core/controllers/router_v2.go | 7 - core/controllers/schedule.go | 221 ---- core/controllers/setting.go | 84 -- core/controllers/spider.go | 1333 ------------------------- core/controllers/stats.go | 87 -- core/controllers/sync.go | 58 -- core/controllers/system_info.go | 28 - core/controllers/system_info_v2.go | 4 +- core/controllers/tag.go | 3 - core/controllers/task.go | 534 ---------- core/controllers/test/base.go | 104 -- core/controllers/test/export_test.go | 59 -- core/controllers/test/filter_test.go | 76 -- core/controllers/test/main_test.go | 44 - core/controllers/test/project_test.go | 305 ------ core/controllers/test/spider_test.go | 183 ---- core/controllers/test/task_test.go | 102 -- core/controllers/token.go | 84 -- core/controllers/user.go | 244 ----- core/controllers/version.go | 23 - core/routes/group.go | 20 - core/routes/router.go | 178 ---- core/routes/router_test.go | 26 - 42 files changed, 4 insertions(+), 5292 deletions(-) delete mode 100644 Jenkinsfile delete mode 100644 core/apps/api.go delete mode 100644 core/controllers/data_collection.go delete mode 100644 core/controllers/data_source.go delete mode 100644 core/controllers/demo.go delete mode 100644 core/controllers/export.go delete mode 100644 core/controllers/filer.go delete mode 100644 core/controllers/filter.go delete mode 100644 core/controllers/git.go delete mode 100644 core/controllers/init.go delete mode 100644 core/controllers/login.go delete mode 100644 core/controllers/node.go delete mode 100644 core/controllers/notification.go delete mode 100644 core/controllers/permission.go delete mode 100644 core/controllers/project.go delete mode 100644 core/controllers/result.go delete mode 100644 core/controllers/role.go delete mode 100644 core/controllers/schedule.go delete mode 100644 core/controllers/setting.go delete mode 100644 core/controllers/spider.go delete mode 100644 core/controllers/stats.go delete mode 100644 core/controllers/sync.go delete mode 100644 core/controllers/system_info.go delete mode 100644 core/controllers/tag.go delete mode 100644 core/controllers/task.go delete mode 100644 core/controllers/test/base.go delete mode 100644 core/controllers/test/export_test.go delete mode 100644 core/controllers/test/filter_test.go delete mode 100644 core/controllers/test/main_test.go delete mode 100644 core/controllers/test/project_test.go delete mode 100644 core/controllers/test/spider_test.go delete mode 100644 core/controllers/test/task_test.go delete mode 100644 core/controllers/token.go delete mode 100644 core/controllers/user.go delete mode 100644 core/controllers/version.go delete mode 100644 core/routes/group.go delete mode 100644 core/routes/router.go delete mode 100644 core/routes/router_test.go diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 24cc31c53..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,56 +0,0 @@ -pipeline { - agent { - node { - label 'crawlab' - } - } - - stages { - stage('Setup') { - steps { - echo "Running Setup..." - script { - if (env.GIT_BRANCH == 'develop') { - env.TAG = 'develop' - env.DOCKERFILE = 'Dockerfile.local' - } else if (env.GIT_BRANCH == 'master') { - env.TAG = 'master' - env.DOCKERFILE = 'Dockerfile.local' - } - } - } - } - stage('Build') { - steps { - echo "Building..." - sh """ - docker build -t tikazyq/crawlab:${ENV:TAG} -f ${ENV:DOCKERFILE} . - """ - } - } - stage('Test') { - steps { - echo 'Testing..' - } - } - stage('Deploy') { - steps { - echo 'Deploying....' - sh """ - # 重启docker compose - cd ./jenkins/${ENV:GIT_BRANCH} - docker-compose down | true - docker-compose up -d | true - """ - } - } - stage('Cleanup') { - steps { - echo 'Cleanup...' - sh """ - docker rmi -f `docker images | grep '' | grep -v IMAGE | awk '{ print \$3 }' | xargs` - """ - } - } - } -} \ No newline at end of file diff --git a/core/apps/api.go b/core/apps/api.go deleted file mode 100644 index 94e887c34..000000000 --- a/core/apps/api.go +++ /dev/null @@ -1,126 +0,0 @@ -package apps - -import ( - "context" - "errors" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/routes" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net" - "net/http" - "time" -) - -func init() { - // set gin mode - if viper.GetString("gin.mode") == "" { - gin.SetMode(gin.ReleaseMode) - } else { - gin.SetMode(viper.GetString("gin.mode")) - } -} - -type Api struct { - // dependencies - interfaces.WithConfigPath - - // internals - app *gin.Engine - ln net.Listener - srv *http.Server - ready bool -} - -func (app *Api) Init() { - // initialize controllers - _ = initModule("controllers", controllers.InitControllers) - - // initialize middlewares - _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) - - // initialize routes - _ = app.initModuleWithApp("routes", routes.InitRoutes) -} - -func (app *Api) Start() { - // address - host := viper.GetString("server.host") - port := viper.GetString("server.port") - address := net.JoinHostPort(host, port) - - // http server - app.srv = &http.Server{ - Handler: app.app, - Addr: address, - } - - // listen - var err error - app.ln, err = net.Listen("tcp", address) - if err != nil { - panic(err) - } - app.ready = true - - // serve - if err := http.Serve(app.ln, app.app); err != nil { - if !errors.Is(err, http.ErrServerClosed) { - log.Error("run server error:" + err.Error()) - } else { - log.Info("server graceful down") - } - } -} - -func (app *Api) Wait() { - DefaultWait() -} - -func (app *Api) Stop() { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := app.srv.Shutdown(ctx); err != nil { - log.Error("run server error:" + err.Error()) - } -} - -func (app *Api) GetGinEngine() *gin.Engine { - return app.app -} - -func (app *Api) GetHttpServer() *http.Server { - return app.srv -} - -func (app *Api) Ready() (ok bool) { - return app.ready -} - -func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { - return initModule(name, func() error { - return fn(app.app) - }) -} - -func NewApi() *Api { - api := &Api{ - app: gin.New(), - } - api.Init() - return api -} - -var api *Api - -func GetApi() *Api { - if api != nil { - return api - } - api = NewApi() - return api -} diff --git a/core/cmd/server.go b/core/cmd/server.go index 40f976ae4..afa7cd422 100644 --- a/core/cmd/server.go +++ b/core/cmd/server.go @@ -16,7 +16,6 @@ var serverCmd = &cobra.Command{ Long: `Start Crawlab node server that can serve as API, task scheduler, task runner, etc.`, Run: func(cmd *cobra.Command, args []string) { // app - //svr := apps.GetServer(opts...) svr := apps.GetServerV2() // start diff --git a/core/config/default_config.go b/core/config/default_config.go index 8f0e7ea74..e2806e670 100644 --- a/core/config/default_config.go +++ b/core/config/default_config.go @@ -1,9 +1,8 @@ package config var DefaultConfigYaml = ` -info: - version: v0.6.3 - edition: global.edition.community +version: v0.6.3 +edition: global.edition.community mongo: host: localhost port: 27017 diff --git a/core/controllers/data_collection.go b/core/controllers/data_collection.go deleted file mode 100644 index cf9ef9a9c..000000000 --- a/core/controllers/data_collection.go +++ /dev/null @@ -1,103 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var DataCollectionController *dataCollectionController - -func getDataCollectionActions() []Action { - ctx := newDataCollectionContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/indexes", - HandlerFunc: ctx.postIndexes, - }, - } -} - -type dataCollectionController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *dataCollectionContext -} - -type dataCollectionContext struct { - modelSvc service.ModelService - resultSvc interfaces.ResultService -} - -func (ctx *dataCollectionContext) postIndexes(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - dc, err := ctx.modelSvc.GetDataCollectionById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - for _, f := range dc.Fields { - if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{ - Keys: f.Key, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) -} - -var _dataCollectionCtx *dataCollectionContext - -func newDataCollectionContext() *dataCollectionContext { - if _dataCollectionCtx != nil { - return _dataCollectionCtx - } - - // context - ctx := &dataCollectionContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - ) { - ctx.modelSvc = modelSvc - }); err != nil { - panic(err) - } - - _dataCollectionCtx = ctx - - return ctx -} - -func newDataCollectionController() *dataCollectionController { - actions := getDataCollectionActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) - d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) - ctx := newDataCollectionContext() - - return &dataCollectionController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/data_source.go b/core/controllers/data_source.go deleted file mode 100644 index a4b905655..000000000 --- a/core/controllers/data_source.go +++ /dev/null @@ -1,148 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/ds" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - interfaces2 "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var DataSourceController *dataSourceController - -func getDataSourceActions() []Action { - ctx := newDataSourceContext() - return []Action{ - { - Path: "/:id/change-password", - Method: http.MethodPost, - HandlerFunc: ctx.changePassword, - }, - } -} - -type dataSourceController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *dataSourceContext -} - -func (ctr *dataSourceController) Post(c *gin.Context) { - // data source - var _ds models.DataSource - if err := c.ShouldBindJSON(&_ds); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // add data source to db - if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error { - if err := delegate.NewModelDelegate(&_ds).Add(); err != nil { - return trace.TraceError(err) - } - pwd, err := utils.EncryptAES(_ds.Password) - if err != nil { - return trace.TraceError(err) - } - p := models.Password{Id: _ds.Id, Password: pwd} - if err := delegate.NewModelDelegate(&p).Add(); err != nil { - return trace.TraceError(err) - } - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() - - HandleSuccess(c) -} - -func (ctr *dataSourceController) Put(c *gin.Context) { - // data source - var _ds models.DataSource - if err := c.ShouldBindJSON(&_ds); err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := delegate.NewModelDelegate(&_ds).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() -} - -type dataSourceContext struct { - dsSvc interfaces.DataSourceService -} - -var _dataSourceCtx *dataSourceContext - -func newDataSourceContext() *dataSourceContext { - if _dataSourceCtx != nil { - return _dataSourceCtx - } - dsSvc, err := ds.GetDataSourceService() - if err != nil { - panic(err) - } - _dataSourceCtx = &dataSourceContext{ - dsSvc: dsSvc, - } - return _dataSourceCtx -} - -func (ctx *dataSourceContext) changePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload map[string]string - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - password, ok := payload["password"] - if !ok { - HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields) - return - } - if err := ctx.dsSvc.ChangePassword(id, password); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func newDataSourceController() *dataSourceController { - actions := getDataSourceActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) - d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) - ctx := newDataSourceContext() - - return &dataSourceController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/demo.go b/core/controllers/demo.go deleted file mode 100644 index 7a09d9889..000000000 --- a/core/controllers/demo.go +++ /dev/null @@ -1,73 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "net/http" -) - -func getDemoActions() []Action { - ctx := newDemoContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/import", - HandlerFunc: ctx.import_, - }, - { - Method: http.MethodGet, - Path: "/reimport", - HandlerFunc: ctx.reimport, - }, - { - Method: http.MethodGet, - Path: "/cleanup", - HandlerFunc: ctx.cleanup, - }, - } -} - -type demoContext struct { -} - -func (ctx *demoContext) import_(c *gin.Context) { - if err := utils.ImportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *demoContext) reimport(c *gin.Context) { - if err := utils.ReimportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *demoContext) cleanup(c *gin.Context) { - if err := utils.ReimportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -var _demoCtx *demoContext - -func newDemoContext() *demoContext { - if _demoCtx != nil { - return _demoCtx - } - - _demoCtx = &demoContext{} - - return _demoCtx -} - -var DemoController ActionController diff --git a/core/controllers/export.go b/core/controllers/export.go deleted file mode 100644 index b99ffcb9d..000000000 --- a/core/controllers/export.go +++ /dev/null @@ -1,127 +0,0 @@ -package controllers - -import ( - "errors" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/export" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" - "net/http" -) - -var ExportController ActionController - -func getExportActions() []Action { - ctx := newExportContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:type", - HandlerFunc: ctx.postExport, - }, - { - Method: http.MethodGet, - Path: "/:type/:id", - HandlerFunc: ctx.getExport, - }, - { - Method: http.MethodGet, - Path: "/:type/:id/download", - HandlerFunc: ctx.getExportDownload, - }, - } -} - -type exportContext struct { - csvSvc interfaces.ExportService - jsonSvc interfaces.ExportService -} - -func (ctx *exportContext) postExport(c *gin.Context) { - exportType := c.Param("type") - exportTarget := c.Query("target") - exportFilter, _ := GetFilter(c) - - var exportId string - var err error - switch exportType { - case constants.ExportTypeCsv: - exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter) - case constants.ExportTypeJson: - exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exportId) -} - -func (ctx *exportContext) getExport(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = ctx.csvSvc.GetExport(exportId) - case constants.ExportTypeJson: - exp, err = ctx.jsonSvc.GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exp) -} - -func (ctx *exportContext) getExportDownload(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = ctx.csvSvc.GetExport(exportId) - case constants.ExportTypeJson: - exp, err = ctx.jsonSvc.GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - switch exportType { - case constants.ExportTypeCsv: - c.Header("Content-Type", "text/csv") - case constants.ExportTypeJson: - c.Header("Content-Type", "text/plain") - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath())) - c.File(exp.GetDownloadPath()) -} - -func newExportContext() *exportContext { - return &exportContext{ - csvSvc: export.GetCsvService(), - jsonSvc: export.GetJsonService(), - } -} diff --git a/core/controllers/filer.go b/core/controllers/filer.go deleted file mode 100644 index 4af5d1424..000000000 --- a/core/controllers/filer.go +++ /dev/null @@ -1,130 +0,0 @@ -package controllers - -import ( - "bufio" - "fmt" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/gin-gonic/gin" - "github.com/imroc/req" - "github.com/spf13/viper" - "net/http" - "strings" -) - -var FilerController ActionController - -func getFilerActions() []Action { - filerCtx := newFilerContext() - return []Action{ - { - Method: http.MethodGet, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodPost, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodPut, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodDelete, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - } -} - -type filerContext struct { - endpoint string -} - -func (ctx *filerContext) do(c *gin.Context) { - // request path - requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1) - - // request url - requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath) - if c.Request.URL.RawQuery != "" { - requestUrl += "?" + c.Request.URL.RawQuery - } - - // request body - bufR := bufio.NewScanner(c.Request.Body) - requestBody := req.BodyJSON(bufR.Bytes()) - - // request file uploads - var requestFileUploads []req.FileUpload - form, err := c.MultipartForm() - if err == nil { - for k, v := range form.File { - for _, fh := range v { - f, err := fh.Open() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - requestFileUploads = append(requestFileUploads, req.FileUpload{ - FileName: fh.Filename, - FieldName: k, - File: f, - }) - } - } - } - - // request header - requestHeader := req.Header{} - for k, v := range c.Request.Header { - if len(v) > 0 { - requestHeader[k] = v[0] - } - } - - // perform request - res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // status code check - statusCode := res.Response().StatusCode - if statusCode == http.StatusNotFound { - HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound) - return - } - - // response - for k, v := range res.Response().Header { - if len(v) > 0 { - c.Header(k, v[0]) - } - } - _, _ = c.Writer.Write(res.Bytes()) - c.AbortWithStatus(statusCode) -} - -var _filerCtx *filerContext - -func newFilerContext() *filerContext { - if _filerCtx != nil { - return _filerCtx - } - - ctx := &filerContext{ - endpoint: "http://localhost:8888", - } - - if viper.GetString("fs.filer.proxy") != "" { - ctx.endpoint = viper.GetString("fs.filer.proxy") - } - - _filerCtx = ctx - - return ctx -} diff --git a/core/controllers/filter.go b/core/controllers/filter.go deleted file mode 100644 index b609f6e0a..000000000 --- a/core/controllers/filter.go +++ /dev/null @@ -1,100 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var FilterController ActionController - -func getFilterActions() []Action { - ctx := newFilterContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:col", - HandlerFunc: ctx.getColFieldOptions, - }, - { - Method: http.MethodGet, - Path: "/:col/:value", - HandlerFunc: ctx.getColFieldOptions, - }, - { - Method: http.MethodGet, - Path: "/:col/:value/:label", - HandlerFunc: ctx.getColFieldOptions, - }, - } -} - -type filterContext struct { -} - -func (ctx *filterContext) getColFieldOptions(c *gin.Context) { - colName := c.Param("col") - value := c.Param("value") - if value == "" { - value = "_id" - } - label := c.Param("label") - if label == "" { - label = "name" - } - query := MustGetFilterQuery(c) - pipelines := mongo2.Pipeline{} - if query != nil { - pipelines = append(pipelines, bson.D{{"$match", query}}) - } - pipelines = append( - pipelines, - bson.D{ - { - "$group", - bson.M{ - "_id": bson.M{ - "value": "$" + value, - "label": "$" + label, - }, - }, - }, - }, - ) - pipelines = append( - pipelines, - bson.D{ - { - "$project", - bson.M{ - "value": "$_id.value", - "label": bson.M{"$toString": "$_id.label"}, - }, - }, - }, - ) - pipelines = append( - pipelines, - bson.D{ - { - "$sort", - bson.D{ - {"value", 1}, - }, - }, - }, - ) - var options []entity.FilterSelectOption - if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, options) -} - -func newFilterContext() *filterContext { - return &filterContext{} -} diff --git a/core/controllers/git.go b/core/controllers/git.go deleted file mode 100644 index 128323d49..000000000 --- a/core/controllers/git.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var GitController ListController diff --git a/core/controllers/init.go b/core/controllers/init.go deleted file mode 100644 index e927b9c31..000000000 --- a/core/controllers/init.go +++ /dev/null @@ -1,42 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" -) - -func InitControllers() (err error) { - modelSvc, err := service.GetService() - if err != nil { - return err - } - - NodeController = newNodeController() - ProjectController = newProjectController() - SpiderController = newSpiderController() - TaskController = newTaskController() - UserController = newUserController() - TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag)) - SettingController = newSettingController() - LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions()) - DataCollectionController = newDataCollectionController() - ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions()) - ScheduleController = newScheduleController() - StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions()) - TokenController = newTokenController() - FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions()) - GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit)) - VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions()) - SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions()) - DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions()) - RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole)) - PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission)) - ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions()) - NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions()) - FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions()) - SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions()) - DataSourceController = newDataSourceController() - EnvironmentController = newEnvironmentController() - - return nil -} diff --git a/core/controllers/login.go b/core/controllers/login.go deleted file mode 100644 index 42926fd51..000000000 --- a/core/controllers/login.go +++ /dev/null @@ -1,64 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gin-gonic/gin" - "net/http" -) - -var LoginController ActionController - -func getLoginActions() []Action { - loginCtx := newLoginContext() - return []Action{ - {Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login}, - {Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout}, - } -} - -type loginContext struct { - userSvc interfaces.UserService -} - -func (ctx *loginContext) login(c *gin.Context) { - var u models.User - if err := c.ShouldBindJSON(&u); err != nil { - HandleErrorBadRequest(c, err) - return - } - token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{ - Username: u.Username, - Password: u.Password, - }) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - c.Set(constants.UserContextKey, loggedInUser) - HandleSuccessWithData(c, token) -} - -func (ctx *loginContext) logout(c *gin.Context) { - c.Set(constants.UserContextKey, nil) - HandleSuccess(c) -} - -func newLoginContext() *loginContext { - // context - ctx := &loginContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - userSvc interfaces.UserService, - ) { - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/node.go b/core/controllers/node.go deleted file mode 100644 index 236ebafcf..000000000 --- a/core/controllers/node.go +++ /dev/null @@ -1,94 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -var NodeController *nodeController - -type nodeController struct { - ListControllerDelegate -} - -func (ctr *nodeController) Post(c *gin.Context) { - var n models.Node - if err := c.ShouldBindJSON(&n); err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := ctr._post(c, &n); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctr *nodeController) PostList(c *gin.Context) { - // bind - var docs []models.Node - if err := c.ShouldBindJSON(&docs); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // success ids - var ids []primitive.ObjectID - - // iterate nodes - for _, n := range docs { - if err := ctr._post(c, &n); err != nil { - trace.PrintError(err) - continue - } - ids = append(ids, n.Id) - } - - // success - HandleSuccessWithData(c, docs) -} - -func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) { - // set default key - if n.Key == "" { - id, err := uuid.NewUUID() - if err != nil { - return trace.TraceError(err) - } - n.Key = id.String() - } - - // set default status - if n.Status == "" { - n.Status = constants.NodeStatusUnregistered - } - - // add - if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func newNodeController() *nodeController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode)) - - return &nodeController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/notification.go b/core/controllers/notification.go deleted file mode 100644 index 31b96c048..000000000 --- a/core/controllers/notification.go +++ /dev/null @@ -1,158 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/notification" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var NotificationController ActionController - -func getNotificationActions() []Action { - ctx := newNotificationContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/settings", - HandlerFunc: ctx.GetSettingList, - }, - { - Method: http.MethodGet, - Path: "/settings/:id", - HandlerFunc: ctx.GetSetting, - }, - { - Method: http.MethodPost, - Path: "/settings", - HandlerFunc: ctx.PostSetting, - }, - { - Method: http.MethodPut, - Path: "/settings/:id", - HandlerFunc: ctx.PutSetting, - }, - { - Method: http.MethodDelete, - Path: "/settings/:id", - HandlerFunc: ctx.DeleteSetting, - }, - { - Method: http.MethodPost, - Path: "/settings/:id/enable", - HandlerFunc: ctx.EnableSetting, - }, - { - Method: http.MethodPost, - Path: "/settings/:id/disable", - HandlerFunc: ctx.DisableSetting, - }, - } -} - -type notificationContext struct { - svc *notification.Service -} - -func (ctx *notificationContext) GetSettingList(c *gin.Context) { - query := MustGetFilterQuery(c) - pagination := MustGetPagination(c) - sort := MustGetSortOption(c) - res, total, err := ctx.svc.GetSettingList(query, pagination, sort) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithListData(c, res, total) -} - -func (ctx *notificationContext) GetSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - res, err := ctx.svc.GetSetting(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, res) -} - -func (ctx *notificationContext) PostSetting(c *gin.Context) { - var s notification.NotificationSetting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.PosSetting(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) PutSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s notification.NotificationSetting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.PutSetting(id, s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) DeleteSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.DeleteSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) EnableSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.EnableSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) DisableSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.DisableSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func newNotificationContext() *notificationContext { - ctx := ¬ificationContext{ - svc: notification.GetService(), - } - return ctx -} diff --git a/core/controllers/permission.go b/core/controllers/permission.go deleted file mode 100644 index 42be45dca..000000000 --- a/core/controllers/permission.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var PermissionController ListController diff --git a/core/controllers/project.go b/core/controllers/project.go deleted file mode 100644 index fe13a9e80..000000000 --- a/core/controllers/project.go +++ /dev/null @@ -1,103 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -var ProjectController *projectController - -type projectController struct { - ListControllerDelegate -} - -func (ctr *projectController) GetList(c *gin.Context) { - // get all if query field "all" is set true - all := MustGetFilterAll(c) - if all { - ctr.getAll(c) - return - } - - // get list - list, total, err := ctr.getList(c) - if err != nil { - return - } - data := list.GetModels() - - // check empty list - if len(list.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // project ids - var ids []primitive.ObjectID - - // count cache - cache := map[primitive.ObjectID]int{} - - // iterate - for _, d := range data { - p, ok := d.(*models.Project) - if !ok { - HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType) - return - } - ids = append(ids, p.Id) - cache[p.Id] = 0 - } - - // spiders - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - spiders, err := modelSvc.GetSpiderList(bson.M{ - "project_id": bson.M{ - "$in": ids, - }, - }, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - for _, s := range spiders { - _, ok := cache[s.ProjectId] - if !ok { - HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache) - return - } - cache[s.ProjectId]++ - } - - // assign - var projects []models.Project - for _, d := range data { - p := d.(*models.Project) - p.Spiders = cache[p.Id] - projects = append(projects, *p) - } - - HandleSuccessWithListData(c, projects, total) -} - -func newProjectController() *projectController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject)) - - return &projectController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/result.go b/core/controllers/result.go deleted file mode 100644 index 0324a4177..000000000 --- a/core/controllers/result.go +++ /dev/null @@ -1,150 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var ResultController ActionController - -func getResultActions() []Action { - var resultCtx = newResultContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id", - HandlerFunc: resultCtx.getList, - }, - } -} - -type resultContext struct { - modelSvc service.ModelService -} - -func (ctx *resultContext) getList(c *gin.Context) { - // data collection id - dcId, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - var dsId primitive.ObjectID - dsIdStr := c.Query("data_source_id") - if dsIdStr != "" { - dsId, err = primitive.ObjectIDFromHex(dsIdStr) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } - - // data collection - dc, err := ctx.modelSvc.GetDataCollectionById(dcId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := ctx.modelSvc.GetDataSourceById(dsId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models.DataSource{} - } else { - HandleErrorInternalServerError(c, err) - return - } - } - - // spider - sq := bson.M{ - "col_id": dc.Id, - "data_source_id": ds.Id, - } - s, err := ctx.modelSvc.GetSpider(sq, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // service - svc, err := result.GetResultService(s.Id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // params - pagination := MustGetPagination(c) - query := ctx.getListQuery(c) - - // get results - data, err := svc.List(query, &generic.ListOptions{ - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccessWithListData(c, nil, 0) - return - } - HandleErrorInternalServerError(c, err) - return - } - - // validate results - if len(data) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // total count - total, err := svc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) { - f, err := GetFilter(c) - if err != nil { - return q - } - for _, cond := range f.Conditions { - q = append(q, generic.ListQueryCondition{ - Key: cond.Key, - Op: cond.Op, - Value: utils.NormalizeObjectId(cond.Value), - }) - } - return q -} - -func newResultContext() *resultContext { - // context - ctx := &resultContext{} - - var err error - ctx.modelSvc, err = service.NewService() - if err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/role.go b/core/controllers/role.go deleted file mode 100644 index d4a286e54..000000000 --- a/core/controllers/role.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var RoleController ListController diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 02dfb3b50..809a2f79c 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -357,13 +357,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSystemInfo, }, }) - RegisterActions(groups.AnonymousGroup, "/version", []Action{ - { - Method: http.MethodGet, - Path: "", - HandlerFunc: GetVersion, - }, - }) RegisterActions(groups.AnonymousGroup, "/", []Action{ { Method: http.MethodPost, diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go deleted file mode 100644 index 71ba2b670..000000000 --- a/core/controllers/schedule.go +++ /dev/null @@ -1,221 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var ScheduleController *scheduleController - -func getScheduleActions() []Action { - scheduleCtx := newScheduleContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/enable", - HandlerFunc: scheduleCtx.enable, - }, - { - Method: http.MethodPost, - Path: "/:id/disable", - HandlerFunc: scheduleCtx.disable, - }, - } -} - -type scheduleController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *scheduleContext -} - -func (ctr *scheduleController) Post(c *gin.Context) { - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if s.Enabled { - if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - HandleSuccessWithData(c, s) -} - -func (ctr *scheduleController) Put(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if s.GetId() != oid { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - if err := delegate.NewModelDelegate(&s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if s.Enabled { - if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - HandleSuccessWithData(c, s) -} - -func (ctr *scheduleController) Delete(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - s, err := ctr.ctx.modelSvc.GetScheduleById(oid) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { - HandleErrorInternalServerError(c, err) - return - } -} - -func (ctr *scheduleController) DeleteList(c *gin.Context) { - payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - for _, id := range payload.Ids { - s, err := ctr.ctx.modelSvc.GetScheduleById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) enable(c *gin.Context) { - s, err := ctx._getSchedule(c) - if err != nil { - return - } - if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) disable(c *gin.Context) { - s, err := ctx._getSchedule(c) - if err != nil { - return - } - if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - s, err = ctx.modelSvc.GetScheduleById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - return s, nil -} - -type scheduleContext struct { - modelSvc service.ModelService - scheduleSvc interfaces.ScheduleService -} - -func newScheduleContext() *scheduleContext { - // context - ctx := &scheduleContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - scheduleSvc interfaces.ScheduleService, - ) { - ctx.modelSvc = modelSvc - ctx.scheduleSvc = scheduleSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newScheduleController() *scheduleController { - actions := getScheduleActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) - d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) - ctx := newScheduleContext() - - return &scheduleController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/setting.go b/core/controllers/setting.go deleted file mode 100644 index 068e23796..000000000 --- a/core/controllers/setting.go +++ /dev/null @@ -1,84 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" -) - -var SettingController *settingController - -type settingController struct { - ListControllerDelegate -} - -func (ctr *settingController) Get(c *gin.Context) { - // key - key := c.Param("id") - - // model service - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // setting - s, err := modelSvc.GetSettingByKey(key, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, s) -} - -func (ctr *settingController) Put(c *gin.Context) { - // key - key := c.Param("id") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // model service - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // setting - _s, err := modelSvc.GetSettingByKey(key, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // save - _s.Value = s.Value - if err := delegate.NewModelDelegate(_s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func newSettingController() *settingController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting)) - - return &settingController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/spider.go b/core/controllers/spider.go deleted file mode 100644 index f05da3729..000000000 --- a/core/controllers/spider.go +++ /dev/null @@ -1,1333 +0,0 @@ -package controllers - -import ( - "bytes" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - fs2 "github.com/crawlab-team/crawlab/core/fs" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" - "github.com/gin-gonic/gin" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" - "math" - "net/http" - "os" - "path/filepath" - "strings" -) - -var SpiderController *spiderController - -func getSpiderActions() []Action { - ctx := newSpiderContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id/files/list", - HandlerFunc: ctx.listDir, - }, - { - Method: http.MethodGet, - Path: "/:id/files/get", - HandlerFunc: ctx.getFile, - }, - { - Method: http.MethodGet, - Path: "/:id/files/info", - HandlerFunc: ctx.getFileInfo, - }, - { - Method: http.MethodPost, - Path: "/:id/files/save", - HandlerFunc: ctx.saveFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/save/dir", - HandlerFunc: ctx.saveDir, - }, - { - Method: http.MethodPost, - Path: "/:id/files/rename", - HandlerFunc: ctx.renameFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/delete", - HandlerFunc: ctx.deleteFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/copy", - HandlerFunc: ctx.copyFile, - }, - { - Path: "/:id/files/export", - Method: http.MethodPost, - HandlerFunc: ctx.postExport, - }, - { - Method: http.MethodPost, - Path: "/:id/run", - HandlerFunc: ctx.run, - }, - { - Method: http.MethodGet, - Path: "/:id/git", - HandlerFunc: ctx.getGit, - }, - { - Method: http.MethodGet, - Path: "/:id/git/remote-refs", - HandlerFunc: ctx.getGitRemoteRefs, - }, - { - Method: http.MethodPost, - Path: "/:id/git/checkout", - HandlerFunc: ctx.gitCheckout, - }, - { - Method: http.MethodPost, - Path: "/:id/git/pull", - HandlerFunc: ctx.gitPull, - }, - { - Method: http.MethodPost, - Path: "/:id/git/commit", - HandlerFunc: ctx.gitCommit, - }, - //{ - // Method: http.MethodPost, - // Path: "/:id/clone", - // HandlerFunc: ctx.clone, - //}, - { - Path: "/:id/data-source", - Method: http.MethodGet, - HandlerFunc: ctx.getDataSource, - }, - { - Path: "/:id/data-source/:ds_id", - Method: http.MethodPost, - HandlerFunc: ctx.postDataSource, - }, - } -} - -type spiderController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *spiderContext -} - -func (ctr *spiderController) Get(c *gin.Context) { - ctr.ctx._get(c) -} - -func (ctr *spiderController) Post(c *gin.Context) { - s, err := ctr.ctx._post(c) - if err != nil { - return - } - HandleSuccessWithData(c, s) -} - -func (ctr *spiderController) Put(c *gin.Context) { - s, err := ctr.ctx._put(c) - if err != nil { - return - } - HandleSuccessWithData(c, s) -} - -func (ctr *spiderController) Delete(c *gin.Context) { - if err := ctr.ctx._delete(c); err != nil { - return - } - HandleSuccess(c) -} - -func (ctr *spiderController) GetList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - ctr.d.GetList(c) - return - } - ctr.ctx._getListWithStats(c) -} - -func (ctr *spiderController) DeleteList(c *gin.Context) { - if err := ctr.ctx._deleteList(c); err != nil { - return - } - HandleSuccess(c) -} - -type spiderContext struct { - modelSvc service.ModelService - modelSpiderSvc interfaces.ModelBaseService - modelSpiderStatSvc interfaces.ModelBaseService - modelTaskSvc interfaces.ModelBaseService - modelTaskStatSvc interfaces.ModelBaseService - adminSvc interfaces.SpiderAdminService -} - -func (ctx *spiderContext) listDir(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - files, err := fsSvc.List(payload.Path) - if err != nil { - if err.Error() != "response status code: 404" { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccessWithData(c, files) -} - -func (ctx *spiderContext) getFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - data, err := fsSvc.GetFile(payload.Path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - data = utils.TrimFileData(data) - - HandleSuccessWithData(c, string(data)) -} - -func (ctx *spiderContext) getFileInfo(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - info, err := fsSvc.GetFileInfo(payload.Path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, info) -} - -func (ctx *spiderContext) saveFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) saveDir(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.CreateDir(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) renameFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) deleteFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Delete(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) copyFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) run(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // options - var opts interfaces.SpiderRunOptions - if err := c.ShouldBindJSON(&opts); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // schedule - taskIds, err := ctx.adminSvc.Schedule(id, &opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *spiderContext) getGit(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // branches - branches, err := gitClient.GetBranches() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if branches == nil || len(branches) == 0 && currentBranch != "" { - branches = []vcs.GitRef{{Name: currentBranch}} - } - - // changes - changes, err := gitClient.GetStatus() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // logs - logs, err := gitClient.GetLogsWithRefs() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // ignore - ignore, err := ctx._getGitIgnore(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // git - _git, err := ctx.modelSvc.GetGitById(id) - if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) - return - } - } - - // response - res := bson.M{ - "current_branch": currentBranch, - "branches": branches, - "changes": changes, - "logs": logs, - "ignore": ignore, - "git": _git, - } - - HandleSuccessWithData(c, res) -} - -func (ctx *spiderContext) getGitRemoteRefs(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // remote name - remoteName := c.Query("remote") - if remoteName == "" { - remoteName = vcs.GitRemoteNameOrigin - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // refs - refs, err := gitClient.GetRemoteRefs(remoteName) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, refs) -} - -func (ctx *spiderContext) gitCheckout(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // branch to pull - var branch string - if payload.Branch == "" { - // by default current branch - branch, err = gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - // payload branch - branch = payload.Branch - } - - // checkout - if err := ctx._gitCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) gitPull(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git - g, err := ctx.modelSvc.GetGitById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // attempt to sync git - _ = ctx.adminSvc.SyncGitOne(g) - - HandleSuccess(c) -} - -func (ctx *spiderContext) gitCommit(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // add - for _, p := range payload.Paths { - if err := gitClient.Add(p); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // commit - if err := gitClient.Commit(payload.CommitMessage); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // push - if err := gitClient.Push( - vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), - ); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) getDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := ctx.modelSvc.GetDataSourceById(s.DataSourceId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, ds) -} - -func (ctx *spiderContext) postDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - if !dsId.IsZero() { - _, err = ctx.modelSvc.GetDataSourceById(dsId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // save data source id - s.DataSourceId = dsId - if err := delegate2.NewModelDelegate(s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) postExport(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // zip file path - zipFilePath, err := ctx.adminSvc.Export(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // download - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) - c.File(zipFilePath) -} - -func (ctx *spiderContext) _get(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - s, err := ctx.modelSvc.GetSpiderById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat - s.Stat, err = ctx.modelSvc.GetSpiderStatById(s.GetId()) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data collection - if !s.ColId.IsZero() { - col, err := ctx.modelSvc.GetDataCollectionById(s.ColId) - if err != nil { - if err != mongo2.ErrNoDocuments { - HandleErrorInternalServerError(c, err) - return - } - } else { - s.ColName = col.Name - } - } - - HandleSuccessWithData(c, s) -} - -func (ctx *spiderContext) _post(c *gin.Context) (s *models.Spider, err error) { - // bind - s = &models.Spider{} - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return nil, err - } - - // upsert data collection - if err := ctx._upsertDataCollection(c, s); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // add - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // add stat - st := &models.SpiderStat{ - Id: s.GetId(), - } - if err := delegate2.NewModelDelegate(st, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - return s, nil -} - -func (ctx *spiderContext) _put(c *gin.Context) (s *models.Spider, err error) { - // bind - s = &models.Spider{} - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return nil, err - } - - // upsert data collection - if err := ctx._upsertDataCollection(c, s); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // save - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - return s, nil -} - -func (ctx *spiderContext) _delete(c *gin.Context) (err error) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete spider - s, err := ctx.modelSvc.GetSpiderById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // delete spider stat - ss, err := ctx.modelSvc.GetSpiderStatById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(ss, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // related tasks - tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": oid}, nil) - if err != nil { - return err - } - - // task ids - var taskIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - } - - // delete related tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - // delete related task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *spiderContext) _getListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - l, err := ctx.modelSpiderSvc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // check empty list - if len(l.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // ids - var ids []primitive.ObjectID - for _, d := range l.GetModels() { - s := d.(*models.Spider) - ids = append(ids, s.GetId()) - } - - // total count - total, err := ctx.modelSpiderSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat list - query = bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - stats, err := ctx.modelSvc.GetSpiderStatList(query, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache stat list to dict - dict := map[primitive.ObjectID]models.SpiderStat{} - var tids []primitive.ObjectID - for _, st := range stats { - if st.Tasks > 0 { - taskCount := int64(st.Tasks) - st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount))) - st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount))) - st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount))) - } - dict[st.GetId()] = st - - if !st.LastTaskId.IsZero() { - tids = append(tids, st.LastTaskId) - } - } - - // task list and stats - var tasks []models.Task - dictTask := map[primitive.ObjectID]models.Task{} - dictTaskStat := map[primitive.ObjectID]models.TaskStat{} - if len(tids) > 0 { - // task list - queryTask := bson.M{ - "_id": bson.M{ - "$in": tids, - }, - } - tasks, err = ctx.modelSvc.GetTaskList(queryTask, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // task stats list - taskStats, err := ctx.modelSvc.GetTaskStatList(queryTask, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache task stats to dict - for _, st := range taskStats { - dictTaskStat[st.GetId()] = st - } - - // cache task list to dict - for _, t := range tasks { - st, ok := dictTaskStat[t.GetId()] - if ok { - t.Stat = &st - } - dictTask[t.GetSpiderId()] = t - } - } - - // iterate list again - var data []interface{} - for _, d := range l.GetModels() { - s := d.(*models.Spider) - - // spider stat - st, ok := dict[s.GetId()] - if ok { - s.Stat = &st - - // last task - t, ok := dictTask[s.GetId()] - if ok { - s.Stat.LastTask = &t - } - } - - // add to list - data = append(data, *s) - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *spiderContext) _deleteList(c *gin.Context) (err error) { - payload, err := NewJsonBinder(ControllerIdSpider).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete spiders - if err := ctx.modelSpiderSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // delete spider stats - if err := ctx.modelSpiderStatSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // related tasks - tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) - if err != nil { - return err - } - - // task ids - var taskIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - } - - // delete related tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - // delete related task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *spiderContext) _processFileRequest(c *gin.Context, method string) (id primitive.ObjectID, payload entity.FileRequestPayload, fsSvc interfaces.FsServiceV2, err error) { - // id - id, err = primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - contentType := c.GetHeader("Content-Type") - if strings.HasPrefix(contentType, "multipart/form-data") { - // multipart/form-data - payload, err = ctx._getFileRequestMultipartPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } else { - // query or application/json - switch method { - case http.MethodGet: - err = c.ShouldBindQuery(&payload) - default: - err = c.ShouldBindJSON(&payload) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // fs service - workspacePath := viper.GetString("workspace") - fsSvc = fs2.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) - - return -} - -func (ctx *spiderContext) _getFileRequestMultipartPayload(c *gin.Context) (payload entity.FileRequestPayload, err error) { - fh, err := c.FormFile("file") - if err != nil { - return - } - f, err := fh.Open() - if err != nil { - return - } - buf := bytes.NewBuffer(nil) - if _, err = io.Copy(buf, f); err != nil { - return - } - payload.Path = c.PostForm("path") - payload.Data = buf.String() - return -} - -func (ctx *spiderContext) _processActionRequest(c *gin.Context) (id primitive.ObjectID, err error) { - // id - id, err = primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - return -} - -func (ctx *spiderContext) _upsertDataCollection(c *gin.Context, s *models.Spider) (err error) { - if s.ColId.IsZero() { - // validate - if s.ColName == "" { - return trace.TraceError(errors.ErrorControllerMissingRequestFields) - } - // no id - dc, err := ctx.modelSvc.GetDataCollectionByName(s.ColName, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - // not exists, add new - dc = &models.DataCollection{Name: s.ColName} - if err := delegate2.NewModelDelegate(dc, GetUserFromContext(c)).Add(); err != nil { - return err - } - } else { - // error - return err - } - } - s.ColId = dc.Id - - // create index - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) - } else { - // with id - dc, err := ctx.modelSvc.GetDataCollectionById(s.ColId) - if err != nil { - return err - } - s.ColId = dc.Id - } - return nil -} - -func (ctx *spiderContext) _getGitIgnore(id primitive.ObjectID) (ignore []string, err error) { - workspacePath := viper.GetString("workspace") - filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") - if !utils.Exists(filePath) { - return nil, nil - } - data, err := os.ReadFile(filePath) - if err != nil { - return nil, trace.TraceError(err) - } - ignore = strings.Split(string(data), "\n") - return ignore, nil -} - -func (ctx *spiderContext) _gitCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { - if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { - return trace.TraceError(err) - } - - // pull - return ctx._gitPull(gitClient, remote, branch) -} - -func (ctx *spiderContext) _gitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { - // pull - if err := gitClient.Pull( - vcs.WithRemoteNamePull(remote), - vcs.WithBranchNamePull(branch), - ); err != nil { - return trace.TraceError(err) - } - - // reset - if err := gitClient.Reset(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func (ctx *spiderContext) _getGitClient(id primitive.ObjectID) (gitClient *vcs.GitClient, err error) { - // git - g, err := ctx.modelSvc.GetGitById(id) - if err != nil { - if err != mongo2.ErrNoDocuments { - return nil, trace.TraceError(err) - } - return nil, nil - } - - // git client - workspacePath := viper.GetString("workspace") - gitClient, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) - if err != nil { - return nil, err - } - - // set auth - utils.InitGitClientAuth(g, gitClient) - - // remote name - remoteName := vcs.GitRemoteNameOrigin - - // update remote - r, err := gitClient.GetRemote(remoteName) - if err == git.ErrRemoteNotFound { - // remote not exists, create - if _, err := gitClient.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } else if err == nil { - // remote exists, update if different - if g.Url != r.Config().URLs[0] { - if err := gitClient.DeleteRemote(remoteName); err != nil { - return nil, trace.TraceError(err) - } - if _, err := gitClient.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } - gitClient.SetRemoteUrl(g.Url) - } else { - // error - return nil, trace.TraceError(err) - } - - // check if head reference exists - _, err = gitClient.GetRepository().Head() - if err == nil { - return gitClient, nil - } - - // align master/main branch - ctx._alignBranch(gitClient) - - return gitClient, nil -} - -func (ctx *spiderContext) _alignBranch(gitClient *vcs.GitClient) { - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - trace.PrintError(err) - return - } - - // skip if current branch is not master - if currentBranch != vcs.GitBranchNameMaster { - return - } - - // remote refs - refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) - if err != nil { - trace.PrintError(err) - return - } - - // main branch - defaultRemoteBranch, err := ctx._getDefaultRemoteBranch(refs) - if err != nil || defaultRemoteBranch == "" { - return - } - - // move branch - if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { - trace.PrintError(err) - } -} - -func (ctx *spiderContext) _getDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { - // remote branch name - for _, r := range refs { - if r.Type != vcs.GitRefTypeBranch { - continue - } - - if r.Name == vcs.GitBranchNameMain { - defaultRemoteBranchName = r.Name - break - } - - if r.Name == vcs.GitBranchNameMaster { - defaultRemoteBranchName = r.Name - continue - } - - if defaultRemoteBranchName == "" { - defaultRemoteBranchName = r.Name - continue - } - } - - return defaultRemoteBranchName, nil -} - -var _spiderCtx *spiderContext - -func newSpiderContext() *spiderContext { - if _spiderCtx != nil { - return _spiderCtx - } - - // context - ctx := &spiderContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - ) { - ctx.modelSvc = modelSvc - ctx.adminSvc = adminSvc - }); err != nil { - panic(err) - } - - // model spider service - ctx.modelSpiderSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpider) - - // model spider stat service - ctx.modelSpiderStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpiderStat) - - // model task service - ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) - - // model task stat service - ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) - - _spiderCtx = ctx - - return ctx -} - -func newSpiderController() *spiderController { - actions := getSpiderActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) - d := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) - ctx := newSpiderContext() - - return &spiderController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/stats.go b/core/controllers/stats.go deleted file mode 100644 index 3d474956c..000000000 --- a/core/controllers/stats.go +++ /dev/null @@ -1,87 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "time" -) - -var StatsController ActionController - -func getStatsActions() []Action { - statsCtx := newStatsContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/overview", - HandlerFunc: statsCtx.getOverview, - }, - { - Method: http.MethodGet, - Path: "/daily", - HandlerFunc: statsCtx.getDaily, - }, - { - Method: http.MethodGet, - Path: "/tasks", - HandlerFunc: statsCtx.getTasks, - }, - } -} - -type statsContext struct { - statsSvc interfaces.StatsService - defaultQuery bson.M -} - -func (svc *statsContext) getOverview(c *gin.Context) { - data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func (svc *statsContext) getDaily(c *gin.Context) { - data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func (svc *statsContext) getTasks(c *gin.Context) { - data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func newStatsContext() *statsContext { - // context - ctx := &statsContext{ - defaultQuery: bson.M{ - "create_ts": bson.M{ - "$gte": time.Now().Add(-30 * 24 * time.Hour), - }, - }, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - statsSvc interfaces.StatsService, - ) { - ctx.statsSvc = statsSvc - }); err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/sync.go b/core/controllers/sync.go deleted file mode 100644 index 5f9874c8f..000000000 --- a/core/controllers/sync.go +++ /dev/null @@ -1,58 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/utils" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" - "path/filepath" -) - -var SyncController ActionController - -func getSyncActions() []Action { - var ctx = newSyncContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id/scan", - HandlerFunc: ctx.scan, - }, - { - Method: http.MethodGet, - Path: "/:id/download", - HandlerFunc: ctx.download, - }, - } -} - -type syncContext struct { -} - -func (ctx *syncContext) scan(c *gin.Context) { - id := c.Param("id") - path := c.Query("path") - dir := ctx._getDir(id, path) - files, err := utils.ScanDirectory(dir) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - c.AbortWithStatusJSON(http.StatusOK, files) -} - -func (ctx *syncContext) download(c *gin.Context) { - id := c.Param("id") - filePath := c.Query("path") - dir := ctx._getDir(id, "") - c.File(filepath.Join(dir, filePath)) -} - -func (ctx *syncContext) _getDir(id string, path string) string { - workspacePath := viper.GetString("workspace") - return filepath.Join(workspacePath, id, path) -} - -func newSyncContext() syncContext { - return syncContext{} -} diff --git a/core/controllers/system_info.go b/core/controllers/system_info.go deleted file mode 100644 index a89edc15d..000000000 --- a/core/controllers/system_info.go +++ /dev/null @@ -1,28 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" -) - -func getSystemInfo(c *gin.Context) { - info := &entity.SystemInfo{ - Edition: viper.GetString("info.edition"), - Version: viper.GetString("info.version"), - } - HandleSuccessWithData(c, info) -} - -func getSystemInfoActions() []Action { - return []Action{ - { - Path: "", - Method: http.MethodGet, - HandlerFunc: getSystemInfo, - }, - } -} - -var SystemInfoController ActionController diff --git a/core/controllers/system_info_v2.go b/core/controllers/system_info_v2.go index fc01ea6f0..9b4fad837 100644 --- a/core/controllers/system_info_v2.go +++ b/core/controllers/system_info_v2.go @@ -8,8 +8,8 @@ import ( func GetSystemInfo(c *gin.Context) { info := &entity.SystemInfo{ - Edition: viper.GetString("info.edition"), - Version: viper.GetString("info.version"), + Edition: viper.GetString("edition"), + Version: viper.GetString("version"), } HandleSuccessWithData(c, info) } diff --git a/core/controllers/tag.go b/core/controllers/tag.go deleted file mode 100644 index 3f74abf24..000000000 --- a/core/controllers/tag.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var TagController ListController diff --git a/core/controllers/task.go b/core/controllers/task.go deleted file mode 100644 index 9a66b9ef4..000000000 --- a/core/controllers/task.go +++ /dev/null @@ -1,534 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/task/log" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" - "strings" -) - -var TaskController *taskController - -func getTaskActions() []Action { - taskCtx := newTaskContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/run", - HandlerFunc: taskCtx.run, - }, - { - Method: http.MethodPost, - Path: "/:id/restart", - HandlerFunc: taskCtx.restart, - }, - { - Method: http.MethodPost, - Path: "/:id/cancel", - HandlerFunc: taskCtx.cancel, - }, - { - Method: http.MethodGet, - Path: "/:id/logs", - HandlerFunc: taskCtx.getLogs, - }, - { - Method: http.MethodGet, - Path: "/:id/data", - HandlerFunc: taskCtx.getData, - }, - } -} - -type taskController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *taskContext -} - -func (ctr *taskController) Get(c *gin.Context) { - ctr.ctx.getWithStatsSpider(c) -} - -func (ctr *taskController) Delete(c *gin.Context) { - if err := ctr.ctx._delete(c); err != nil { - return - } - HandleSuccess(c) -} - -func (ctr *taskController) GetList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - ctr.d.GetList(c) - return - } - ctr.ctx.getListWithStats(c) -} - -func (ctr *taskController) DeleteList(c *gin.Context) { - if err := ctr.ctx._deleteList(c); err != nil { - return - } - HandleSuccess(c) -} - -type taskContext struct { - modelSvc service.ModelService - modelTaskSvc interfaces.ModelBaseService - modelTaskStatSvc interfaces.ModelBaseService - adminSvc interfaces.SpiderAdminService - schedulerSvc interfaces.TaskSchedulerService - l log.Driver -} - -func (ctx *taskContext) run(c *gin.Context) { - // task - var t models.Task - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // validate spider id - if t.GetSpiderId().IsZero() { - HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId()) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // run - taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *taskContext) restart(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // run - taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *taskContext) cancel(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // validate - if !utils.IsCancellable(t.Status) { - HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable) - return - } - - // cancel - if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *taskContext) getLogs(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // logs - logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) - if err != nil { - if strings.HasSuffix(err.Error(), "Status:404 Not Found") { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - total, err := ctx.l.Count(id.Hex(), "") - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, logs, total) -} - -func (ctx *taskContext) getListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // check empty list - if len(list.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // ids - var ids []primitive.ObjectID - for _, d := range list.GetModels() { - t := d.(interfaces.Model) - ids = append(ids, t.GetId()) - } - - // total count - total, err := ctx.modelTaskSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat list - query = bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - stats, err := ctx.modelSvc.GetTaskStatList(query, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache stat list to dict - dict := map[primitive.ObjectID]models.TaskStat{} - for _, s := range stats { - dict[s.GetId()] = s - } - - // iterate list again - var data []interface{} - for _, d := range list.GetModels() { - t := d.(*models.Task) - s, ok := dict[t.GetId()] - if ok { - t.Stat = &s - } - data = append(data, *t) - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *taskContext) getWithStatsSpider(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // spider - t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId) - - // skip if task status is pending - if t.Status == constants.TaskStatusPending { - HandleSuccessWithData(c, t) - return - } - - // task stat - t.Stat, _ = ctx.modelSvc.GetTaskStatById(id) - - HandleSuccessWithData(c, t) -} - -func (ctx *taskContext) getData(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // result service - resultSvc, err := result.GetResultService(t.SpiderId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // query - query := generic.ListQuery{ - generic.ListQueryCondition{ - Key: constants.TaskKey, - Op: generic.OpEqual, - Value: t.Id, - }, - } - - // list - data, err := resultSvc.List(query, &generic.ListOptions{ - Skip: (p.Page - 1) * p.Size, - Limit: p.Size, - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - }) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // total - total, err := resultSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, data, total) -} - -func (ctx *taskContext) _delete(c *gin.Context) (err error) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete task - task, err := ctx.modelSvc.GetTaskById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // delete task stat - taskStat, err := ctx.modelSvc.GetTaskStatById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *taskContext) _deleteList(c *gin.Context) (err error) { - payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { - // delete tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // delete task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func newTaskContext() *taskContext { - // context - ctx := &taskContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - schedulerSvc interfaces.TaskSchedulerService, - ) { - ctx.modelSvc = modelSvc - ctx.adminSvc = adminSvc - ctx.schedulerSvc = schedulerSvc - }); err != nil { - panic(err) - } - - // model task service - ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) - - // model task stat service - ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) - - // log driver - l, err := log.GetLogDriver(log.DriverTypeFile) - if err != nil { - panic(err) - } - ctx.l = l - - return ctx -} - -func newTaskController() *taskController { - actions := getTaskActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) - d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) - ctx := newTaskContext() - - return &taskController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/test/base.go b/core/controllers/test/base.go deleted file mode 100644 index 1e3c2906a..000000000 --- a/core/controllers/test/base.go +++ /dev/null @@ -1,104 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/routes" - "github.com/crawlab-team/crawlab/trace" - "github.com/gavv/httpexpect/v2" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "go.uber.org/dig" - "net/http/httptest" - "testing" - "time" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -type Test struct { - // dependencies - modelSvc service.ModelService - - // internals - app *gin.Engine - svr *httptest.Server - - // test data - TestUsername string - TestPassword string - TestToken string -} - -func (t *Test) Setup(t2 *testing.T) { - //if err := controllers.InitControllers(); err != nil { - // panic(err) - //} - //t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.modelSvc.DropAll() - time.Sleep(200 * time.Millisecond) -} - -func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) { - e = httpexpect.New(t2, t.svr.URL) - res := e.POST("/login").WithJSON(map[string]string{ - "username": t.TestUsername, - "password": t.TestPassword, - }).Expect().JSON().Object() - t.TestToken = res.Path("$.data").String().Raw() - require.NotEmpty(t2, t.TestToken) - return e -} - -func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request { - return req.WithHeader("Authorization", t.TestToken) -} - -var T *Test - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // gin app - t.app = gin.New() - - // http test server - t.svr = httptest.NewServer(t.app) - - // init controllers - if err := controllers.InitControllers(); err != nil { - return nil, err - } - - // init routes - if err := routes.InitRoutes(t.app); err != nil { - return nil, err - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.NewService); err != nil { - return nil, trace.TraceError(err) - } - if err := c.Invoke(func(modelSvc service.ModelService) { - t.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // test data - t.TestUsername = "admin" - t.TestPassword = "admin" - - return t, nil -} diff --git a/core/controllers/test/export_test.go b/core/controllers/test/export_test.go deleted file mode 100644 index 7623feb60..000000000 --- a/core/controllers/test/export_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "testing" - "time" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") -} - -func TestExportController_Csv(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // mongo collection - colName := "test_collection_for_export" - col := mongo.GetMongoCol(colName) - - // insert test data to mongo collection - for i := 0; i < 10; i++ { - _, err := col.Insert(bson.M{ - "field1": i + 1, - "field2": i + 2, - "field3": i + 3, - "field4": i + 4, - }) - require.Nil(t, err) - } - - // export from mongo collection - res := T.WithAuth(e.POST("/export/csv")). - WithQuery("target", colName). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - - // export id - exportId := res.Path("$.data").String().Raw() - - // poll export with export id - for i := 0; i < 10; i++ { - res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object() - status := res.Path("$.data.status").String().Raw() - if status == constants.TaskStatusFinished { - break - } - time.Sleep(1 * time.Second) - } - - // download exported csv file - csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body() - csvFileBody.NotEmpty() -} diff --git a/core/controllers/test/filter_test.go b/core/controllers/test/filter_test.go deleted file mode 100644 index e7579e0fe..000000000 --- a/core/controllers/test/filter_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") -} - -func TestFilterController_GetColFieldOptions(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // mongo collection - colName := "test_collection_for_filter" - field1 := "field1" - field2 := "field2" - value1 := "value1" - col := mongo.GetMongoCol(colName) - n := 10 - var ids []primitive.ObjectID - var names []string - for i := 0; i < n; i++ { - _id := primitive.NewObjectID() - ids = append(ids, _id) - name := fmt.Sprintf("name_%d", i) - names = append(names, name) - _, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id}) - require.Nil(t, err) - } - - // validate filter options field 1 - res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(1) - res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1) - - // validate filter options field 2 - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(2) - - // validate filter options with query - conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}} - conditionsJson, err := json.Marshal(conditions) - conditionsJsonStr := string(conditionsJson) - require.Nil(t, err) - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). - WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(1) - - // validate filter options (basic path) - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(n) - for i := 0; i < n; i++ { - res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i]) - res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i]) - } -} diff --git a/core/controllers/test/main_test.go b/core/controllers/test/main_test.go deleted file mode 100644 index 2b9bb76bf..000000000 --- a/core/controllers/test/main_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/user" - "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestMain(m *testing.M) { - // init user - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - _, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) - if err != nil { - if err.Error() != mongo.ErrNoDocuments.Error() { - panic(err) - } - userSvc, err := user.GetUserService() - if err != nil { - panic(err) - } - if err := userSvc.Create(&interfaces.UserCreateOptions{ - Username: constants.DefaultAdminUsername, - Password: constants.DefaultAdminPassword, - Role: constants.RoleAdmin, - }); err != nil { - panic(err) - } - } - - if err := controllers.InitControllers(); err != nil { - panic(err) - } - - m.Run() - - T.Cleanup() -} diff --git a/core/controllers/test/project_test.go b/core/controllers/test/project_test.go deleted file mode 100644 index b0a1b2c68..000000000 --- a/core/controllers/test/project_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" - "time" -) - -func TestProjectController_Get(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - } - res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - res.Path("$.data.name").Equal("test project") -} - -func TestProjectController_Put(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "old name", - Description: "old description", - } - - // add - res := T.WithAuth(e.POST("/projects")). - WithJSON(p). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // change object - p.Id = oid - p.Name = "new name" - p.Description = "new description" - - // update - T.WithAuth(e.PUT("/projects/" + id)). - WithJSON(p). - Expect().Status(http.StatusOK) - - // check - res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").Equal(id) - res.Path("$.data.name").Equal("new name") - res.Path("$.data.description").Equal("new description") -} - -func TestProjectController_Post(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - Description: "this is a test project", - } - - res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - res.Path("$.data.name").Equal("test project") - res.Path("$.data.description").Equal("this is a test project") -} - -func TestProjectController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - Description: "this is a test project", - } - - // add - res := T.WithAuth(e.POST("/projects")). - WithJSON(p). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // get - res = T.WithAuth(e.GET("/projects/" + id)). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id = res.Path("$.data._id").String().Raw() - oid, err = primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // delete - T.WithAuth(e.DELETE("/projects/" + id)). - Expect().Status(http.StatusOK). - JSON().Object() - - // get - T.WithAuth(e.GET("/projects/" + id)). - Expect().Status(http.StatusNotFound) -} - -func TestProjectController_GetList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 100 // total - bn := 10 // batch - - for i := 0; i < n; i++ { - p := models.Project{ - Name: fmt.Sprintf("test name %d", i+1), - } - obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - obj.Path("$.data._id").NotNull() - } - - f := entity.Filter{ - //IsOr: false, - Conditions: []*entity.Condition{ - {Key: "name", Op: constants.FilterOpContains, Value: "test name"}, - }, - } - condBytes, err := json.Marshal(&f.Conditions) - require.Nil(t, err) - - pagination := entity.Pagination{ - Page: 1, - Size: bn, - } - - // get list with pagination - res := T.WithAuth(e.GET("/projects")). - WithQuery("conditions", string(condBytes)). - WithQueryObject(pagination). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").Array().Length().Equal(bn) - res.Path("$.total").Number().Equal(n) - - data := res.Path("$.data").Array() - for i := 0; i < bn; i++ { - obj := data.Element(i) - obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1)) - } - -} - -func TestProjectController_PostList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: fmt.Sprintf("project %d", i+1), - Description: "this is a project", - }) - } - - T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK) - - res := T.WithAuth(e.GET("/projects")). - WithQueryObject(entity.Pagination{Page: 1, Size: 10}). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data").Array().Length().Equal(n) -} - -func TestProjectController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: fmt.Sprintf("project %d", i+1), - Description: "this is a project", - }) - } - - // add - res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). - JSON().Object() - var ids []primitive.ObjectID - data := res.Path("$.data").Array() - for i := 0; i < n; i++ { - obj := data.Element(i) - id := obj.Path("$._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - ids = append(ids, oid) - } - - // delete - payload := entity.BatchRequestPayload{ - Ids: ids, - } - T.WithAuth(e.DELETE("/projects")). - WithJSON(payload). - Expect().Status(http.StatusOK) - - // check - for _, id := range ids { - T.WithAuth(e.GET("/projects/" + id.Hex())). - Expect().Status(http.StatusNotFound) - } - -} - -func TestProjectController_PutList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // now - now := time.Now() - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: "old name", - Description: "old description", - }) - } - - // add - res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). - JSON().Object() - var ids []primitive.ObjectID - data := res.Path("$.data").Array() - for i := 0; i < n; i++ { - obj := data.Element(i) - id := obj.Path("$._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - ids = append(ids, oid) - } - - // wait for 100 millisecond - time.Sleep(100 * time.Millisecond) - - // update - p := models.Project{ - Name: "new name", - Description: "new description", - } - dataBytes, err := json.Marshal(&p) - require.Nil(t, err) - payload := entity.BatchRequestPayloadWithStringData{ - Ids: ids, - Data: string(dataBytes), - Fields: []string{ - "name", - "description", - }, - } - T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK) - - // check response data - for i := 0; i < n; i++ { - res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data.name").Equal("new name") - res.Path("$.data.description").Equal("new description") - } - - // check artifacts - pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil) - require.Nil(t, err) - for _, p := range pl { - a, err := delegate.NewModelDelegate(&p).GetArtifact() - require.Nil(t, err) - require.True(t, a.GetSys().GetUpdateTs().After(now)) - require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs())) - } -} diff --git a/core/controllers/test/spider_test.go b/core/controllers/test/spider_test.go deleted file mode 100644 index b8c4dc06b..000000000 --- a/core/controllers/test/spider_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func TestSpiderController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - s := models.Spider{ - Name: "test spider", - Description: "this is a test spider", - ColName: "test col name", - } - - // add spider - res := T.WithAuth(e.POST("/spiders")). - WithJSON(s). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // add tasks - var taskIds []primitive.ObjectID - tasks := []models.Task{ - { - Id: primitive.NewObjectID(), - SpiderId: oid, - }, - { - Id: primitive.NewObjectID(), - SpiderId: oid, - }, - } - for _, task := range tasks { - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - taskIds = append(taskIds, task.Id) - } - - // delete - T.WithAuth(e.DELETE("/spiders/" + id)). - Expect().Status(http.StatusOK) - - // get - T.WithAuth(e.GET("/spiders/" + id)). - Expect().Status(http.StatusNotFound) - - // get tasks - for _, task := range tasks { - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // spider stat - modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) - spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ - "_id": oid, - }) - require.Nil(t, err) - require.Zero(t, spiderStatCount) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} - -func TestSpiderController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - spiders := []models.Spider{ - { - Id: primitive.NewObjectID(), - Name: "test spider 1", - Description: "this is a test spider 1", - ColName: "test col name 1", - }, - { - Id: primitive.NewObjectID(), - Name: "test spider 2", - Description: "this is a test spider 2", - ColName: "test col name 2", - }, - } - - // add spiders - for _, spider := range spiders { - T.WithAuth(e.POST("/spiders")). - WithJSON(spider). - Expect().Status(http.StatusOK) - } - - var spiderIds []primitive.ObjectID - var taskIds []primitive.ObjectID - for _, spider := range spiders { - // task id - taskId := primitive.NewObjectID() - - // add task - err := delegate.NewModelDelegate(&models.Task{ - Id: taskId, - SpiderId: spider.Id, - }).Add() - require.Nil(t, err) - - // add task stats - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: taskId, - }).Add() - require.Nil(t, err) - - spiderIds = append(spiderIds, spider.Id) - taskIds = append(taskIds, taskId) - } - - // delete spiders - T.WithAuth(e.DELETE("/spiders")). - WithJSON(entity.BatchRequestPayload{ - Ids: spiderIds, - }).Expect().Status(http.StatusOK) - - // get spiders - for _, spider := range spiders { - // get - T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // get tasks - for _, taskId := range taskIds { - T.WithAuth(e.GET("/tasks/" + taskId.Hex())). - Expect().Status(http.StatusNotFound) - } - - // spider stat - modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) - spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - }) - require.Nil(t, err) - require.Zero(t, spiderStatCount) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} diff --git a/core/controllers/test/task_test.go b/core/controllers/test/task_test.go deleted file mode 100644 index 4d81738b8..000000000 --- a/core/controllers/test/task_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func TestTaskController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - task := models.Task{ - Id: primitive.NewObjectID(), - } - - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - // delete - T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusOK) - - // get - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": task.Id, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} - -func TestTaskController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - tasks := []models.Task{ - { - Id: primitive.NewObjectID(), - }, - { - Id: primitive.NewObjectID(), - }, - } - - // add spiders - var taskIds []primitive.ObjectID - for _, task := range tasks { - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - taskIds = append(taskIds, task.Id) - } - - // delete tasks - T.WithAuth(e.DELETE("/tasks")). - WithJSON(entity.BatchRequestPayload{ - Ids: taskIds, - }).Expect().Status(http.StatusOK) - - // get tasks - for _, task := range tasks { - // get - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} diff --git a/core/controllers/token.go b/core/controllers/token.go deleted file mode 100644 index f94f3b688..000000000 --- a/core/controllers/token.go +++ /dev/null @@ -1,84 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" -) - -var TokenController *tokenController - -var TokenActions []Action - -type tokenController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *tokenContext -} - -func (ctr *tokenController) Post(c *gin.Context) { - var err error - var t models.Token - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } - u, err := ctr.ctx.userSvc.GetCurrentUser(c) - if err != nil { - HandleErrorUnauthorized(c, err) - return - } - t.Token, err = ctr.ctx.userSvc.MakeToken(u) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -type tokenContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func newTokenContext() *tokenContext { - // context - ctx := &tokenContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newTokenController() *tokenController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) - d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) - ctx := newTokenContext() - - return &tokenController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/user.go b/core/controllers/user.go deleted file mode 100644 index 5c22a7293..000000000 --- a/core/controllers/user.go +++ /dev/null @@ -1,244 +0,0 @@ -package controllers - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var UserController *userController - -func getUserActions() []Action { - userCtx := newUserContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/change-password", - HandlerFunc: userCtx.changePassword, - }, - { - Method: http.MethodGet, - Path: "/me", - HandlerFunc: userCtx.getMe, - }, - { - Method: http.MethodPut, - Path: "/me", - HandlerFunc: userCtx.putMe, - }, - } -} - -type userController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *userContext -} - -func (ctr *userController) Post(c *gin.Context) { - var u models.User - if err := c.ShouldBindJSON(&u); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ - Username: u.Username, - Password: u.Password, - Email: u.Email, - Role: u.Role, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctr *userController) PostList(c *gin.Context) { - // users - var users []models.User - if err := c.ShouldBindJSON(&users); err != nil { - HandleErrorBadRequest(c, err) - return - } - - for _, u := range users { - if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ - Username: u.Username, - Password: u.Password, - Email: u.Email, - Role: u.Role, - }); err != nil { - trace.PrintError(err) - } - } - - HandleSuccess(c) -} - -func (ctr *userController) PutList(c *gin.Context) { - // payload - var payload entity.BatchRequestPayloadWithStringData - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // doc to update - var doc models.User - if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // query - query := bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - } - - // update users - if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // update passwords - if utils.Contains(payload.Fields, "password") { - for _, id := range payload.Ids { - if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil { - trace.PrintError(err) - } - } - } - - HandleSuccess(c) -} - -type userContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func (ctx *userContext) changePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload map[string]string - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - password, ok := payload["password"] - if !ok { - HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields) - return - } - if len(password) < 5 { - HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword) - return - } - if err := ctx.userSvc.ChangePassword(id, password); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *userContext) getMe(c *gin.Context) { - u, err := ctx._getMe(c) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - HandleSuccessWithData(c, u) -} - -func (ctx *userContext) putMe(c *gin.Context) { - // current user - u, err := ctx._getMe(c) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - - // payload - doc, err := NewJsonBinder(ControllerIdUser).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if doc.GetId() != u.GetId() { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - - // save to db - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, doc) -} - -func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) { - res, ok := c.Get(constants.UserContextKey) - if !ok { - return nil, trace.TraceError(errors.ErrorUserNotExistsInContext) - } - u, ok = res.(interfaces.User) - if !ok { - return nil, trace.TraceError(errors.ErrorUserInvalidType) - } - return u, nil -} - -func newUserContext() *userContext { - // context - ctx := &userContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newUserController() *userController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) - d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) - ctx := newUserContext() - - return &userController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/version.go b/core/controllers/version.go deleted file mode 100644 index 83e3fe951..000000000 --- a/core/controllers/version.go +++ /dev/null @@ -1,23 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/config" - "github.com/gin-gonic/gin" - "net/http" -) - -func GetVersion(c *gin.Context) { - HandleSuccessWithData(c, config.GetVersion()) -} - -func getVersionActions() []Action { - return []Action{ - { - Method: http.MethodGet, - Path: "", - HandlerFunc: GetVersion, - }, - } -} - -var VersionController ActionController diff --git a/core/routes/group.go b/core/routes/group.go deleted file mode 100644 index cade1af40..000000000 --- a/core/routes/group.go +++ /dev/null @@ -1,20 +0,0 @@ -package routes - -import ( - "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/gin-gonic/gin" -) - -type RouterGroups struct { - AuthGroup *gin.RouterGroup - AnonymousGroup *gin.RouterGroup - FilerGroup *gin.RouterGroup -} - -func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { - return &RouterGroups{ - AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), - AnonymousGroup: app.Group("/"), - FilerGroup: app.Group("/filer", middlewares.FilerAuthorizationMiddleware()), - } -} diff --git a/core/routes/router.go b/core/routes/router.go deleted file mode 100644 index c82603bcd..000000000 --- a/core/routes/router.go +++ /dev/null @@ -1,178 +0,0 @@ -package routes - -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/gin-gonic/gin" - "net/http" - "path" -) - -type RouterServiceInterface interface { - RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) - RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) -} - -type RouterService struct { - app *gin.Engine -} - -func NewRouterService(app *gin.Engine) (svc *RouterService) { - return &RouterService{ - app: app, - } -} - -func (svc *RouterService) RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.BasicController) { - group.GET(basePath, ctr.Get) - group.POST(basePath, ctr.Post) - group.PUT(basePath, ctr.Put) - group.DELETE(basePath, ctr.Delete) -} - -func (svc *RouterService) RegisterListControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) { - group.GET(basePath+"/:id", ctr.Get) - group.GET(basePath, ctr.GetList) - group.POST(basePath, ctr.Post) - group.POST(basePath+"/batch", ctr.PostList) - group.PUT(basePath+"/:id", ctr.Put) - group.PUT(basePath, ctr.PutList) - group.DELETE(basePath+"/:id", ctr.Delete) - group.DELETE(basePath, ctr.DeleteList) -} - -func (svc *RouterService) RegisterActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ActionController) { - for _, action := range ctr.Actions() { - routerPath := path.Join(basePath, action.Path) - switch action.Method { - case http.MethodGet: - group.GET(routerPath, action.HandlerFunc) - case http.MethodPost: - group.POST(routerPath, action.HandlerFunc) - case http.MethodPut: - group.PUT(routerPath, action.HandlerFunc) - case http.MethodDelete: - group.DELETE(routerPath, action.HandlerFunc) - } - } -} - -func (svc *RouterService) RegisterListActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListActionController) { - svc.RegisterListControllerToGroup(group, basePath, ctr) - svc.RegisterActionControllerToGroup(group, basePath, ctr) -} - -func (svc *RouterService) RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) { - switch method { - case http.MethodGet: - group.GET(path, handler) - case http.MethodPost: - group.POST(path, handler) - case http.MethodPut: - group.PUT(path, handler) - case http.MethodDelete: - group.DELETE(path, handler) - default: - log.Warn(fmt.Sprintf("%s is not a valid http method", method)) - } -} - -func InitRoutes(app *gin.Engine) (err error) { - // routes groups - groups := NewRouterGroups(app) - - // router service - svc := NewRouterService(app) - - // register routes - registerRoutesAnonymousGroup(svc, groups) - registerRoutesAuthGroup(svc, groups) - registerRoutesFilterGroup(svc, groups) - - return nil -} - -func registerRoutesAnonymousGroup(svc *RouterService, groups *RouterGroups) { - // login - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/", controllers.LoginController) - - // version - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/version", controllers.VersionController) - - // system info - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/system-info", controllers.SystemInfoController) - - // demo - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/demo", controllers.DemoController) - - // sync - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/sync", controllers.SyncController) -} - -func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) { - // node - svc.RegisterListControllerToGroup(groups.AuthGroup, "/nodes", controllers.NodeController) - - // project - svc.RegisterListControllerToGroup(groups.AuthGroup, "/projects", controllers.ProjectController) - - // user - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/users", controllers.UserController) - - // spider - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/spiders", controllers.SpiderController) - - // task - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/tasks", controllers.TaskController) - - // tag - svc.RegisterListControllerToGroup(groups.AuthGroup, "/tags", controllers.TagController) - - // setting - svc.RegisterListControllerToGroup(groups.AuthGroup, "/settings", controllers.SettingController) - - // data collection - svc.RegisterListControllerToGroup(groups.AuthGroup, "/data/collections", controllers.DataCollectionController) - - // result - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/results", controllers.ResultController) - - // schedule - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/schedules", controllers.ScheduleController) - - // stats - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/stats", controllers.StatsController) - - // token - svc.RegisterListControllerToGroup(groups.AuthGroup, "/tokens", controllers.TokenController) - - // git - svc.RegisterListControllerToGroup(groups.AuthGroup, "/gits", controllers.GitController) - - // role - svc.RegisterListControllerToGroup(groups.AuthGroup, "/roles", controllers.RoleController) - - // permission - svc.RegisterListControllerToGroup(groups.AuthGroup, "/permissions", controllers.PermissionController) - - // export - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/export", controllers.ExportController) - - // notification - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/notifications", controllers.NotificationController) - - // filter - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController) - - // data sources - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController) - - // environments - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/environments", controllers.EnvironmentController) -} - -func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) { - // filer - svc.RegisterActionControllerToGroup(groups.FilerGroup, "", controllers.FilerController) -} diff --git a/core/routes/router_test.go b/core/routes/router_test.go deleted file mode 100644 index 7b72c8524..000000000 --- a/core/routes/router_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package routes - -import ( - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "net/http" - "testing" - "time" -) - -func TestInitRoutes(t *testing.T) { - app := gin.New() - err := InitRoutes(app) - require.Nil(t, err) - - srv := &http.Server{ - Handler: app, - Addr: "localhost:8000", - } - go func() { - err = srv.ListenAndServe() - require.Nil(t, err) - }() - - time.Sleep(5 * time.Second) -} From e64ffb3deca40f8b3e827244c3af5e5322358476 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 10 Jul 2024 18:39:39 +0800 Subject: [PATCH 045/106] fix: config issue --- backend/conf/config.yml | 30 ++++++++++++++++++++++++--- core/config/config.go | 13 ++++-------- core/config/default_config.go | 39 ----------------------------------- 3 files changed, 31 insertions(+), 51 deletions(-) delete mode 100644 core/config/default_config.go diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 322ade507..dbcfed876 100755 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -1,3 +1,27 @@ -info: - edition: global.edition.community - version: v0.6.3 +# Crawlab Configuration File +edition: global.edition.community +version: v0.6.3 + +mongo: + host: localhost + port: 27017 + db: crawlab_test + username: "" + password: "" + authSource: "admin" + +server: + host: 0.0.0.0 + port: 8000 + +grpc: + address: localhost:9666 + server: + address: 0.0.0.0:9666 + authKey: Crawlab2021! + +api: + endpoint: http://localhost:8000 + +log: + path: /var/log/crawlab diff --git a/core/config/config.go b/core/config/config.go index 1caa14d7f..1eace82a1 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -1,7 +1,6 @@ package config import ( - "bytes" "github.com/apex/log" "github.com/crawlab-team/crawlab/trace" "github.com/fsnotify/fsnotify" @@ -63,14 +62,10 @@ func (c *Config) Init() (err error) { replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) - // read default config - defaultConfBuf := bytes.NewBufferString(DefaultConfigYaml) - if err := viper.ReadConfig(defaultConfBuf); err != nil { - return trace.TraceError(err) - } - - // merge config - if err := viper.MergeInConfig(); err != nil { // viper parsing config file + // read in config + if err := viper.ReadInConfig(); err != nil { + log.Errorf("Error reading config file, %s", err) + trace.PrintError(err) return err } diff --git a/core/config/default_config.go b/core/config/default_config.go deleted file mode 100644 index e2806e670..000000000 --- a/core/config/default_config.go +++ /dev/null @@ -1,39 +0,0 @@ -package config - -var DefaultConfigYaml = ` -version: v0.6.3 -edition: global.edition.community -mongo: - host: localhost - port: 27017 - db: crawlab_test - username: "" - password: "" - authSource: "admin" -server: - host: 0.0.0.0 - port: 8000 -spider: - fs: "/spiders" - workspace: "/workspace" - repo: "/repo" -task: - workers: 16 - cancelWaitSeconds: 30 -grpc: - address: localhost:9666 - server: - address: 0.0.0.0:9666 - authKey: Crawlab2021! -fs: - filer: - proxy: http://localhost:8888 - url: http://localhost:8000/filer - authKey: Crawlab2021! -node: - master: Y -api: - endpoint: http://localhost:8000 -log: - path: /var/log/crawlab -` From 13596436cd47c8be525320703aa18e5f29d31f07 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 10 Jul 2024 18:59:35 +0800 Subject: [PATCH 046/106] refactor: update viper config key for Pro edition check --- core/utils/system.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/system.go b/core/utils/system.go index 9b161f112..6e5011160 100644 --- a/core/utils/system.go +++ b/core/utils/system.go @@ -3,5 +3,5 @@ package utils import "github.com/spf13/viper" func IsPro() bool { - return viper.GetString("info.edition") == "global.edition.pro" + return viper.GetString("edition") == "global.edition.pro" } From f2f298d8f8299927c51e8bfb9d2de79510d71853 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 11 Jul 2024 12:45:20 +0800 Subject: [PATCH 047/106] refactor: removed unnecessary code --- core/controllers/base.go | 69 ------ core/controllers/binder.go | 14 -- core/controllers/binder_json.go | 208 ---------------- core/controllers/data_source_v2.go | 2 +- core/controllers/delegate_action.go | 17 -- core/controllers/delegate_basic.go | 99 -------- core/controllers/delegate_list.go | 222 ------------------ core/controllers/delegate_list_action.go | 17 -- core/controllers/environment.go | 57 ----- core/controllers/project_v2.go | 14 +- core/controllers/result_v2.go | 10 +- core/controllers/router_v2.go | 22 +- core/controllers/router_v2_test.go | 4 +- core/controllers/schedule_v2.go | 2 +- core/controllers/setting_v2.go | 5 +- core/controllers/spider_v2.go | 102 ++++---- core/controllers/task_v2.go | 44 ++-- core/controllers/token_v2.go | 2 +- core/controllers/user_v2.go | 2 +- core/controllers/user_v2_test.go | 2 +- core/controllers/utils_context.go | 2 +- core/models/models/git_v2.go | 26 -- core/models/models/metric_v2.go | 23 -- core/models/models/node_v2.go | 24 -- core/models/models/notification_setting_v2.go | 31 --- core/models/models/permission_v2.go | 13 - core/models/models/project_v2.go | 9 - core/models/models/role_permission_v2.go | 12 - core/models/models/role_v2.go | 9 - core/models/models/schedule_v2.go | 23 -- core/models/models/setting_v2.go | 12 - core/models/models/spider_stat_v2.go | 20 -- core/models/models/spider_v2.go | 32 --- core/models/models/task_queue_item_v2.go | 12 - core/models/models/task_stat_v2.go | 18 -- core/models/models/task_v2.go | 30 --- core/models/models/token_v2.go | 8 - core/models/models/user_role_v2.go | 12 - core/models/models/user_v2.go | 10 - core/models/models/{ => v2}/base_v2.go | 0 .../models/{ => v2}/data_collection_v2.go | 0 core/models/models/{ => v2}/data_source_v2.go | 0 .../models/{ => v2}/dependency_log_v2.go | 0 .../models/{ => v2}/dependency_setting_v2.go | 0 .../models/{ => v2}/dependency_task_v2.go | 0 core/models/models/{ => v2}/dependency_v2.go | 0 core/models/models/{ => v2}/environment_v2.go | 0 core/models/models/v2/git_v2.go | 27 +++ core/models/models/v2/metric_v2.go | 26 ++ core/models/models/v2/node_v2.go | 25 ++ .../models/v2/notification_channel_v2.go | 1 + .../models/v2/notification_setting_v2.go | 33 +++ core/models/models/v2/permission_v2.go | 15 ++ core/models/models/v2/project_v2.go | 11 + core/models/models/v2/role_permission_v2.go | 13 + core/models/models/v2/role_v2.go | 11 + core/models/models/v2/schedule_v2.go | 24 ++ core/models/models/v2/setting_v2.go | 13 + core/models/models/v2/spider_stat_v2.go | 21 ++ core/models/models/v2/spider_v2.go | 33 +++ core/models/models/v2/task_queue_item_v2.go | 13 + core/models/models/v2/task_stat_v2.go | 19 ++ core/models/models/v2/task_v2.go | 31 +++ core/models/models/{test.go => v2/test_v2.go} | 0 core/models/models/v2/token_v2.go | 10 + core/models/models/v2/user_role_v2.go | 13 + core/models/models/v2/user_v2.go | 12 + core/models/models/v2/variable_v2.go | 11 + core/models/models/variable_v2.go | 9 - 69 files changed, 469 insertions(+), 1142 deletions(-) delete mode 100644 core/controllers/base.go delete mode 100644 core/controllers/binder.go delete mode 100644 core/controllers/binder_json.go delete mode 100644 core/controllers/delegate_action.go delete mode 100644 core/controllers/delegate_basic.go delete mode 100644 core/controllers/delegate_list.go delete mode 100644 core/controllers/delegate_list_action.go delete mode 100644 core/controllers/environment.go delete mode 100644 core/models/models/git_v2.go delete mode 100644 core/models/models/metric_v2.go delete mode 100644 core/models/models/node_v2.go delete mode 100644 core/models/models/notification_setting_v2.go delete mode 100644 core/models/models/permission_v2.go delete mode 100644 core/models/models/project_v2.go delete mode 100644 core/models/models/role_permission_v2.go delete mode 100644 core/models/models/role_v2.go delete mode 100644 core/models/models/schedule_v2.go delete mode 100644 core/models/models/setting_v2.go delete mode 100644 core/models/models/spider_stat_v2.go delete mode 100644 core/models/models/spider_v2.go delete mode 100644 core/models/models/task_queue_item_v2.go delete mode 100644 core/models/models/task_stat_v2.go delete mode 100644 core/models/models/task_v2.go delete mode 100644 core/models/models/token_v2.go delete mode 100644 core/models/models/user_role_v2.go delete mode 100644 core/models/models/user_v2.go rename core/models/models/{ => v2}/base_v2.go (100%) rename core/models/models/{ => v2}/data_collection_v2.go (100%) rename core/models/models/{ => v2}/data_source_v2.go (100%) rename core/models/models/{ => v2}/dependency_log_v2.go (100%) rename core/models/models/{ => v2}/dependency_setting_v2.go (100%) rename core/models/models/{ => v2}/dependency_task_v2.go (100%) rename core/models/models/{ => v2}/dependency_v2.go (100%) rename core/models/models/{ => v2}/environment_v2.go (100%) create mode 100644 core/models/models/v2/git_v2.go create mode 100644 core/models/models/v2/metric_v2.go create mode 100644 core/models/models/v2/node_v2.go create mode 100644 core/models/models/v2/notification_channel_v2.go create mode 100644 core/models/models/v2/notification_setting_v2.go create mode 100644 core/models/models/v2/permission_v2.go create mode 100644 core/models/models/v2/project_v2.go create mode 100644 core/models/models/v2/role_permission_v2.go create mode 100644 core/models/models/v2/role_v2.go create mode 100644 core/models/models/v2/schedule_v2.go create mode 100644 core/models/models/v2/setting_v2.go create mode 100644 core/models/models/v2/spider_stat_v2.go create mode 100644 core/models/models/v2/spider_v2.go create mode 100644 core/models/models/v2/task_queue_item_v2.go create mode 100644 core/models/models/v2/task_stat_v2.go create mode 100644 core/models/models/v2/task_v2.go rename core/models/models/{test.go => v2/test_v2.go} (100%) create mode 100644 core/models/models/v2/token_v2.go create mode 100644 core/models/models/v2/user_role_v2.go create mode 100644 core/models/models/v2/user_v2.go create mode 100644 core/models/models/v2/variable_v2.go delete mode 100644 core/models/models/variable_v2.go diff --git a/core/controllers/base.go b/core/controllers/base.go deleted file mode 100644 index 261be1bff..000000000 --- a/core/controllers/base.go +++ /dev/null @@ -1,69 +0,0 @@ -package controllers - -import "github.com/gin-gonic/gin" - -const ( - ControllerIdNode = iota << 1 - ControllerIdProject - ControllerIdSpider - ControllerIdTask - ControllerIdJob - ControllerIdSchedule - ControllerIdUser - ControllerIdSetting - ControllerIdToken - ControllerIdVariable - ControllerIdTag - ControllerIdLogin - ControllerIdColor - ControllerIdDataSource - ControllerIdDataCollection - ControllerIdResult - ControllerIdStats - ControllerIdFiler - ControllerIdGit - ControllerIdRole - ControllerIdPermission - ControllerIdExport - ControllerIdNotification - ControllerIdFilter - ControllerIdEnvironment - ControllerIdSync - - ControllerIdVersion - ControllerIdI18n - ControllerIdSystemInfo - ControllerIdDemo -) - -type ControllerId int - -type BasicController interface { - Get(c *gin.Context) - Post(c *gin.Context) - Put(c *gin.Context) - Delete(c *gin.Context) -} - -type ListController interface { - BasicController - GetList(c *gin.Context) - PutList(c *gin.Context) - PostList(c *gin.Context) - DeleteList(c *gin.Context) -} - -type Action struct { - Method string - Path string - HandlerFunc gin.HandlerFunc -} - -type ActionController interface { - Actions() (actions []Action) -} - -type ListActionController interface { - ListController - ActionController -} diff --git a/core/controllers/binder.go b/core/controllers/binder.go deleted file mode 100644 index aa27508a9..000000000 --- a/core/controllers/binder.go +++ /dev/null @@ -1,14 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" -) - -type BinderInterface interface { - Bind(c *gin.Context) (res interfaces.Model, err error) - BindList(c *gin.Context) (res []interfaces.Model, err error) - BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) - BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) -} diff --git a/core/controllers/binder_json.go b/core/controllers/binder_json.go deleted file mode 100644 index e60ac30c8..000000000 --- a/core/controllers/binder_json.go +++ /dev/null @@ -1,208 +0,0 @@ -package controllers - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gin-gonic/gin" -) - -func NewJsonBinder(id ControllerId) (b *JsonBinder) { - return &JsonBinder{ - id: id, - } -} - -type JsonBinder struct { - id ControllerId -} - -func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) { - // declare - m := models.NewModelMap() - - switch b.id { - case ControllerIdNode: - err = c.ShouldBindJSON(&m.Node) - return &m.Node, err - case ControllerIdProject: - err = c.ShouldBindJSON(&m.Project) - return &m.Project, err - case ControllerIdSpider: - err = c.ShouldBindJSON(&m.Spider) - return &m.Spider, err - case ControllerIdTask: - err = c.ShouldBindJSON(&m.Task) - return &m.Task, err - case ControllerIdJob: - err = c.ShouldBindJSON(&m.Job) - return &m.Job, err - case ControllerIdSchedule: - err = c.ShouldBindJSON(&m.Schedule) - return &m.Schedule, err - case ControllerIdUser: - err = c.ShouldBindJSON(&m.User) - return &m.User, nil - case ControllerIdSetting: - err = c.ShouldBindJSON(&m.Setting) - return &m.Setting, nil - case ControllerIdToken: - err = c.ShouldBindJSON(&m.Token) - return &m.Token, nil - case ControllerIdVariable: - err = c.ShouldBindJSON(&m.Variable) - return &m.Variable, nil - case ControllerIdTag: - err = c.ShouldBindJSON(&m.Tag) - return &m.Tag, nil - case ControllerIdDataSource: - err = c.ShouldBindJSON(&m.DataSource) - return &m.DataSource, nil - case ControllerIdDataCollection: - err = c.ShouldBindJSON(&m.DataCollection) - return &m.DataCollection, nil - case ControllerIdGit: - err = c.ShouldBindJSON(&m.Git) - return &m.Git, nil - case ControllerIdRole: - err = c.ShouldBindJSON(&m.Role) - return &m.Role, nil - case ControllerIdPermission: - err = c.ShouldBindJSON(&m.Permission) - return &m.Permission, nil - case ControllerIdEnvironment: - err = c.ShouldBindJSON(&m.Environment) - return &m.Environment, nil - default: - return nil, errors.ErrorControllerInvalidControllerId - } -} - -func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) { - // declare - m := models.NewModelListMap() - - // bind - switch b.id { - case ControllerIdNode: - err = c.ShouldBindJSON(&m.Nodes) - return m.Nodes, err - case ControllerIdProject: - err = c.ShouldBindJSON(&m.Projects) - return m.Projects, err - case ControllerIdSpider: - err = c.ShouldBindJSON(&m.Spiders) - return m.Spiders, err - case ControllerIdTask: - err = c.ShouldBindJSON(&m.Tasks) - return m.Tasks, err - case ControllerIdJob: - err = c.ShouldBindJSON(&m.Jobs) - return m.Jobs, err - case ControllerIdSchedule: - err = c.ShouldBindJSON(&m.Schedules) - return m.Schedules, err - case ControllerIdUser: - err = c.ShouldBindJSON(&m.Users) - return m.Users, nil - case ControllerIdSetting: - err = c.ShouldBindJSON(&m.Settings) - return m.Settings, nil - case ControllerIdToken: - err = c.ShouldBindJSON(&m.Tokens) - return m.Tokens, nil - case ControllerIdVariable: - err = c.ShouldBindJSON(&m.Variables) - return m.Variables, nil - case ControllerIdTag: - err = c.ShouldBindJSON(&m.Tags) - return m.Tags, nil - case ControllerIdDataSource: - err = c.ShouldBindJSON(&m.DataSources) - return m.DataSources, nil - case ControllerIdDataCollection: - err = c.ShouldBindJSON(&m.DataCollections) - return m.DataCollections, nil - case ControllerIdGit: - err = c.ShouldBindJSON(&m.Gits) - return m.Gits, nil - case ControllerIdRole: - err = c.ShouldBindJSON(&m.Roles) - return m.Roles, nil - case ControllerIdEnvironment: - err = c.ShouldBindJSON(&m.Environments) - return m.Environments, nil - default: - return nil, errors.ErrorControllerInvalidControllerId - } -} - -func (b *JsonBinder) BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) { - if err := c.ShouldBindJSON(&payload); err != nil { - return payload, err - } - return payload, nil -} - -func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) { - // declare - m := models.NewModelMap() - - // bind - if err := c.ShouldBindJSON(&payload); err != nil { - return payload, nil, err - } - - // validate - if len(payload.Ids) == 0 || - len(payload.Fields) == 0 { - return payload, nil, errors.ErrorControllerRequestPayloadInvalid - } - - // unmarshall - switch b.id { - case ControllerIdNode: - err = json.Unmarshal([]byte(payload.Data), &m.Node) - return payload, &m.Node, err - case ControllerIdProject: - err = json.Unmarshal([]byte(payload.Data), &m.Project) - return payload, &m.Project, err - case ControllerIdSpider: - err = json.Unmarshal([]byte(payload.Data), &m.Spider) - return payload, &m.Spider, err - case ControllerIdTask: - err = json.Unmarshal([]byte(payload.Data), &m.Task) - return payload, &m.Task, err - case ControllerIdJob: - err = json.Unmarshal([]byte(payload.Data), &m.Job) - return payload, &m.Job, err - case ControllerIdSchedule: - err = json.Unmarshal([]byte(payload.Data), &m.Schedule) - return payload, &m.Schedule, err - case ControllerIdUser: - err = json.Unmarshal([]byte(payload.Data), &m.User) - return payload, &m.User, err - case ControllerIdSetting: - err = json.Unmarshal([]byte(payload.Data), &m.Setting) - return payload, &m.Setting, err - case ControllerIdToken: - err = json.Unmarshal([]byte(payload.Data), &m.Token) - return payload, &m.Token, err - case ControllerIdVariable: - err = json.Unmarshal([]byte(payload.Data), &m.Variable) - return payload, &m.Variable, err - case ControllerIdDataSource: - err = json.Unmarshal([]byte(payload.Data), &m.DataSource) - return payload, &m.DataSource, err - case ControllerIdDataCollection: - err = json.Unmarshal([]byte(payload.Data), &m.DataCollection) - return payload, &m.DataCollection, err - case ControllerIdEnvironment: - err = json.Unmarshal([]byte(payload.Data), &m.Environment) - return payload, &m.Environment, err - default: - return payload, nil, errors.ErrorControllerInvalidControllerId - } -} diff --git a/core/controllers/data_source_v2.go b/core/controllers/data_source_v2.go index 6962dc957..84a3ac534 100644 --- a/core/controllers/data_source_v2.go +++ b/core/controllers/data_source_v2.go @@ -3,7 +3,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/ds" "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/core/controllers/delegate_action.go b/core/controllers/delegate_action.go deleted file mode 100644 index dccb0787a..000000000 --- a/core/controllers/delegate_action.go +++ /dev/null @@ -1,17 +0,0 @@ -package controllers - -func NewActionControllerDelegate(id ControllerId, actions []Action) (d *ActionControllerDelegate) { - return &ActionControllerDelegate{ - id: id, - actions: actions, - } -} - -type ActionControllerDelegate struct { - id ControllerId - actions []Action -} - -func (ctr *ActionControllerDelegate) Actions() (actions []Action) { - return ctr.actions -} diff --git a/core/controllers/delegate_basic.go b/core/controllers/delegate_basic.go deleted file mode 100644 index 86a26f3e5..000000000 --- a/core/controllers/delegate_basic.go +++ /dev/null @@ -1,99 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" -) - -func NewBasicControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *BasicControllerDelegate) { - return &BasicControllerDelegate{ - id: id, - svc: svc, - } -} - -type BasicControllerDelegate struct { - id ControllerId - svc interfaces.ModelBaseService -} - -func (d *BasicControllerDelegate) Get(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - doc, err := d.svc.GetById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, doc) -} - -func (d *BasicControllerDelegate) Post(c *gin.Context) { - doc, err := NewJsonBinder(d.id).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, doc) -} - -func (d *BasicControllerDelegate) Put(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - doc, err := NewJsonBinder(d.id).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if doc.GetId() != id { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - _, err = d.svc.GetById(id) - if err != nil { - HandleErrorNotFound(c, err) - return - } - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, doc) -} - -func (d *BasicControllerDelegate) Delete(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - doc, err := d.svc.GetById(oid) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Delete(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} diff --git a/core/controllers/delegate_list.go b/core/controllers/delegate_list.go deleted file mode 100644 index 403a89301..000000000 --- a/core/controllers/delegate_list.go +++ /dev/null @@ -1,222 +0,0 @@ -package controllers - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "reflect" - "time" -) - -func NewListControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *ListControllerDelegate) { - if svc == nil { - panic(errors.ErrorControllerNoModelService) - } - - return &ListControllerDelegate{ - id: id, - svc: svc, - bc: NewBasicControllerDelegate(id, svc), - } -} - -type ListControllerDelegate struct { - id ControllerId - svc interfaces.ModelBaseService - bc BasicController -} - -func (d *ListControllerDelegate) Get(c *gin.Context) { - d.bc.Get(c) -} - -func (d *ListControllerDelegate) Post(c *gin.Context) { - d.bc.Post(c) -} - -func (d *ListControllerDelegate) Put(c *gin.Context) { - d.bc.Put(c) -} - -func (d *ListControllerDelegate) Delete(c *gin.Context) { - d.bc.Delete(c) -} - -func (d *ListControllerDelegate) GetList(c *gin.Context) { - // get all if query field "all" is set true - all := MustGetFilterAll(c) - if all { - d.getAll(c) - return - } - - // get list and total - l, total, err := d.getList(c) - if err != nil { - return - } - - // response - HandleSuccessWithListData(c, l, total) -} - -func (d *ListControllerDelegate) PostList(c *gin.Context) { - // bind - docs, err := NewJsonBinder(d.id).BindList(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // success ids - var ids []primitive.ObjectID - - // reflect - switch reflect.TypeOf(docs).Kind() { - case reflect.Slice, reflect.Array: - s := reflect.ValueOf(docs) - for i := 0; i < s.Len(); i++ { - item := s.Index(i) - if !item.CanAddr() { - HandleErrorInternalServerError(c, errors.ErrorModelInvalidType) - return - } - ptr := item.Addr() - doc, ok := ptr.Interface().(interfaces.Model) - if !ok { - HandleErrorInternalServerError(c, errors.ErrorModelInvalidType) - return - } - if err := delegate.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil { - _ = trace.TraceError(err) - continue - } - ids = append(ids, doc.GetId()) - } - } - - // check - items, err := utils.GetArrayItems(docs) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if len(ids) < len(items) { - HandleErrorInternalServerError(c, errors.ErrorControllerAddError) - return - } - - // success - HandleSuccessWithData(c, docs) -} - -func (d *ListControllerDelegate) PutList(c *gin.Context) { - payload, doc, err := NewJsonBinder(d.id).BindBatchRequestPayloadWithStringData(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // query - query := bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - } - - // update - if err := d.svc.UpdateDoc(query, doc, payload.Fields); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (d *ListControllerDelegate) DeleteList(c *gin.Context) { - payload, err := NewJsonBinder(d.id).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := d.svc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (d *ListControllerDelegate) getAll(c *gin.Context) { - // get list - tic := time.Now() - log.Debugf("getAll -> d.svc.GetMany:start") - list, err := d.svc.GetList(nil, &mongo.FindOptions{ - Sort: bson.D{{"_id", -1}}, - }) - if err != nil { - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - log.Debugf("getAll -> d.svc.GetMany:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) - tic = time.Now() - - // total count - tic = time.Now() - log.Debugf("getAll -> d.svc.Count:start") - total, err := d.svc.Count(nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - log.Debugf("getAll -> d.svc.Count:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) - - // response - HandleSuccessWithListData(c, list, total) -} - -func (d *ListControllerDelegate) getList(c *gin.Context) (l interfaces.List, total int, err error) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - l, err = d.svc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccessWithListData(c, nil, 0) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // total count - total, err = d.svc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - return l, total, nil -} diff --git a/core/controllers/delegate_list_action.go b/core/controllers/delegate_list_action.go deleted file mode 100644 index deacf347e..000000000 --- a/core/controllers/delegate_list_action.go +++ /dev/null @@ -1,17 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" -) - -func NewListPostActionControllerDelegate(id ControllerId, svc interfaces.ModelBaseService, actions []Action) (d *ListActionControllerDelegate) { - return &ListActionControllerDelegate{ - NewListControllerDelegate(id, svc), - NewActionControllerDelegate(id, actions), - } -} - -type ListActionControllerDelegate struct { - ListController - ActionController -} diff --git a/core/controllers/environment.go b/core/controllers/environment.go deleted file mode 100644 index 560b02c9d..000000000 --- a/core/controllers/environment.go +++ /dev/null @@ -1,57 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" -) - -var EnvironmentController *environmentController - -var EnvironmentActions []Action - -type environmentController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *environmentContext -} - -type environmentContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func newEnvironmentContext() *environmentContext { - // context - ctx := &environmentContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newEnvironmentController() *environmentController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions) - d := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions) - ctx := newEnvironmentContext() - - return &environmentController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/project_v2.go b/core/controllers/project_v2.go index 34fb6b8dd..03396fd58 100644 --- a/core/controllers/project_v2.go +++ b/core/controllers/project_v2.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/gin-gonic/gin" @@ -15,7 +15,7 @@ func GetProjectList(c *gin.Context) { // get all list all := MustGetFilterAll(c) if all { - NewControllerV2[models.ProjectV2]().getAll(c) + NewControllerV2[models2.ProjectV2]().getAll(c) return } @@ -25,7 +25,7 @@ func GetProjectList(c *gin.Context) { sort := MustGetSortOption(c) // get list - projects, err := service.NewModelServiceV2[models.ProjectV2]().GetMany(query, &mongo.FindOptions{ + projects, err := service.NewModelServiceV2[models2.ProjectV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -37,12 +37,12 @@ func GetProjectList(c *gin.Context) { return } if len(projects) == 0 { - HandleSuccessWithListData(c, []models.ProjectV2{}, 0) + HandleSuccessWithListData(c, []models2.ProjectV2{}, 0) return } // total count - total, err := service.NewModelServiceV2[models.ProjectV2]().Count(query) + total, err := service.NewModelServiceV2[models2.ProjectV2]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return @@ -61,7 +61,7 @@ func GetProjectList(c *gin.Context) { } // spiders - spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{ + spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(bson.M{ "project_id": bson.M{ "$in": ids, }, @@ -80,7 +80,7 @@ func GetProjectList(c *gin.Context) { } // assign - var data []models.ProjectV2 + var data []models2.ProjectV2 for _, p := range projects { p.Spiders = cache[p.Id] data = append(data, p) diff --git a/core/controllers/result_v2.go b/core/controllers/result_v2.go index 9033367ff..2f7fb7372 100644 --- a/core/controllers/result_v2.go +++ b/core/controllers/result_v2.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/utils" @@ -32,17 +32,17 @@ func GetResultList(c *gin.Context) { } // data collection - dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId) + dc, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(dcId) if err != nil { HandleErrorInternalServerError(c, err) return } // data source - ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId) + ds, err := service.NewModelServiceV2[models2.DataSourceV2]().GetById(dsId) if err != nil { if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models.DataSourceV2{} + ds = &models2.DataSourceV2{} } else { HandleErrorInternalServerError(c, err) return @@ -54,7 +54,7 @@ func GetResultList(c *gin.Context) { "col_id": dc.Id, "data_source_id": ds.Id, } - s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetOne(sq, nil) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 809a2f79c..4f2ff1d6a 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/gin-gonic/gin" "net/http" ) @@ -55,8 +55,8 @@ func InitRoutes(app *gin.Engine) (err error) { // routes groups groups := NewRouterGroups(app) - RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]()) - RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]([]Action{ + RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]()) + RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models2.DataSourceV2]([]Action{ { Method: http.MethodPost, Path: "", @@ -73,16 +73,16 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostDataSourceChangePassword, }, }...)) - RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) - RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) - RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]([]Action{ + RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]()) + RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models2.NodeV2]()) + RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models2.ProjectV2]([]Action{ { Method: http.MethodGet, Path: "", HandlerFunc: GetProjectList, }, }...)) - RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]([]Action{ + RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models2.ScheduleV2]([]Action{ { Method: http.MethodPost, Path: "", @@ -104,7 +104,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostScheduleDisable, }, }...)) - RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2]([]Action{ + RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models2.SpiderV2]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -202,7 +202,7 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostSpiderDataSource, }, }...)) - RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2]([]Action{ + RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models2.TaskV2]([]Action{ { Method: http.MethodGet, Path: "/:id", @@ -249,14 +249,14 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetTaskData, }, }...)) - RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2]([]Action{ + RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models2.TokenV2]([]Action{ { Method: http.MethodPost, Path: "", HandlerFunc: PostToken, }, }...)) - RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2]([]Action{ + RegisterController(groups.AuthGroup, "/users", NewControllerV2[models2.UserV2]([]Action{ { Method: http.MethodPost, Path: "", diff --git a/core/controllers/router_v2_test.go b/core/controllers/router_v2_test.go index 47642324d..e0e6f5306 100644 --- a/core/controllers/router_v2_test.go +++ b/core/controllers/router_v2_test.go @@ -2,7 +2,7 @@ package controllers_test import ( "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "testing" @@ -28,7 +28,7 @@ func TestRouterGroups(t *testing.T) { func TestRegisterController_Routes(t *testing.T) { router := gin.Default() groups := controllers.NewRouterGroups(router) - ctr := controllers.NewControllerV2[models.TestModel]() + ctr := controllers.NewControllerV2[models.TestModelV2]() basePath := "/testmodels" controllers.RegisterController(groups.AuthGroup, basePath, ctr) diff --git a/core/controllers/schedule_v2.go b/core/controllers/schedule_v2.go index 3e19da769..9149f2a17 100644 --- a/core/controllers/schedule_v2.go +++ b/core/controllers/schedule_v2.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/schedule" "github.com/gin-gonic/gin" diff --git a/core/controllers/setting_v2.go b/core/controllers/setting_v2.go index 326873b99..d2d913391 100644 --- a/core/controllers/setting_v2.go +++ b/core/controllers/setting_v2.go @@ -2,6 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" @@ -12,7 +13,7 @@ func GetSetting(c *gin.Context) { key := c.Param("id") // setting - s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil) + s, err := service.NewModelServiceV2[models2.SettingV2]().GetOne(bson.M{"key": key}, nil) if err != nil { HandleErrorInternalServerError(c, err) return @@ -32,7 +33,7 @@ func PutSetting(c *gin.Context) { return } - modelSvc := service.NewModelServiceV2[models.SettingV2]() + modelSvc := service.NewModelServiceV2[models2.SettingV2]() // setting _s, err := modelSvc.GetOne(bson.M{"key": key}, nil) diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 7523d5407..a0abfb371 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" @@ -29,7 +29,7 @@ func GetSpiderById(c *gin.Context) { HandleErrorBadRequest(c, err) return } - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorNotFound(c, err) return @@ -40,7 +40,7 @@ func GetSpiderById(c *gin.Context) { } // stat - s.Stat, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(s.Id) + s.Stat, err = service.NewModelServiceV2[models2.SpiderStatV2]().GetById(s.Id) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -50,7 +50,7 @@ func GetSpiderById(c *gin.Context) { // data collection if !s.ColId.IsZero() { - col, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(s.ColId) + col, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(s.ColId) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -63,7 +63,7 @@ func GetSpiderById(c *gin.Context) { // git if utils.IsPro() && !s.GitId.IsZero() { - s.Git, err = service.NewModelServiceV2[models.GitV2]().GetById(s.GitId) + s.Git, err = service.NewModelServiceV2[models2.GitV2]().GetById(s.GitId) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -79,14 +79,14 @@ func GetSpiderList(c *gin.Context) { // get all list all := MustGetFilterAll(c) if all { - NewControllerV2[models.SpiderV2]().getAll(c) + NewControllerV2[models2.SpiderV2]().getAll(c) return } // get list withStats := c.Query("stats") if withStats == "" { - NewControllerV2[models.SpiderV2]().GetList(c) + NewControllerV2[models2.SpiderV2]().GetList(c) return } @@ -101,7 +101,7 @@ func getSpiderListWithStats(c *gin.Context) { sort := MustGetSortOption(c) // get list - spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(query, &mongo.FindOptions{ + spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -113,7 +113,7 @@ func getSpiderListWithStats(c *gin.Context) { return } if len(spiders) == 0 { - HandleSuccessWithListData(c, []models.SpiderV2{}, 0) + HandleSuccessWithListData(c, []models2.SpiderV2{}, 0) return } @@ -128,21 +128,21 @@ func getSpiderListWithStats(c *gin.Context) { } // total count - total, err := service.NewModelServiceV2[models.SpiderV2]().Count(query) + total, err := service.NewModelServiceV2[models2.SpiderV2]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return } // stat list - spiderStats, err := service.NewModelServiceV2[models.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) + spiderStats, err := service.NewModelServiceV2[models2.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // cache stat list to dict - dict := map[primitive.ObjectID]models.SpiderStatV2{} + dict := map[primitive.ObjectID]models2.SpiderStatV2{} var taskIds []primitive.ObjectID for _, st := range spiderStats { if st.Tasks > 0 { @@ -159,9 +159,9 @@ func getSpiderListWithStats(c *gin.Context) { } // task list and stats - var tasks []models.TaskV2 - dictTask := map[primitive.ObjectID]models.TaskV2{} - dictTaskStat := map[primitive.ObjectID]models.TaskStatV2{} + var tasks []models2.TaskV2 + dictTask := map[primitive.ObjectID]models2.TaskV2{} + dictTaskStat := map[primitive.ObjectID]models2.TaskStatV2{} if len(taskIds) > 0 { // task list queryTask := bson.M{ @@ -169,14 +169,14 @@ func getSpiderListWithStats(c *gin.Context) { "$in": taskIds, }, } - tasks, err = service.NewModelServiceV2[models.TaskV2]().GetMany(queryTask, nil) + tasks, err = service.NewModelServiceV2[models2.TaskV2]().GetMany(queryTask, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // task stats list - taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(queryTask, nil) + taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(queryTask, nil) if err != nil { HandleErrorInternalServerError(c, err) return @@ -198,9 +198,9 @@ func getSpiderListWithStats(c *gin.Context) { } // git list - var gits []models.GitV2 + var gits []models2.GitV2 if len(gitIds) > 0 && utils.IsPro() { - gits, err = service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) + gits, err = service.NewModelServiceV2[models2.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) if err != nil { HandleErrorInternalServerError(c, err) return @@ -208,13 +208,13 @@ func getSpiderListWithStats(c *gin.Context) { } // cache git list to dict - dictGit := map[primitive.ObjectID]models.GitV2{} + dictGit := map[primitive.ObjectID]models2.GitV2{} for _, g := range gits { dictGit[g.Id] = g } // iterate list again - var data []models.SpiderV2 + var data []models2.SpiderV2 for _, s := range spiders { // spider stat st, ok := dict[s.Id] @@ -246,7 +246,7 @@ func getSpiderListWithStats(c *gin.Context) { func PostSpider(c *gin.Context) { // bind - var s models.SpiderV2 + var s models2.SpiderV2 if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return @@ -264,7 +264,7 @@ func PostSpider(c *gin.Context) { // add s.SetCreated(u.Id) s.SetUpdated(u.Id) - id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(s) + id, err := service.NewModelServiceV2[models2.SpiderV2]().InsertOne(s) if err != nil { HandleErrorInternalServerError(c, err) return @@ -272,11 +272,11 @@ func PostSpider(c *gin.Context) { s.SetId(id) // add stat - st := models.SpiderStatV2{} + st := models2.SpiderStatV2{} st.SetId(id) st.SetCreated(u.Id) st.SetUpdated(u.Id) - _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(st) + _, err = service.NewModelServiceV2[models2.SpiderStatV2]().InsertOne(st) if err != nil { HandleErrorInternalServerError(c, err) return @@ -305,7 +305,7 @@ func PutSpiderById(c *gin.Context) { } // bind - var s models.SpiderV2 + var s models2.SpiderV2 if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return @@ -319,7 +319,7 @@ func PutSpiderById(c *gin.Context) { u := GetUserFromContextV2(c) - modelSvc := service.NewModelServiceV2[models.SpiderV2]() + modelSvc := service.NewModelServiceV2[models2.SpiderV2]() // save s.SetUpdated(u.Id) @@ -348,19 +348,19 @@ func DeleteSpiderById(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete spider - err = service.NewModelServiceV2[models.SpiderV2]().DeleteById(id) + err = service.NewModelServiceV2[models2.SpiderV2]().DeleteById(id) if err != nil { return err } // delete spider stat - err = service.NewModelServiceV2[models.SpiderStatV2]().DeleteById(id) + err = service.NewModelServiceV2[models2.SpiderStatV2]().DeleteById(id) if err != nil { return err } // related tasks - tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": id}, nil) + tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{"spider_id": id}, nil) if err != nil { return err } @@ -376,13 +376,13 @@ func DeleteSpiderById(c *gin.Context) { } // delete related tasks - err = service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + err = service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) if err != nil { return err } // delete related task stats - err = service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + err = service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) if err != nil { return err } @@ -410,7 +410,7 @@ func DeleteSpiderById(c *gin.Context) { go func() { // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { log.Errorf("failed to get spider: %s", err.Error()) trace.PrintError(err) @@ -451,7 +451,7 @@ func DeleteSpiderList(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete spiders - if err := service.NewModelServiceV2[models.SpiderV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.SpiderV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -460,7 +460,7 @@ func DeleteSpiderList(c *gin.Context) { } // delete spider stats - if err := service.NewModelServiceV2[models.SpiderStatV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.SpiderStatV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -469,7 +469,7 @@ func DeleteSpiderList(c *gin.Context) { } // related tasks - tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) + tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) if err != nil { return err } @@ -485,12 +485,12 @@ func DeleteSpiderList(c *gin.Context) { } // delete related tasks - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { return err } // delete related task stats - if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { return err } @@ -524,7 +524,7 @@ func DeleteSpiderList(c *gin.Context) { defer wg.Done() // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { log.Errorf("failed to get spider: %s", err.Error()) trace.PrintError(err) @@ -692,14 +692,14 @@ func GetSpiderDataSource(c *gin.Context) { } // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return } // data source - ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(s.DataSourceId) + ds, err := service.NewModelServiceV2[models2.DataSourceV2]().GetById(s.DataSourceId) if err != nil { if err.Error() == mongo2.ErrNoDocuments.Error() { HandleSuccess(c) @@ -728,7 +728,7 @@ func PostSpiderDataSource(c *gin.Context) { } // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -736,7 +736,7 @@ func PostSpiderDataSource(c *gin.Context) { // data source if !dsId.IsZero() { - _, err = service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId) + _, err = service.NewModelServiceV2[models2.DataSourceV2]().GetById(dsId) if err != nil { HandleErrorInternalServerError(c, err) return @@ -747,7 +747,7 @@ func PostSpiderDataSource(c *gin.Context) { u := GetUserFromContextV2(c) s.DataSourceId = dsId s.SetUpdatedBy(u.Id) - _, err = service.NewModelServiceV2[models.SpiderV2]().InsertOne(*s) + _, err = service.NewModelServiceV2[models2.SpiderV2]().InsertOne(*s) if err != nil { HandleErrorInternalServerError(c, err) return @@ -756,7 +756,7 @@ func PostSpiderDataSource(c *gin.Context) { HandleSuccess(c) } -func getSpiderFsSvc(s *models.SpiderV2) (svc interfaces.FsServiceV2, err error) { +func getSpiderFsSvc(s *models2.SpiderV2) (svc interfaces.FsServiceV2, err error) { workspacePath := viper.GetString("workspace") fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, s.Id.Hex())) @@ -764,7 +764,7 @@ func getSpiderFsSvc(s *models.SpiderV2) (svc interfaces.FsServiceV2, err error) } func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) { - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { log.Errorf("failed to get spider: %s", err.Error()) trace.PrintError(err) @@ -773,8 +773,8 @@ func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err return getSpiderFsSvc(s) } -func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { - modelSvc := service.NewModelServiceV2[models.DataCollectionV2]() +func upsertSpiderDataCollection(s *models2.SpiderV2) (err error) { + modelSvc := service.NewModelServiceV2[models2.DataCollectionV2]() if s.ColId.IsZero() { // validate if s.ColName == "" { @@ -785,7 +785,7 @@ func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { // not exists, add new - dc = &models.DataCollectionV2{Name: s.ColName} + dc = &models2.DataCollectionV2{Name: s.ColName} dcId, err := modelSvc.InsertOne(*dc) if err != nil { return err @@ -812,7 +812,7 @@ func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { return nil } -func UpsertSpiderDataCollection(s *models.SpiderV2) (err error) { +func UpsertSpiderDataCollection(s *models2.SpiderV2) (err error) { return upsertSpiderDataCollection(s) } @@ -824,7 +824,7 @@ func getSpiderRootPath(c *gin.Context) (rootPath string, err error) { } // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { return "", err } diff --git a/core/controllers/task_v2.go b/core/controllers/task_v2.go index aad4fcf3f..5284210a4 100644 --- a/core/controllers/task_v2.go +++ b/core/controllers/task_v2.go @@ -5,7 +5,7 @@ import ( log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/spider/admin" @@ -34,7 +34,7 @@ func GetTaskById(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorNotFound(c, err) return @@ -45,7 +45,7 @@ func GetTaskById(c *gin.Context) { } // spider - t.Spider, _ = service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId) + t.Spider, _ = service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId) // skip if task status is pending if t.Status == constants.TaskStatusPending { @@ -54,7 +54,7 @@ func GetTaskById(c *gin.Context) { } // task stat - t.Stat, _ = service.NewModelServiceV2[models.TaskStatV2]().GetById(id) + t.Stat, _ = service.NewModelServiceV2[models2.TaskStatV2]().GetById(id) HandleSuccessWithData(c, t) } @@ -62,7 +62,7 @@ func GetTaskById(c *gin.Context) { func GetTaskList(c *gin.Context) { withStats := c.Query("stats") if withStats == "" { - NewControllerV2[models.TaskV2]().GetList(c) + NewControllerV2[models2.TaskV2]().GetList(c) return } @@ -72,7 +72,7 @@ func GetTaskList(c *gin.Context) { sort := MustGetSortOption(c) // get tasks - tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ + tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -101,14 +101,14 @@ func GetTaskList(c *gin.Context) { } // total count - total, err := service.NewModelServiceV2[models.TaskV2]().Count(query) + total, err := service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return } // stat list - stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ + stats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{ "_id": bson.M{ "$in": taskIds, }, @@ -119,13 +119,13 @@ func GetTaskList(c *gin.Context) { } // cache stat list to dict - statsDict := map[primitive.ObjectID]models.TaskStatV2{} + statsDict := map[primitive.ObjectID]models2.TaskStatV2{} for _, s := range stats { statsDict[s.Id] = s } // spider list - spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{ + spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(bson.M{ "_id": bson.M{ "$in": spiderIds, }, @@ -136,7 +136,7 @@ func GetTaskList(c *gin.Context) { } // cache spider list to dict - spiderDict := map[primitive.ObjectID]models.SpiderV2{} + spiderDict := map[primitive.ObjectID]models2.SpiderV2{} for _, s := range spiders { spiderDict[s.Id] = s } @@ -170,22 +170,22 @@ func DeleteTaskById(c *gin.Context) { // delete in db if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete task - _, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + _, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { return err } - err = service.NewModelServiceV2[models.TaskV2]().DeleteById(id) + err = service.NewModelServiceV2[models2.TaskV2]().DeleteById(id) if err != nil { return err } // delete task stat - _, err = service.NewModelServiceV2[models.TaskStatV2]().GetById(id) + _, err = service.NewModelServiceV2[models2.TaskStatV2]().GetById(id) if err != nil { log2.Warnf("delete task stat error: %s", err.Error()) return nil } - err = service.NewModelServiceV2[models.TaskStatV2]().DeleteById(id) + err = service.NewModelServiceV2[models2.TaskStatV2]().DeleteById(id) if err != nil { log2.Warnf("delete task stat error: %s", err.Error()) return nil @@ -217,7 +217,7 @@ func DeleteList(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { // delete tasks - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -226,7 +226,7 @@ func DeleteList(c *gin.Context) { } // delete task stats - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -261,7 +261,7 @@ func DeleteList(c *gin.Context) { func PostTaskRun(c *gin.Context) { // task - var t models.TaskV2 + var t models2.TaskV2 if err := c.ShouldBindJSON(&t); err != nil { HandleErrorBadRequest(c, err) return @@ -274,7 +274,7 @@ func PostTaskRun(c *gin.Context) { } // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId) if err != nil { HandleErrorInternalServerError(c, err) return @@ -319,7 +319,7 @@ func PostTaskRestart(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -363,7 +363,7 @@ func PostTaskCancel(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -446,7 +446,7 @@ func GetTaskData(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/token_v2.go b/core/controllers/token_v2.go index bac880b2c..b66f873b0 100644 --- a/core/controllers/token_v2.go +++ b/core/controllers/token_v2.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/gin-gonic/gin" diff --git a/core/controllers/user_v2.go b/core/controllers/user_v2.go index 434b20614..152dcb35b 100644 --- a/core/controllers/user_v2.go +++ b/core/controllers/user_v2.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" diff --git a/core/controllers/user_v2_test.go b/core/controllers/user_v2_test.go index c86739fc6..889015372 100644 --- a/core/controllers/user_v2_test.go +++ b/core/controllers/user_v2_test.go @@ -3,7 +3,7 @@ package controllers_test import ( "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" diff --git a/core/controllers/utils_context.go b/core/controllers/utils_context.go index 9365bd0de..00e2cc9c6 100644 --- a/core/controllers/utils_context.go +++ b/core/controllers/utils_context.go @@ -3,7 +3,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/gin-gonic/gin" ) diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go deleted file mode 100644 index 730c2c89d..000000000 --- a/core/models/models/git_v2.go +++ /dev/null @@ -1,26 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/vcs" - "time" -) - -type GitV2 struct { - any `collection:"gits"` - BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` - Refs []vcs.GitRef `json:"refs" bson:"refs"` - RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` - CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` - - // settings - AutoPull bool `json:"auto_pull" bson:"auto_pull"` -} diff --git a/core/models/models/metric_v2.go b/core/models/models/metric_v2.go deleted file mode 100644 index d7dbf0aac..000000000 --- a/core/models/models/metric_v2.go +++ /dev/null @@ -1,23 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type MetricV2 struct { - any `collection:"metrics"` - BaseModelV2[MetricV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` - DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` - NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` - NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` -} diff --git a/core/models/models/node_v2.go b/core/models/models/node_v2.go deleted file mode 100644 index 6d5a84421..000000000 --- a/core/models/models/node_v2.go +++ /dev/null @@ -1,24 +0,0 @@ -package models - -import ( - "time" -) - -type NodeV2 struct { - any `collection:"nodes"` - BaseModelV2[NodeV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_at" bson:"active_ts"` - AvailableRunners int `json:"available_runners" bson:"available_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` -} diff --git a/core/models/models/notification_setting_v2.go b/core/models/models/notification_setting_v2.go deleted file mode 100644 index b07a01970..000000000 --- a/core/models/models/notification_setting_v2.go +++ /dev/null @@ -1,31 +0,0 @@ -package models - -type NotificationSettingV2 struct { - any `collection:"notification_settings"` - BaseModelV2[NotificationSettingV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` -} - -type NotificationSettingMail struct { - Server string `json:"server" bson:"server"` - Port string `json:"port,omitempty" bson:"port,omitempty"` - User string `json:"user,omitempty" bson:"user,omitempty"` - Password string `json:"password,omitempty" bson:"password,omitempty"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` - To string `json:"to,omitempty" bson:"to,omitempty"` - Cc string `json:"cc,omitempty" bson:"cc,omitempty"` -} - -type NotificationSettingMobile struct { - Webhook string `json:"webhook" bson:"webhook"` -} diff --git a/core/models/models/permission_v2.go b/core/models/models/permission_v2.go deleted file mode 100644 index 77c830a38..000000000 --- a/core/models/models/permission_v2.go +++ /dev/null @@ -1,13 +0,0 @@ -package models - -type PermissionV2 struct { - any `collection:"permissions"` - BaseModelV2[PermissionV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type string `json:"type" bson:"type"` - Target []string `json:"target" bson:"target"` - Allow []string `json:"allow" bson:"allow"` - Deny []string `json:"deny" bson:"deny"` -} diff --git a/core/models/models/project_v2.go b/core/models/models/project_v2.go deleted file mode 100644 index ee07e705a..000000000 --- a/core/models/models/project_v2.go +++ /dev/null @@ -1,9 +0,0 @@ -package models - -type ProjectV2 struct { - any `collection:"projects"` - BaseModelV2[ProjectV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Spiders int `json:"spiders" bson:"-"` -} diff --git a/core/models/models/role_permission_v2.go b/core/models/models/role_permission_v2.go deleted file mode 100644 index 386fa0529..000000000 --- a/core/models/models/role_permission_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type RolePermissionV2 struct { - any `collection:"role_permissions"` - BaseModelV2[RolePermissionV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` -} diff --git a/core/models/models/role_v2.go b/core/models/models/role_v2.go deleted file mode 100644 index de287a61f..000000000 --- a/core/models/models/role_v2.go +++ /dev/null @@ -1,9 +0,0 @@ -package models - -type RoleV2 struct { - any `collection:"roles"` - BaseModelV2[RoleV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` -} diff --git a/core/models/models/schedule_v2.go b/core/models/models/schedule_v2.go deleted file mode 100644 index a52f1b920..000000000 --- a/core/models/models/schedule_v2.go +++ /dev/null @@ -1,23 +0,0 @@ -package models - -import ( - "github.com/robfig/cron/v3" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ScheduleV2 struct { - any `collection:"schedules"` - BaseModelV2[ScheduleV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` -} diff --git a/core/models/models/setting_v2.go b/core/models/models/setting_v2.go deleted file mode 100644 index 86481607e..000000000 --- a/core/models/models/setting_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson" -) - -type SettingV2 struct { - any `collection:"settings"` - BaseModelV2[SettingV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` -} diff --git a/core/models/models/spider_stat_v2.go b/core/models/models/spider_stat_v2.go deleted file mode 100644 index ea01c0cc6..000000000 --- a/core/models/models/spider_stat_v2.go +++ /dev/null @@ -1,20 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SpiderStatV2 struct { - any `collection:"spider_stats"` - BaseModelV2[SpiderStatV2] `bson:",inline"` - LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` - LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` - Tasks int `json:"tasks" bson:"tasks"` - Results int `json:"results" bson:"results"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second - AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second - AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second - AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second -} diff --git a/core/models/models/spider_v2.go b/core/models/models/spider_v2.go deleted file mode 100644 index ac3bf3007..000000000 --- a/core/models/models/spider_v2.go +++ /dev/null @@ -1,32 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SpiderV2 struct { - any `collection:"spiders"` - BaseModelV2[SpiderV2] `bson:",inline"` - Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id - ColName string `json:"col_name,omitempty" bson:"-"` // data collection name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id - GitRootPath string `json:"git_root_path" bson:"git_root_path"` - Git *GitV2 `json:"git,omitempty" bson:"-"` - - // stats - Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` - - // execution - Cmd string `json:"cmd" bson:"cmd"` // execute command - Param string `json:"param" bson:"param"` // default task param - Priority int `json:"priority" bson:"priority"` - AutoInstall bool `json:"auto_install" bson:"auto_install"` -} diff --git a/core/models/models/task_queue_item_v2.go b/core/models/models/task_queue_item_v2.go deleted file mode 100644 index f222aef3e..000000000 --- a/core/models/models/task_queue_item_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type TaskQueueItemV2 struct { - any `collection:"task_queue"` - BaseModelV2[TaskQueueItemV2] `bson:",inline"` - Priority int `json:"p" bson:"p"` - NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` -} diff --git a/core/models/models/task_stat_v2.go b/core/models/models/task_stat_v2.go deleted file mode 100644 index 5456946d4..000000000 --- a/core/models/models/task_stat_v2.go +++ /dev/null @@ -1,18 +0,0 @@ -package models - -import ( - "time" -) - -type TaskStatV2 struct { - any `collection:"task_stats"` - BaseModelV2[TaskStatV2] `bson:",inline"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` -} diff --git a/core/models/models/task_v2.go b/core/models/models/task_v2.go deleted file mode 100644 index 3473b6dba..000000000 --- a/core/models/models/task_v2.go +++ /dev/null @@ -1,30 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" - "time" -) - -type TaskV2 struct { - any `collection:"tasks"` - BaseModelV2[TaskV2] `bson:",inline"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` - Priority int `json:"priority" bson:"priority"` - Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` - HasSub bool `json:"has_sub" json:"has_sub"` - SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` - Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` - UserId primitive.ObjectID `json:"-" bson:"-"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` -} diff --git a/core/models/models/token_v2.go b/core/models/models/token_v2.go deleted file mode 100644 index c792c9ee4..000000000 --- a/core/models/models/token_v2.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -type TokenV2 struct { - any `collection:"tokens"` - BaseModelV2[TokenV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` -} diff --git a/core/models/models/user_role_v2.go b/core/models/models/user_role_v2.go deleted file mode 100644 index aa0ac34a0..000000000 --- a/core/models/models/user_role_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -import ( - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type UserRoleV2 struct { - any `collection:"user_roles"` - BaseModelV2[UserRoleV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` -} diff --git a/core/models/models/user_v2.go b/core/models/models/user_v2.go deleted file mode 100644 index acfbe2b64..000000000 --- a/core/models/models/user_v2.go +++ /dev/null @@ -1,10 +0,0 @@ -package models - -type UserV2 struct { - any `collection:"users"` - BaseModelV2[UserV2] `bson:",inline"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - Role string `json:"role" bson:"role"` - Email string `json:"email" bson:"email"` -} diff --git a/core/models/models/base_v2.go b/core/models/models/v2/base_v2.go similarity index 100% rename from core/models/models/base_v2.go rename to core/models/models/v2/base_v2.go diff --git a/core/models/models/data_collection_v2.go b/core/models/models/v2/data_collection_v2.go similarity index 100% rename from core/models/models/data_collection_v2.go rename to core/models/models/v2/data_collection_v2.go diff --git a/core/models/models/data_source_v2.go b/core/models/models/v2/data_source_v2.go similarity index 100% rename from core/models/models/data_source_v2.go rename to core/models/models/v2/data_source_v2.go diff --git a/core/models/models/dependency_log_v2.go b/core/models/models/v2/dependency_log_v2.go similarity index 100% rename from core/models/models/dependency_log_v2.go rename to core/models/models/v2/dependency_log_v2.go diff --git a/core/models/models/dependency_setting_v2.go b/core/models/models/v2/dependency_setting_v2.go similarity index 100% rename from core/models/models/dependency_setting_v2.go rename to core/models/models/v2/dependency_setting_v2.go diff --git a/core/models/models/dependency_task_v2.go b/core/models/models/v2/dependency_task_v2.go similarity index 100% rename from core/models/models/dependency_task_v2.go rename to core/models/models/v2/dependency_task_v2.go diff --git a/core/models/models/dependency_v2.go b/core/models/models/v2/dependency_v2.go similarity index 100% rename from core/models/models/dependency_v2.go rename to core/models/models/v2/dependency_v2.go diff --git a/core/models/models/environment_v2.go b/core/models/models/v2/environment_v2.go similarity index 100% rename from core/models/models/environment_v2.go rename to core/models/models/v2/environment_v2.go diff --git a/core/models/models/v2/git_v2.go b/core/models/models/v2/git_v2.go new file mode 100644 index 000000000..577be132d --- /dev/null +++ b/core/models/models/v2/git_v2.go @@ -0,0 +1,27 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/vcs" + "time" +) + +type GitV2 struct { + any `collection:"gits"` + models.BaseModelV2[GitV2] `bson:",inline"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + + // settings + AutoPull bool `json:"auto_pull" bson:"auto_pull"` +} diff --git a/core/models/models/v2/metric_v2.go b/core/models/models/v2/metric_v2.go new file mode 100644 index 000000000..08cc7749a --- /dev/null +++ b/core/models/models/v2/metric_v2.go @@ -0,0 +1,26 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type MetricV2 struct { + any `collection:"metrics"` + models.BaseModelV2[MetricV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` +} diff --git a/core/models/models/v2/node_v2.go b/core/models/models/v2/node_v2.go new file mode 100644 index 000000000..4031b97c7 --- /dev/null +++ b/core/models/models/v2/node_v2.go @@ -0,0 +1,25 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "time" +) + +type NodeV2 struct { + any `collection:"nodes"` + models.BaseModelV2[NodeV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Port string `json:"port" bson:"port"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts"` + AvailableRunners int `json:"available_runners" bson:"available_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` +} diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go new file mode 100644 index 000000000..2640e7f93 --- /dev/null +++ b/core/models/models/v2/notification_channel_v2.go @@ -0,0 +1 @@ +package models diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go new file mode 100644 index 000000000..d24796b89 --- /dev/null +++ b/core/models/models/v2/notification_setting_v2.go @@ -0,0 +1,33 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type NotificationSettingV2 struct { + any `collection:"notification_settings"` + models.BaseModelV2[NotificationSettingV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` +} + +type NotificationSettingMail struct { + Server string `json:"server" bson:"server"` + Port string `json:"port,omitempty" bson:"port,omitempty"` + User string `json:"user,omitempty" bson:"user,omitempty"` + Password string `json:"password,omitempty" bson:"password,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` + To string `json:"to,omitempty" bson:"to,omitempty"` + Cc string `json:"cc,omitempty" bson:"cc,omitempty"` +} + +type NotificationSettingMobile struct { + Webhook string `json:"webhook" bson:"webhook"` +} diff --git a/core/models/models/v2/permission_v2.go b/core/models/models/v2/permission_v2.go new file mode 100644 index 000000000..555583841 --- /dev/null +++ b/core/models/models/v2/permission_v2.go @@ -0,0 +1,15 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type PermissionV2 struct { + any `collection:"permissions"` + models.BaseModelV2[PermissionV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type string `json:"type" bson:"type"` + Target []string `json:"target" bson:"target"` + Allow []string `json:"allow" bson:"allow"` + Deny []string `json:"deny" bson:"deny"` +} diff --git a/core/models/models/v2/project_v2.go b/core/models/models/v2/project_v2.go new file mode 100644 index 000000000..fb1152c16 --- /dev/null +++ b/core/models/models/v2/project_v2.go @@ -0,0 +1,11 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type ProjectV2 struct { + any `collection:"projects"` + models.BaseModelV2[ProjectV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` +} diff --git a/core/models/models/v2/role_permission_v2.go b/core/models/models/v2/role_permission_v2.go new file mode 100644 index 000000000..8a77d2fbd --- /dev/null +++ b/core/models/models/v2/role_permission_v2.go @@ -0,0 +1,13 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type RolePermissionV2 struct { + any `collection:"role_permissions"` + models.BaseModelV2[RolePermissionV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` +} diff --git a/core/models/models/v2/role_v2.go b/core/models/models/v2/role_v2.go new file mode 100644 index 000000000..3deda9ebe --- /dev/null +++ b/core/models/models/v2/role_v2.go @@ -0,0 +1,11 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type RoleV2 struct { + any `collection:"roles"` + models.BaseModelV2[RoleV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` +} diff --git a/core/models/models/v2/schedule_v2.go b/core/models/models/v2/schedule_v2.go new file mode 100644 index 000000000..f304942b5 --- /dev/null +++ b/core/models/models/v2/schedule_v2.go @@ -0,0 +1,24 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ScheduleV2 struct { + any `collection:"schedules"` + models.BaseModelV2[ScheduleV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} diff --git a/core/models/models/v2/setting_v2.go b/core/models/models/v2/setting_v2.go new file mode 100644 index 000000000..bdb356947 --- /dev/null +++ b/core/models/models/v2/setting_v2.go @@ -0,0 +1,13 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson" +) + +type SettingV2 struct { + any `collection:"settings"` + models.BaseModelV2[SettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` +} diff --git a/core/models/models/v2/spider_stat_v2.go b/core/models/models/v2/spider_stat_v2.go new file mode 100644 index 000000000..3cdc8954c --- /dev/null +++ b/core/models/models/v2/spider_stat_v2.go @@ -0,0 +1,21 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderStatV2 struct { + any `collection:"spider_stats"` + models.BaseModelV2[SpiderStatV2] `bson:",inline"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` + Tasks int `json:"tasks" bson:"tasks"` + Results int `json:"results" bson:"results"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second +} diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go new file mode 100644 index 000000000..19cdf9c5a --- /dev/null +++ b/core/models/models/v2/spider_v2.go @@ -0,0 +1,33 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SpiderV2 struct { + any `collection:"spiders"` + models.BaseModelV2[SpiderV2] `bson:",inline"` + Name string `json:"name" bson:"name"` // spider name + Type string `json:"type" bson:"type"` // spider type + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id + ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *GitV2 `json:"git,omitempty" bson:"-"` + + // stats + Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` + + // execution + Cmd string `json:"cmd" bson:"cmd"` // execute command + Param string `json:"param" bson:"param"` // default task param + Priority int `json:"priority" bson:"priority"` + AutoInstall bool `json:"auto_install" bson:"auto_install"` +} diff --git a/core/models/models/v2/task_queue_item_v2.go b/core/models/models/v2/task_queue_item_v2.go new file mode 100644 index 000000000..28b2f88d2 --- /dev/null +++ b/core/models/models/v2/task_queue_item_v2.go @@ -0,0 +1,13 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type TaskQueueItemV2 struct { + any `collection:"task_queue"` + models.BaseModelV2[TaskQueueItemV2] `bson:",inline"` + Priority int `json:"p" bson:"p"` + NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` +} diff --git a/core/models/models/v2/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go new file mode 100644 index 000000000..5cf1fff18 --- /dev/null +++ b/core/models/models/v2/task_stat_v2.go @@ -0,0 +1,19 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "time" +) + +type TaskStatV2 struct { + any `collection:"task_stats"` + models.BaseModelV2[TaskStatV2] `bson:",inline"` + CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` + ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` +} diff --git a/core/models/models/v2/task_v2.go b/core/models/models/v2/task_v2.go new file mode 100644 index 000000000..31f46706a --- /dev/null +++ b/core/models/models/v2/task_v2.go @@ -0,0 +1,31 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type TaskV2 struct { + any `collection:"tasks"` + models.BaseModelV2[TaskV2] `bson:",inline"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` + Priority int `json:"priority" bson:"priority"` + Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` + HasSub bool `json:"has_sub" json:"has_sub"` + SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` + Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` +} diff --git a/core/models/models/test.go b/core/models/models/v2/test_v2.go similarity index 100% rename from core/models/models/test.go rename to core/models/models/v2/test_v2.go diff --git a/core/models/models/v2/token_v2.go b/core/models/models/v2/token_v2.go new file mode 100644 index 000000000..1fe20d886 --- /dev/null +++ b/core/models/models/v2/token_v2.go @@ -0,0 +1,10 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type TokenV2 struct { + any `collection:"tokens"` + models.BaseModelV2[TokenV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` +} diff --git a/core/models/models/v2/user_role_v2.go b/core/models/models/v2/user_role_v2.go new file mode 100644 index 000000000..e36e2f702 --- /dev/null +++ b/core/models/models/v2/user_role_v2.go @@ -0,0 +1,13 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type UserRoleV2 struct { + any `collection:"user_roles"` + models.BaseModelV2[UserRoleV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` +} diff --git a/core/models/models/v2/user_v2.go b/core/models/models/v2/user_v2.go new file mode 100644 index 000000000..131159aae --- /dev/null +++ b/core/models/models/v2/user_v2.go @@ -0,0 +1,12 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type UserV2 struct { + any `collection:"users"` + models.BaseModelV2[UserV2] `bson:",inline"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` +} diff --git a/core/models/models/v2/variable_v2.go b/core/models/models/v2/variable_v2.go new file mode 100644 index 000000000..cd8be4b4d --- /dev/null +++ b/core/models/models/v2/variable_v2.go @@ -0,0 +1,11 @@ +package models + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type VariableV2 struct { + any `collection:"variables"` + models.BaseModelV2[VariableV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Remark string `json:"remark" bson:"remark"` +} diff --git a/core/models/models/variable_v2.go b/core/models/models/variable_v2.go deleted file mode 100644 index 497888322..000000000 --- a/core/models/models/variable_v2.go +++ /dev/null @@ -1,9 +0,0 @@ -package models - -type VariableV2 struct { - any `collection:"variables"` - BaseModelV2[VariableV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` - Remark string `json:"remark" bson:"remark"` -} From aca0c0ebce8bdf7cdcec93b94e3b14a46c3507e8 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 11 Jul 2024 12:45:29 +0800 Subject: [PATCH 048/106] refactor: removed unnecessary code --- core/controllers/base_v2.go | 6 ++ core/controllers/base_v2_test.go | 4 +- core/controllers/spider_v2_test.go | 2 +- core/ds/service_v2.go | 2 +- core/grpc/server/dependencies_server_v2.go | 28 +++++----- core/grpc/server/metrics_server_v2.go | 8 +-- .../server/model_base_service_v2_server.go | 56 +++++++++---------- core/grpc/server/node_server_v2.go | 2 +- core/grpc/server/task_server_v2.go | 18 +++--- core/models/client/model_service_v2_test.go | 4 +- core/models/common/index_service_v2.go | 44 +++++++-------- core/models/models/v2/data_collection_v2.go | 10 ++-- core/models/models/v2/data_source_v2.go | 32 +++++------ .../models/models/v2/dependency_setting_v2.go | 18 +++--- core/models/models/v2/git_v2.go | 29 +++++----- core/models/models/v2/metric_v2.go | 35 ++++++------ core/models/models/v2/node_v2.go | 33 ++++++----- .../models/v2/notification_channel_v2.go | 3 + .../models/v2/notification_setting_v2.go | 26 ++++----- core/models/models/v2/permission_v2.go | 20 +++---- core/models/models/v2/project_v2.go | 12 ++-- core/models/models/v2/role_permission_v2.go | 9 ++- core/models/models/v2/role_v2.go | 12 ++-- core/models/models/v2/schedule_v2.go | 29 +++++----- core/models/models/v2/setting_v2.go | 9 ++- core/models/models/v2/spider_stat_v2.go | 25 ++++----- core/models/models/v2/spider_v2.go | 31 +++++----- core/models/models/v2/task_queue_item_v2.go | 9 ++- core/models/models/v2/task_stat_v2.go | 21 ++++--- core/models/models/v2/task_v2.go | 43 +++++++------- core/models/models/v2/test_v2.go | 8 +-- core/models/models/v2/token_v2.go | 10 ++-- core/models/models/v2/user_role_v2.go | 9 ++- core/models/models/v2/user_v2.go | 14 ++--- core/models/models/v2/variable_v2.go | 12 ++-- core/models/service/base_service_v2_test.go | 2 +- core/node/service/master_service_v2.go | 34 +++++------ core/node/service/worker_service_v2.go | 5 +- core/notification/mail.go | 2 +- core/notification/service.go | 2 +- core/notification/service_v2.go | 2 +- core/schedule/service_v2.go | 14 ++--- core/spider/admin/service_v2.go | 14 ++--- core/system/service_v2.go | 2 +- core/task/handler/runner_v2.go | 23 ++++---- core/task/handler/service_v2.go | 30 +++++----- core/task/scheduler/service_v2.go | 42 +++++++------- core/task/stats/service_v2.go | 6 +- core/user/service_v2.go | 2 +- core/utils/cockroachdb.go | 5 +- core/utils/es.go | 5 +- core/utils/kafka.go | 5 +- core/utils/mongo.go | 5 +- core/utils/mssql.go | 5 +- core/utils/mysql.go | 5 +- core/utils/postgresql.go | 5 +- core/utils/sqlite.go | 5 +- 57 files changed, 423 insertions(+), 430 deletions(-) diff --git a/core/controllers/base_v2.go b/core/controllers/base_v2.go index be6eebff5..82130a3c5 100644 --- a/core/controllers/base_v2.go +++ b/core/controllers/base_v2.go @@ -11,6 +11,12 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) +type Action struct { + Method string + Path string + HandlerFunc gin.HandlerFunc +} + type BaseControllerV2[T any] struct { modelSvc *service.ModelServiceV2[T] actions []Action diff --git a/core/controllers/base_v2_test.go b/core/controllers/base_v2_test.go index 3e0351ef8..5259ca9ad 100644 --- a/core/controllers/base_v2_test.go +++ b/core/controllers/base_v2_test.go @@ -6,7 +6,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/spf13/viper" @@ -24,7 +24,7 @@ func init() { } // TestModel is a simple struct to be used as a model in tests -type TestModel models.TestModel +type TestModel models.TestModelV2 var TestToken string diff --git a/core/controllers/spider_v2_test.go b/core/controllers/spider_v2_test.go index 69c50f280..a744ba852 100644 --- a/core/controllers/spider_v2_test.go +++ b/core/controllers/spider_v2_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" diff --git a/core/ds/service_v2.go b/core/ds/service_v2.go index 120939494..e96a96453 100644 --- a/core/ds/service_v2.go +++ b/core/ds/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" constants2 "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/utils" diff --git a/core/grpc/server/dependencies_server_v2.go b/core/grpc/server/dependencies_server_v2.go index f7ede5c05..32c397607 100644 --- a/core/grpc/server/dependencies_server_v2.go +++ b/core/grpc/server/dependencies_server_v2.go @@ -4,7 +4,7 @@ import ( "context" "errors" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" mongo2 "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/grpc" @@ -40,12 +40,12 @@ func (svr DependenciesServerV2) Connect(req *grpc.DependenciesServiceV2ConnectRe } func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) { - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) if err != nil { return nil, err } - depsDb, err := service.NewModelServiceV2[models.DependencyV2]().GetMany(bson.M{ + depsDb, err := service.NewModelServiceV2[models2.DependencyV2]().GetMany(bson.M{ "node_id": n.Id, "type": request.Lang, }, nil) @@ -56,15 +56,15 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen } } - depsDbMap := make(map[string]*models.DependencyV2) + depsDbMap := make(map[string]*models2.DependencyV2) for _, d := range depsDb { depsDbMap[d.Name] = &d } - var depsToInsert []models.DependencyV2 - depsMap := make(map[string]*models.DependencyV2) + var depsToInsert []models2.DependencyV2 + depsMap := make(map[string]*models2.DependencyV2) for _, dep := range request.Dependencies { - d := models.DependencyV2{ + d := models2.DependencyV2{ Name: dep.Name, NodeId: n.Id, Type: request.Lang, @@ -90,7 +90,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen err = mongo2.RunTransaction(func(ctx mongo.SessionContext) (err error) { if len(depIdsToDelete) > 0 { - err = service.NewModelServiceV2[models.DependencyV2]().DeleteMany(bson.M{ + err = service.NewModelServiceV2[models2.DependencyV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": depIdsToDelete}, }) if err != nil { @@ -101,7 +101,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen } if len(depsToInsert) > 0 { - _, err = service.NewModelServiceV2[models.DependencyV2]().InsertMany(depsToInsert) + _, err = service.NewModelServiceV2[models2.DependencyV2]().InsertMany(depsToInsert) if err != nil { log.Errorf("[DependenciesServerV2] insert dependencies in db error: %v", err) trace.PrintError(err) @@ -116,7 +116,7 @@ func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.Dependen } func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_UpdateTaskLogServer) (err error) { - var t *models.DependencyTaskV2 + var t *models2.DependencyTaskV2 for { req, err := stream.Recv() if err == io.EOF { @@ -131,21 +131,21 @@ func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_ return err } if t == nil { - t, err = service.NewModelServiceV2[models.DependencyTaskV2]().GetById(taskId) + t, err = service.NewModelServiceV2[models2.DependencyTaskV2]().GetById(taskId) if err != nil { return err } } - var logs []models.DependencyLogV2 + var logs []models2.DependencyLogV2 for _, line := range req.LogLines { - l := models.DependencyLogV2{ + l := models2.DependencyLogV2{ TaskId: taskId, Content: line, } l.SetCreated(t.CreatedBy) logs = append(logs, l) } - _, err = service.NewModelServiceV2[models.DependencyLogV2]().InsertMany(logs) + _, err = service.NewModelServiceV2[models2.DependencyLogV2]().InsertMany(logs) if err != nil { return err } diff --git a/core/grpc/server/metrics_server_v2.go b/core/grpc/server/metrics_server_v2.go index fc51ad01a..60a925695 100644 --- a/core/grpc/server/metrics_server_v2.go +++ b/core/grpc/server/metrics_server_v2.go @@ -3,7 +3,7 @@ package server import ( "context" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" @@ -17,12 +17,12 @@ type MetricsServerV2 struct { func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2SendRequest) (res *grpc.Response, err error) { log.Info("[MetricsServerV2] received metric from node: " + req.NodeKey) - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) if err != nil { log.Errorf("[MetricsServerV2] error getting node: %v", err) return HandleError(err) } - metric := models.MetricV2{ + metric := models2.MetricV2{ Type: req.Type, NodeId: n.Id, CpuUsagePercent: req.CpuUsagePercent, @@ -40,7 +40,7 @@ func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2Sen NetworkBytesRecvRate: req.NetworkBytesRecvRate, } metric.CreatedAt = time.Unix(req.Timestamp, 0) - _, err = service.NewModelServiceV2[models.MetricV2]().InsertOne(metric) + _, err = service.NewModelServiceV2[models2.MetricV2]().InsertOne(metric) if err != nil { log.Errorf("[MetricsServerV2] error inserting metric: %v", err) return HandleError(err) diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index f777da61a..eefd389a6 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -3,10 +3,10 @@ package server import ( "context" "encoding/json" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "reflect" @@ -16,32 +16,32 @@ var ( typeNameColNameMap = make(map[string]string) typeOneNameModelMap = make(map[string]any) typeOneInstances = []any{ - *new(models.TestModel), - *new(models.DataCollectionV2), - *new(models.DataSourceV2), - *new(models.DependencyV2), - *new(models.DependencyLogV2), - *new(models.DependencySettingV2), - *new(models.DependencyTaskV2), - *new(models.EnvironmentV2), - *new(models.GitV2), - *new(models.MetricV2), - *new(models.NodeV2), - *new(models.NotificationSettingV2), - *new(models.PermissionV2), - *new(models.ProjectV2), - *new(models.RolePermissionV2), - *new(models.RoleV2), - *new(models.ScheduleV2), - *new(models.SettingV2), - *new(models.SpiderV2), - *new(models.SpiderStatV2), - *new(models.TaskQueueItemV2), - *new(models.TaskStatV2), - *new(models.TaskV2), - *new(models.TokenV2), - *new(models.UserRoleV2), - *new(models.UserV2), + *new(models2.TestModelV2), + *new(models2.DataCollectionV2), + *new(models2.DataSourceV2), + *new(models2.DependencyV2), + *new(models2.DependencyLogV2), + *new(models2.DependencySettingV2), + *new(models2.DependencyTaskV2), + *new(models2.EnvironmentV2), + *new(models2.GitV2), + *new(models2.MetricV2), + *new(models2.NodeV2), + *new(models2.NotificationSettingV2), + *new(models2.PermissionV2), + *new(models2.ProjectV2), + *new(models2.RolePermissionV2), + *new(models2.RoleV2), + *new(models2.ScheduleV2), + *new(models2.SettingV2), + *new(models2.SpiderV2), + *new(models2.SpiderStatV2), + *new(models2.TaskQueueItemV2), + *new(models2.TaskStatV2), + *new(models2.TaskV2), + *new(models2.TokenV2), + *new(models2.UserRoleV2), + *new(models2.UserV2), } ) diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index a45e222e4..4ef7de742 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/grpc" diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index 62b6ab328..806914929 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -72,7 +72,7 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo if nodeKey == "" { return nil, errors.New("invalid node key") } - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return nil, trace.TraceError(err) } @@ -111,11 +111,11 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { svc := notification.GetNotificationServiceV2() - var t = new(models.TaskV2) + var t = new(models2.TaskV2) if err := json.Unmarshal(request.Data, t); err != nil { return nil, trace.TraceError(err) } - t, err = service.NewModelServiceV2[models.TaskV2]().GetById(t.Id) + t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(t.Id) if err != nil { return nil, trace.TraceError(err) } @@ -127,7 +127,7 @@ func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Requ if err := json.Unmarshal(td, &e); err != nil { return nil, trace.TraceError(err) } - ts, err := service.NewModelServiceV2[models.TaskStatV2]().GetById(t.Id) + ts, err := service.NewModelServiceV2[models2.TaskStatV2]().GetById(t.Id) if err != nil { return nil, trace.TraceError(err) } @@ -190,22 +190,22 @@ func (svr TaskServerV2) handleInsertLogs(msg *grpc.StreamMessage) (err error) { } func (svr TaskServerV2) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { - tq, err := service.NewModelServiceV2[models.TaskQueueItemV2]().GetOne(query, opts) + tq, err := service.NewModelServiceV2[models2.TaskQueueItemV2]().GetOne(query, opts) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return tid, nil } return tid, trace.TraceError(err) } - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(tq.Id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(tq.Id) if err == nil { t.NodeId = nid - err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + err = service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) if err != nil { return tid, trace.TraceError(err) } } - err = service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(tq.Id) + err = service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(tq.Id) if err != nil { return tid, trace.TraceError(err) } diff --git a/core/models/client/model_service_v2_test.go b/core/models/client/model_service_v2_test.go index c4053bb14..7bd5ac5fd 100644 --- a/core/models/client/model_service_v2_test.go +++ b/core/models/client/model_service_v2_test.go @@ -4,7 +4,7 @@ import ( "context" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" @@ -17,7 +17,7 @@ import ( "time" ) -type TestModel models.TestModel +type TestModel models.TestModelV2 func setupTestDB() { viper.Set("mongo.db", "testdb") diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index e87ec7f35..9db29c88c 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -1,7 +1,7 @@ package common import ( - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" @@ -11,7 +11,7 @@ import ( func CreateIndexesV2() { // nodes - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, // key {Keys: bson.M{"name": 1}}, // name {Keys: bson.M{"is_master": 1}}, // is_master @@ -21,12 +21,12 @@ func CreateIndexesV2() { }) // projects - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // spiders - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, {Keys: bson.M{"type": 1}}, {Keys: bson.M{"col_id": 1}}, @@ -34,7 +34,7 @@ func CreateIndexesV2() { }) // tasks - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"spider_id": 1}}, {Keys: bson.M{"status": 1}}, {Keys: bson.M{"node_id": 1}}, @@ -48,73 +48,73 @@ func CreateIndexesV2() { }) // task stats - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"create_ts": 1}}, }) // schedules - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, {Keys: bson.M{"spider_id": 1}}, {Keys: bson.M{"enabled": 1}}, }) // users - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"username": 1}}, {Keys: bson.M{"role": 1}}, {Keys: bson.M{"email": 1}}, }) // settings - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, }) // tokens - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // variables - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"key": 1}}, }) // data sources - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // data collections - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) // roles - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, }) // user role relations - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, }) // permissions - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, }) // role permission relations - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, }) // dependencies - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"type", 1}, @@ -126,7 +126,7 @@ func CreateIndexesV2() { }) // dependency settings - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"type", 1}, @@ -138,7 +138,7 @@ func CreateIndexesV2() { }) // dependency logs - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{{"task_id", 1}}, }, @@ -149,7 +149,7 @@ func CreateIndexesV2() { }) // dependency tasks - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"update_ts", 1}, @@ -159,7 +159,7 @@ func CreateIndexesV2() { }) // metrics - mongo.GetMongoCol(service.GetCollectionNameByInstance(models.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ { Keys: bson.D{ {"created_ts", -1}, diff --git a/core/models/models/v2/data_collection_v2.go b/core/models/models/v2/data_collection_v2.go index 1a3c724e3..8ec5e8cb9 100644 --- a/core/models/models/v2/data_collection_v2.go +++ b/core/models/models/v2/data_collection_v2.go @@ -5,11 +5,11 @@ import ( ) type DataCollectionV2 struct { - any `collection:"data_collections"` - BaseModelV2[DataCollection] `bson:",inline"` - Name string `json:"name" bson:"name"` - Fields []entity.DataField `json:"fields" bson:"fields"` - Dedup struct { + any `collection:"data_collections"` + BaseModelV2[DataCollectionV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { Enabled bool `json:"enabled" bson:"enabled"` Keys []string `json:"keys" bson:"keys"` Type string `json:"type" bson:"type"` diff --git a/core/models/models/v2/data_source_v2.go b/core/models/models/v2/data_source_v2.go index 16415510c..7e9201cd6 100644 --- a/core/models/models/v2/data_source_v2.go +++ b/core/models/models/v2/data_source_v2.go @@ -1,20 +1,20 @@ package models type DataSourceV2 struct { - any `collection:"data_sources"` - BaseModelV2[DataSource] `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - Description string `json:"description" bson:"description"` - Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` - Url string `json:"url" bson:"url"` - Hosts []string `json:"hosts" bson:"hosts"` - Database string `json:"database" bson:"database"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - ConnectType string `json:"connect_type" bson:"connect_type"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` + any `collection:"data_sources"` + BaseModelV2[DataSourceV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + Description string `json:"description" bson:"description"` + Host string `json:"host" bson:"host"` + Port string `json:"port" bson:"port"` + Url string `json:"url" bson:"url"` + Hosts []string `json:"hosts" bson:"hosts"` + Database string `json:"database" bson:"database"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + ConnectType string `json:"connect_type" bson:"connect_type"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` } diff --git a/core/models/models/v2/dependency_setting_v2.go b/core/models/models/v2/dependency_setting_v2.go index d4c241e69..528c44d0a 100644 --- a/core/models/models/v2/dependency_setting_v2.go +++ b/core/models/models/v2/dependency_setting_v2.go @@ -5,13 +5,13 @@ import ( ) type DependencySettingV2 struct { - any `collection:"dependency_settings"` - BaseModelV2[DependencySetting] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Cmd string `json:"cmd" bson:"cmd"` - Proxy string `json:"proxy" bson:"proxy"` - LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` + any `collection:"dependency_settings"` + BaseModelV2[DependencySettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Cmd string `json:"cmd" bson:"cmd"` + Proxy string `json:"proxy" bson:"proxy"` + LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` } diff --git a/core/models/models/v2/git_v2.go b/core/models/models/v2/git_v2.go index 577be132d..730c2c89d 100644 --- a/core/models/models/v2/git_v2.go +++ b/core/models/models/v2/git_v2.go @@ -1,26 +1,25 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/vcs" "time" ) type GitV2 struct { - any `collection:"gits"` - models.BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - Name string `json:"name" bson:"name"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` - Refs []vcs.GitRef `json:"refs" bson:"refs"` - RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` - CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + any `collection:"gits"` + BaseModelV2[GitV2] `bson:",inline"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` // settings AutoPull bool `json:"auto_pull" bson:"auto_pull"` diff --git a/core/models/models/v2/metric_v2.go b/core/models/models/v2/metric_v2.go index 08cc7749a..8d23dd91a 100644 --- a/core/models/models/v2/metric_v2.go +++ b/core/models/models/v2/metric_v2.go @@ -1,26 +1,25 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type MetricV2 struct { - any `collection:"metrics"` - models.BaseModelV2[MetricV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` - TotalMemory uint64 `json:"total_memory" bson:"total_memory"` - AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` - UsedMemory uint64 `json:"used_memory" bson:"used_memory"` - UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` - TotalDisk uint64 `json:"total_disk" bson:"total_disk"` - AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` - UsedDisk uint64 `json:"used_disk" bson:"used_disk"` - UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` - DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` - DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` - NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` - NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` + any `collection:"metrics"` + BaseModelV2[MetricV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` } diff --git a/core/models/models/v2/node_v2.go b/core/models/models/v2/node_v2.go index 4031b97c7..6d5a84421 100644 --- a/core/models/models/v2/node_v2.go +++ b/core/models/models/v2/node_v2.go @@ -1,25 +1,24 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "time" ) type NodeV2 struct { - any `collection:"nodes"` - models.BaseModelV2[NodeV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` - Mac string `json:"mac" bson:"mac"` - Hostname string `json:"hostname" bson:"hostname"` - Description string `json:"description" bson:"description"` - IsMaster bool `json:"is_master" bson:"is_master"` - Status string `json:"status" bson:"status"` - Enabled bool `json:"enabled" bson:"enabled"` - Active bool `json:"active" bson:"active"` - ActiveAt time.Time `json:"active_at" bson:"active_ts"` - AvailableRunners int `json:"available_runners" bson:"available_runners"` - MaxRunners int `json:"max_runners" bson:"max_runners"` + any `collection:"nodes"` + BaseModelV2[NodeV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Ip string `json:"ip" bson:"ip"` + Port string `json:"port" bson:"port"` + Mac string `json:"mac" bson:"mac"` + Hostname string `json:"hostname" bson:"hostname"` + Description string `json:"description" bson:"description"` + IsMaster bool `json:"is_master" bson:"is_master"` + Status string `json:"status" bson:"status"` + Enabled bool `json:"enabled" bson:"enabled"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_at" bson:"active_ts"` + AvailableRunners int `json:"available_runners" bson:"available_runners"` + MaxRunners int `json:"max_runners" bson:"max_runners"` } diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 2640e7f93..cd3172219 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -1 +1,4 @@ package models + +type NotificationChannel struct { +} diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index d24796b89..b07a01970 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -1,20 +1,18 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type NotificationSettingV2 struct { - any `collection:"notification_settings"` - models.BaseModelV2[NotificationSettingV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` + any `collection:"notification_settings"` + BaseModelV2[NotificationSettingV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` } type NotificationSettingMail struct { diff --git a/core/models/models/v2/permission_v2.go b/core/models/models/v2/permission_v2.go index 555583841..77c830a38 100644 --- a/core/models/models/v2/permission_v2.go +++ b/core/models/models/v2/permission_v2.go @@ -1,15 +1,13 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type PermissionV2 struct { - any `collection:"permissions"` - models.BaseModelV2[PermissionV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Type string `json:"type" bson:"type"` - Target []string `json:"target" bson:"target"` - Allow []string `json:"allow" bson:"allow"` - Deny []string `json:"deny" bson:"deny"` + any `collection:"permissions"` + BaseModelV2[PermissionV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type string `json:"type" bson:"type"` + Target []string `json:"target" bson:"target"` + Allow []string `json:"allow" bson:"allow"` + Deny []string `json:"deny" bson:"deny"` } diff --git a/core/models/models/v2/project_v2.go b/core/models/models/v2/project_v2.go index fb1152c16..ee07e705a 100644 --- a/core/models/models/v2/project_v2.go +++ b/core/models/models/v2/project_v2.go @@ -1,11 +1,9 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type ProjectV2 struct { - any `collection:"projects"` - models.BaseModelV2[ProjectV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Spiders int `json:"spiders" bson:"-"` + any `collection:"projects"` + BaseModelV2[ProjectV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Spiders int `json:"spiders" bson:"-"` } diff --git a/core/models/models/v2/role_permission_v2.go b/core/models/models/v2/role_permission_v2.go index 8a77d2fbd..386fa0529 100644 --- a/core/models/models/v2/role_permission_v2.go +++ b/core/models/models/v2/role_permission_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type RolePermissionV2 struct { - any `collection:"role_permissions"` - models.BaseModelV2[RolePermissionV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` + any `collection:"role_permissions"` + BaseModelV2[RolePermissionV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + PermissionId primitive.ObjectID `json:"permission_id" bson:"permission_id"` } diff --git a/core/models/models/v2/role_v2.go b/core/models/models/v2/role_v2.go index 3deda9ebe..de287a61f 100644 --- a/core/models/models/v2/role_v2.go +++ b/core/models/models/v2/role_v2.go @@ -1,11 +1,9 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type RoleV2 struct { - any `collection:"roles"` - models.BaseModelV2[RoleV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` + any `collection:"roles"` + BaseModelV2[RoleV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` } diff --git a/core/models/models/v2/schedule_v2.go b/core/models/models/v2/schedule_v2.go index f304942b5..a52f1b920 100644 --- a/core/models/models/v2/schedule_v2.go +++ b/core/models/models/v2/schedule_v2.go @@ -1,24 +1,23 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/robfig/cron/v3" "go.mongodb.org/mongo-driver/bson/primitive" ) type ScheduleV2 struct { - any `collection:"schedules"` - models.BaseModelV2[ScheduleV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Cron string `json:"cron" bson:"cron"` - EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - Priority int `json:"priority" bson:"priority"` - Enabled bool `json:"enabled" bson:"enabled"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` + any `collection:"schedules"` + BaseModelV2[ScheduleV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Cron string `json:"cron" bson:"cron"` + EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + Priority int `json:"priority" bson:"priority"` + Enabled bool `json:"enabled" bson:"enabled"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/models/v2/setting_v2.go b/core/models/models/v2/setting_v2.go index bdb356947..86481607e 100644 --- a/core/models/models/v2/setting_v2.go +++ b/core/models/models/v2/setting_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson" ) type SettingV2 struct { - any `collection:"settings"` - models.BaseModelV2[SettingV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value bson.M `json:"value" bson:"value"` + any `collection:"settings"` + BaseModelV2[SettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value bson.M `json:"value" bson:"value"` } diff --git a/core/models/models/v2/spider_stat_v2.go b/core/models/models/v2/spider_stat_v2.go index 3cdc8954c..ea01c0cc6 100644 --- a/core/models/models/v2/spider_stat_v2.go +++ b/core/models/models/v2/spider_stat_v2.go @@ -1,21 +1,20 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type SpiderStatV2 struct { - any `collection:"spider_stats"` - models.BaseModelV2[SpiderStatV2] `bson:",inline"` - LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` - LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` - Tasks int `json:"tasks" bson:"tasks"` - Results int `json:"results" bson:"results"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second - AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second - AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second - AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second + any `collection:"spider_stats"` + BaseModelV2[SpiderStatV2] `bson:",inline"` + LastTaskId primitive.ObjectID `json:"last_task_id" bson:"last_task_id,omitempty"` + LastTask *TaskV2 `json:"last_task,omitempty" bson:"-"` + Tasks int `json:"tasks" bson:"tasks"` + Results int `json:"results" bson:"results"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in second + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in second + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in second + AverageWaitDuration int64 `json:"average_wait_duration" bson:"-"` // in second + AverageRuntimeDuration int64 `json:"average_runtime_duration" bson:"-"` // in second + AverageTotalDuration int64 `json:"average_total_duration" bson:"-"` // in second } diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go index 19cdf9c5a..ac3bf3007 100644 --- a/core/models/models/v2/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -1,26 +1,25 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type SpiderV2 struct { - any `collection:"spiders"` - models.BaseModelV2[SpiderV2] `bson:",inline"` - Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id - ColName string `json:"col_name,omitempty" bson:"-"` // data collection name - DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *models.DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source - Description string `json:"description" bson:"description"` // description - ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id - Mode string `json:"mode" bson:"mode"` // default Task.Mode - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id - GitRootPath string `json:"git_root_path" bson:"git_root_path"` - Git *GitV2 `json:"git,omitempty" bson:"-"` + any `collection:"spiders"` + BaseModelV2[SpiderV2] `bson:",inline"` + Name string `json:"name" bson:"name"` // spider name + Type string `json:"type" bson:"type"` // spider type + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id + ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id + DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source + Description string `json:"description" bson:"description"` // description + ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id + Mode string `json:"mode" bson:"mode"` // default Task.Mode + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *GitV2 `json:"git,omitempty" bson:"-"` // stats Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` diff --git a/core/models/models/v2/task_queue_item_v2.go b/core/models/models/v2/task_queue_item_v2.go index 28b2f88d2..f222aef3e 100644 --- a/core/models/models/v2/task_queue_item_v2.go +++ b/core/models/models/v2/task_queue_item_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type TaskQueueItemV2 struct { - any `collection:"task_queue"` - models.BaseModelV2[TaskQueueItemV2] `bson:",inline"` - Priority int `json:"p" bson:"p"` - NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` + any `collection:"task_queue"` + BaseModelV2[TaskQueueItemV2] `bson:",inline"` + Priority int `json:"p" bson:"p"` + NodeId primitive.ObjectID `json:"nid,omitempty" bson:"nid,omitempty"` } diff --git a/core/models/models/v2/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go index 5cf1fff18..5456946d4 100644 --- a/core/models/models/v2/task_stat_v2.go +++ b/core/models/models/v2/task_stat_v2.go @@ -1,19 +1,18 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "time" ) type TaskStatV2 struct { - any `collection:"task_stats"` - models.BaseModelV2[TaskStatV2] `bson:",inline"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` - StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` - EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` - WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond - RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond - TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond - ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` + any `collection:"task_stats"` + BaseModelV2[TaskStatV2] `bson:",inline"` + CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` + StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` + EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` + WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond + RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond + TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond + ResultCount int64 `json:"result_count" bson:"result_count"` + ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` } diff --git a/core/models/models/v2/task_v2.go b/core/models/models/v2/task_v2.go index 31f46706a..3473b6dba 100644 --- a/core/models/models/v2/task_v2.go +++ b/core/models/models/v2/task_v2.go @@ -1,31 +1,30 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" "time" ) type TaskV2 struct { - any `collection:"tasks"` - models.BaseModelV2[TaskV2] `bson:",inline"` - SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` - Status string `json:"status" bson:"status"` - NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` - Cmd string `json:"cmd" bson:"cmd"` - Param string `json:"param" bson:"param"` - Error string `json:"error" bson:"error"` - Pid int `json:"pid" bson:"pid"` - ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` - Type string `json:"type" bson:"type"` - Mode string `json:"mode" bson:"mode"` - NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` - ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` - Priority int `json:"priority" bson:"priority"` - Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` - HasSub bool `json:"has_sub" json:"has_sub"` - SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` - Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` - UserId primitive.ObjectID `json:"-" bson:"-"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` + any `collection:"tasks"` + BaseModelV2[TaskV2] `bson:",inline"` + SpiderId primitive.ObjectID `json:"spider_id" bson:"spider_id"` + Status string `json:"status" bson:"status"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` + Error string `json:"error" bson:"error"` + Pid int `json:"pid" bson:"pid"` + ScheduleId primitive.ObjectID `json:"schedule_id" bson:"schedule_id"` + Type string `json:"type" bson:"type"` + Mode string `json:"mode" bson:"mode"` + NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` + ParentId primitive.ObjectID `json:"parent_id" bson:"parent_id"` + Priority int `json:"priority" bson:"priority"` + Stat *TaskStatV2 `json:"stat,omitempty" bson:"-"` + HasSub bool `json:"has_sub" json:"has_sub"` + SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` + Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` + UserId primitive.ObjectID `json:"-" bson:"-"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` } diff --git a/core/models/models/v2/test_v2.go b/core/models/models/v2/test_v2.go index 67027d19c..1468abc15 100644 --- a/core/models/models/v2/test_v2.go +++ b/core/models/models/v2/test_v2.go @@ -1,7 +1,7 @@ package models -type TestModel struct { - any `collection:"testmodels"` - BaseModelV2[TestModel] `bson:",inline"` - Name string `json:"name" bson:"name"` +type TestModelV2 struct { + any `collection:"testmodels"` + BaseModelV2[TestModelV2] `bson:",inline"` + Name string `json:"name" bson:"name"` } diff --git a/core/models/models/v2/token_v2.go b/core/models/models/v2/token_v2.go index 1fe20d886..c792c9ee4 100644 --- a/core/models/models/v2/token_v2.go +++ b/core/models/models/v2/token_v2.go @@ -1,10 +1,8 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type TokenV2 struct { - any `collection:"tokens"` - models.BaseModelV2[TokenV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Token string `json:"token" bson:"token"` + any `collection:"tokens"` + BaseModelV2[TokenV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Token string `json:"token" bson:"token"` } diff --git a/core/models/models/v2/user_role_v2.go b/core/models/models/v2/user_role_v2.go index e36e2f702..aa0ac34a0 100644 --- a/core/models/models/v2/user_role_v2.go +++ b/core/models/models/v2/user_role_v2.go @@ -1,13 +1,12 @@ package models import ( - "github.com/crawlab-team/crawlab/core/models/models/v2" "go.mongodb.org/mongo-driver/bson/primitive" ) type UserRoleV2 struct { - any `collection:"user_roles"` - models.BaseModelV2[UserRoleV2] `bson:",inline"` - RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` + any `collection:"user_roles"` + BaseModelV2[UserRoleV2] `bson:",inline"` + RoleId primitive.ObjectID `json:"role_id" bson:"role_id"` + UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/models/v2/user_v2.go b/core/models/models/v2/user_v2.go index 131159aae..acfbe2b64 100644 --- a/core/models/models/v2/user_v2.go +++ b/core/models/models/v2/user_v2.go @@ -1,12 +1,10 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type UserV2 struct { - any `collection:"users"` - models.BaseModelV2[UserV2] `bson:",inline"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - Role string `json:"role" bson:"role"` - Email string `json:"email" bson:"email"` + any `collection:"users"` + BaseModelV2[UserV2] `bson:",inline"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + Role string `json:"role" bson:"role"` + Email string `json:"email" bson:"email"` } diff --git a/core/models/models/v2/variable_v2.go b/core/models/models/v2/variable_v2.go index cd8be4b4d..497888322 100644 --- a/core/models/models/v2/variable_v2.go +++ b/core/models/models/v2/variable_v2.go @@ -1,11 +1,9 @@ package models -import "github.com/crawlab-team/crawlab/core/models/models/v2" - type VariableV2 struct { - any `collection:"variables"` - models.BaseModelV2[VariableV2] `bson:",inline"` - Key string `json:"key" bson:"key"` - Value string `json:"value" bson:"value"` - Remark string `json:"remark" bson:"remark"` + any `collection:"variables"` + BaseModelV2[VariableV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Value string `json:"value" bson:"value"` + Remark string `json:"remark" bson:"remark"` } diff --git a/core/models/service/base_service_v2_test.go b/core/models/service/base_service_v2_test.go index 18eec6975..2d23cb47a 100644 --- a/core/models/service/base_service_v2_test.go +++ b/core/models/service/base_service_v2_test.go @@ -2,7 +2,7 @@ package service_test import ( "context" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index cbf2249d6..73f9cf344 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -9,7 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -142,11 +142,11 @@ func (svc *MasterServiceV2) SetMonitorInterval(duration time.Duration) { func (svc *MasterServiceV2) Register() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() nodeName := svc.GetConfigService().GetNodeName() - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { // not exists log.Infof("master[%s] does not exist in db", nodeKey) - node := models.NodeV2{ + node := models2.NodeV2{ Key: nodeKey, Name: nodeName, MaxRunners: config.DefaultConfigOptions.MaxRunners, @@ -158,7 +158,7 @@ func (svc *MasterServiceV2) Register() (err error) { } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - id, err := service.NewModelServiceV2[models.NodeV2]().InsertOne(node) + id, err := service.NewModelServiceV2[models2.NodeV2]().InsertOne(node) if err != nil { return err } @@ -170,7 +170,7 @@ func (svc *MasterServiceV2) Register() (err error) { node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } @@ -209,7 +209,7 @@ func (svc *MasterServiceV2) monitor() (err error) { wg := sync.WaitGroup{} wg.Add(len(workerNodes)) for _, n := range workerNodes { - go func(n *models.NodeV2) { + go func(n *models2.NodeV2) { // subscribe ok := svc.subscribeNode(n) if !ok { @@ -243,12 +243,12 @@ func (svc *MasterServiceV2) monitor() (err error) { return nil } -func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err error) { +func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models2.NodeV2, err error) { query := bson.M{ "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self "active": true, // active } - nodes, err = service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err = service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return nil, nil @@ -260,32 +260,32 @@ func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err erro func (svc *MasterServiceV2) updateMasterNodeStatus() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return err } node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } return nil } -func (svc *MasterServiceV2) setWorkerNodeOffline(node *models.NodeV2) { +func (svc *MasterServiceV2) setWorkerNodeOffline(node *models2.NodeV2) { node.Status = constants.NodeStatusOffline node.Active = false err := backoff.Retry(func() error { - return service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + return service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 3)) if err != nil { trace.PrintError(err) } } -func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { +func (svc *MasterServiceV2) subscribeNode(n *models2.NodeV2) (ok bool) { _, err := svc.server.GetSubscribe("node:" + n.Key) if err != nil { log.Errorf("cannot subscribe worker node[%s]: %v", n.Key, err) @@ -294,7 +294,7 @@ func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { return true } -func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { +func (svc *MasterServiceV2) pingNodeClient(n *models2.NodeV2) (ok bool) { if err := svc.server.SendStreamMessage("node:"+n.Key, grpc.StreamMessageCode_PING); err != nil { log.Errorf("cannot ping worker node client[%s]: %v", n.Key, err) return false @@ -302,17 +302,17 @@ func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { return true } -func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models.NodeV2) (err error) { +func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models2.NodeV2) (err error) { query := bson.M{ "node_id": node.Id, "status": constants.TaskStatusRunning, } - runningTasksCount, err := service.NewModelServiceV2[models.TaskV2]().Count(query) + runningTasksCount, err := service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { return trace.TraceError(err) } node.AvailableRunners = node.MaxRunners - runningTasksCount - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index da4047581..a041877c6 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -9,6 +9,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" client2 "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" @@ -31,7 +32,7 @@ type WorkerServiceV2 struct { heartbeatInterval time.Duration // internals - n *models.NodeV2 + n *models2.NodeV2 s grpc.NodeService_SubscribeClient } @@ -87,7 +88,7 @@ func (svc *WorkerServiceV2) Register() { if err != nil { panic(err) } - svc.n, err = client2.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) + svc.n, err = client2.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) if err != nil { panic(err) } diff --git a/core/notification/mail.go b/core/notification/mail.go index 905a5446a..c7f71a0ec 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -3,7 +3,7 @@ package notification import ( "errors" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/matcornic/hermes/v2" "gopkg.in/gomail.v2" "net/mail" diff --git a/core/notification/service.go b/core/notification/service.go index 37a7f93d0..0f3bf7bfc 100644 --- a/core/notification/service.go +++ b/core/notification/service.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 0d8a4c889..a29784315 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" mongo2 "github.com/crawlab-team/crawlab/db/mongo" parser "github.com/crawlab-team/crawlab/template-parser" diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go index 1d2d09da1..a1a244ac7 100644 --- a/core/schedule/service_v2.go +++ b/core/schedule/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" @@ -19,7 +19,7 @@ import ( type ServiceV2 struct { // dependencies interfaces.WithConfigPath - modelSvc *service.ModelServiceV2[models.ScheduleV2] + modelSvc *service.ModelServiceV2[models2.ScheduleV2] adminSvc *admin.ServiceV2 // settings variables @@ -31,7 +31,7 @@ type ServiceV2 struct { // internals cron *cron.Cron logger cron.Logger - schedules []models.ScheduleV2 + schedules []models2.ScheduleV2 stopped bool mu sync.Mutex } @@ -87,7 +87,7 @@ func (svc *ServiceV2) Stop() { svc.cron.Stop() } -func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) Enable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -101,7 +101,7 @@ func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err er return svc.modelSvc.ReplaceById(s.Id, s) } -func (svc *ServiceV2) Disable(s models.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) Disable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -191,7 +191,7 @@ func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { } // spider - spider, err := service.NewModelServiceV2[models.SpiderV2]().GetById(s.SpiderId) + spider, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(s.SpiderId) if err != nil { trace.PrintError(err) return @@ -250,7 +250,7 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { if err != nil { return nil, err } - svc.modelSvc = service.NewModelServiceV2[models.ScheduleV2]() + svc.modelSvc = service.NewModelServiceV2[models2.ScheduleV2]() // logger svc.logger = NewLogger() diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index 4ead1ec84..cf821201d 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -6,7 +6,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/scheduler" @@ -35,7 +35,7 @@ func (svc *ServiceV2) Start() (err error) { func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { return nil, err } @@ -44,9 +44,9 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun return svc.scheduleTasks(s, opts) } -func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { +func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task - mainTask := &models.TaskV2{ + mainTask := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, NodeIds: opts.NodeIds, @@ -83,7 +83,7 @@ func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderR return nil, err } for _, nodeId := range nodeIds { - t := &models.TaskV2{ + t := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, Cmd: opts.Cmd, @@ -127,7 +127,7 @@ func (svc *ServiceV2) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []p "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { return nil, err } @@ -147,7 +147,7 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { trace.PrintError(err) return false diff --git a/core/system/service_v2.go b/core/system/service_v2.go index a893c9d97..d82a6048c 100644 --- a/core/system/service_v2.go +++ b/core/system/service_v2.go @@ -1,7 +1,7 @@ package system import ( - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "go.mongodb.org/mongo-driver/bson" "sync" diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 885b1c018..7fec5e18c 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -15,6 +15,7 @@ import ( "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" service2 "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" @@ -47,8 +48,8 @@ type RunnerV2 struct { cmd *exec.Cmd // process command instance pid int // process id tid primitive.ObjectID // task id - t *models.TaskV2 // task model.Task - s *models.SpiderV2 // spider model.Spider + t *models2.TaskV2 // task model.Task + s *models2.SpiderV2 // spider model.Spider ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 err error // standard process error envs []models.Env // environment variables @@ -315,7 +316,7 @@ func (r *RunnerV2) configureEnv() { } // global environment variables - envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil) + envs, err := client.NewModelServiceV2[models2.EnvironmentV2]().GetMany(nil, nil) if err != nil { trace.PrintError(err) return @@ -510,12 +511,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { r.t.Error = e.Error() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = service2.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } } else { - err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } @@ -568,7 +569,7 @@ func (r *RunnerV2) writeLogLines(lines []string) { } func (r *RunnerV2) _updateTaskStat(status string) { - ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -589,13 +590,13 @@ func (r *RunnerV2) _updateTaskStat(status string) { ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = client.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return @@ -622,7 +623,7 @@ func (r *RunnerV2) sendNotification() { func (r *RunnerV2) _updateSpiderStat(status string) { // task stat - ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -660,13 +661,13 @@ func (r *RunnerV2) _updateSpiderStat(status string) { // perform update if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = service2.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = client.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index e6edb2b22..be7c2725c 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -10,7 +10,7 @@ import ( grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/trace" @@ -121,7 +121,7 @@ func (svc *ServiceV2) Fetch() { t.Error = err.Error() t.Status = constants.TaskStatusError t.SetUpdated(t.CreatedBy) - _ = client.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + _ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) continue } continue @@ -202,15 +202,15 @@ func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigServic return svc.cfgSvc } -func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { +func (svc *ServiceV2) GetCurrentNode() (n *models2.NodeV2, err error) { // node key nodeKey := svc.cfgSvc.GetNodeKey() // current node if svc.cfgSvc.IsMaster() { - n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) } else { - n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = client.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) } if err != nil { return nil, err @@ -219,11 +219,11 @@ func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { return n, nil } -func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) { +func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) { if svc.cfgSvc.IsMaster() { - t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id) } else { - t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err = client.NewModelServiceV2[models2.TaskV2]().GetById(id) } if err != nil { return nil, err @@ -232,11 +232,11 @@ func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err return t, nil } -func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) { +func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) { if svc.cfgSvc.IsMaster() { - s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err = service.NewModelServiceV2[models2.SpiderV2]().GetById(id) } else { - s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err = client.NewModelServiceV2[models2.SpiderV2]().GetById(id) } if err != nil { return nil, err @@ -267,13 +267,13 @@ func (svc *ServiceV2) getRunnerCount() (count int) { "status": constants.TaskStatusRunning, } if svc.cfgSvc.IsMaster() { - count, err = service.NewModelServiceV2[models.TaskV2]().Count(query) + count, err = service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { trace.PrintError(err) return } } else { - count, err = client.NewModelServiceV2[models.TaskV2]().Count(query) + count, err = client.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { trace.PrintError(err) return @@ -323,9 +323,9 @@ func (svc *ServiceV2) reportStatus() (err error) { // save node n.SetUpdated(n.CreatedBy) if svc.cfgSvc.IsMaster() { - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n) } else { - err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + err = client.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n) } if err != nil { return err diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index 740bf3e3c..81d5907e7 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -7,7 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" @@ -36,40 +36,40 @@ func (svc *ServiceV2) Start() { utils.DefaultWait() } -func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) { +func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) { // set task status t.Status = constants.TaskStatusPending t.SetCreatedBy(by) t.SetUpdated(by) // add task - taskModelSvc := service.NewModelServiceV2[models.TaskV2]() + taskModelSvc := service.NewModelServiceV2[models2.TaskV2]() _, err = taskModelSvc.InsertOne(*t) if err != nil { return nil, err } // task queue item - tq := models.TaskQueueItemV2{ + tq := models2.TaskQueueItemV2{ Priority: t.Priority, NodeId: t.NodeId, } tq.SetId(t.Id) // task stat - ts := models.TaskStatV2{ + ts := models2.TaskStatV2{ CreateTs: time.Now(), } ts.SetId(t.Id) // enqueue task - _, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq) + _, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq) if err != nil { return nil, trace.TraceError(err) } // add task stat - _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts) + _, err = service.NewModelServiceV2[models2.TaskStatV2]().InsertOne(ts) if err != nil { return nil, trace.TraceError(err) } @@ -80,7 +80,7 @@ func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *mode func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) { // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { return trace.TraceError(err) } @@ -95,7 +95,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err // set status of pending tasks as "cancelled" and remove from task item queue if initialStatus == constants.TaskStatusPending { // remove from task item queue - if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil { + if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(t.Id); err != nil { return trace.TraceError(err) } return nil @@ -110,7 +110,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err } // node - n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId) if err != nil { return trace.TraceError(err) } @@ -136,22 +136,22 @@ func (svc *ServiceV2) SetInterval(interval time.Duration) { svc.interval = interval } -func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err error) { if t.Id.IsZero() { t.SetCreated(by) t.SetUpdated(by) - _, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t) + _, err = service.NewModelServiceV2[models2.TaskV2]().InsertOne(*t) return err } else { t.SetUpdated(by) - return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + return service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) } } // initTaskStatus initialize task status of existing tasks func (svc *ServiceV2) initTaskStatus() { // set status of running tasks as TaskStatusAbnormal - runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{ + runningTasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{ "status": bson.M{ "$in": []string{ constants.TaskStatusPending, @@ -166,23 +166,23 @@ func (svc *ServiceV2) initTaskStatus() { trace.PrintError(err) } for _, t := range runningTasks { - go func(t *models.TaskV2) { + go func(t *models2.TaskV2) { t.Status = constants.TaskStatusAbnormal if err := svc.SaveTask(t, primitive.NilObjectID); err != nil { trace.PrintError(err) } }(&t) } - if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil { + if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteMany(nil); err != nil { return } } -func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { +func (svc *ServiceV2) isMasterNode(t *models2.TaskV2) (ok bool, err error) { if t.NodeId.IsZero() { return false, trace.TraceError(errors.ErrorTaskNoNodeId) } - n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId) if err != nil { if errors2.Is(err, mongo2.ErrNoDocuments) { return false, trace.TraceError(errors.ErrorTaskNodeNotFound) @@ -195,7 +195,7 @@ func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { func (svc *ServiceV2) cleanupTasks() { for { // task stats over 30 days ago - taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ + taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{ "create_ts": bson.M{ "$lt": time.Now().Add(-30 * 24 * time.Hour), }, @@ -213,14 +213,14 @@ func (svc *ServiceV2) cleanupTasks() { if len(ids) > 0 { // remove tasks - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) } // remove task stats - if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go index fc39cbd50..0637e1e16 100644 --- a/core/task/stats/service_v2.go +++ b/core/task/stats/service_v2.go @@ -2,7 +2,7 @@ package stats import ( "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/result" @@ -63,7 +63,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa } // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfa } func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) { - err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{ + err := service.NewModelServiceV2[models2.TaskStatV2]().UpdateById(id, bson.M{ "$inc": bson.M{ "result_count": resultCount, }, diff --git a/core/user/service_v2.go b/core/user/service_v2.go index 7784003c5..c0ab759c2 100644 --- a/core/user/service_v2.go +++ b/core/user/service_v2.go @@ -4,7 +4,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go index feccc5eca..3534a1de0 100644 --- a/core/utils/cockroachdb.go +++ b/core/utils/cockroachdb.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mssql" "time" @@ -59,13 +60,13 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses return s, err } -func GetCockroachdbSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetCockroachdbSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getCockroachdbSessionV2(ctx, ds) } -func getCockroachdbSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getCockroachdbSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/es.go b/core/utils/es.go index 7723205b0..d2db5e56c 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -8,6 +8,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/trace" "github.com/elastic/go-elasticsearch/v8" @@ -83,13 +84,13 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas return c, err } -func GetElasticsearchClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) { +func GetElasticsearchClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getElasticsearchClientV2(ctx, ds) } -func getElasticsearchClientV2(ctx context.Context, ds *models.DataSourceV2) (c *elasticsearch.Client, err error) { +func getElasticsearchClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *elasticsearch.Client, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/kafka.go b/core/utils/kafka.go index e33603922..e47d2107b 100644 --- a/core/utils/kafka.go +++ b/core/utils/kafka.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/segmentio/kafka-go" "time" ) @@ -40,13 +41,13 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co return kafka.DialLeader(ctx, network, address, topic, partition) } -func GetKafkaConnectionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) { +func GetKafkaConnectionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getKafkaConnectionV2(ctx, ds) } -func getKafkaConnectionV2(ctx context.Context, ds *models.DataSourceV2) (c *kafka.Conn, err error) { +func getKafkaConnectionV2(ctx context.Context, ds *models2.DataSourceV2) (c *kafka.Conn, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/mongo.go b/core/utils/mongo.go index 23e92541d..a2fd3b030 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -4,6 +4,7 @@ import ( "context" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" @@ -54,7 +55,7 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c return getMongoClient(ctx, ds) } -func GetMongoClientWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) { +func GetMongoClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMongoClientV2(ctx, ds) @@ -99,7 +100,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien return mongo.GetMongoClient(opts...) } -func getMongoClientV2(ctx context.Context, ds *models.DataSourceV2) (c *mongo2.Client, err error) { +func getMongoClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *mongo2.Client, err error) { // normalize settings if ds.Host == "" { ds.Host = constants.DefaultHost diff --git a/core/utils/mssql.go b/core/utils/mssql.go index 03329eae5..7333706ba 100644 --- a/core/utils/mssql.go +++ b/core/utils/mssql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mssql" "time" @@ -59,13 +60,13 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetMssqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetMssqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMssqlSessionV2(ctx, ds) } -func getMssqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getMssqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/mysql.go b/core/utils/mysql.go index c1e78a20c..4ea293cff 100644 --- a/core/utils/mysql.go +++ b/core/utils/mysql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mysql" "time" @@ -59,13 +60,13 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetMysqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetMysqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMysqlSessionV2(ctx, ds) } -func getMysqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getMysqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go index cf3ba7dc5..a3086762e 100644 --- a/core/utils/postgresql.go +++ b/core/utils/postgresql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/postgresql" "time" @@ -59,13 +60,13 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess return s, err } -func GetPostgresqlSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetPostgresqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getPostgresqlSessionV2(ctx, ds) } -func getPostgresqlSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getPostgresqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go index 83a06cbcf..b8e877867 100644 --- a/core/utils/sqlite.go +++ b/core/utils/sqlite.go @@ -3,6 +3,7 @@ package utils import ( "context" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/sqlite" "time" @@ -44,13 +45,13 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetSqliteSessionWithTimeoutV2(ds *models.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetSqliteSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getSqliteSessionV2(ctx, ds) } -func getSqliteSessionV2(ctx context.Context, ds *models.DataSourceV2) (s db.Session, err error) { +func getSqliteSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { // connect settings settings := sqlite.ConnectionURL{ Database: ds.Database, From 7b0cd9da9f86c99d013a5080c2b60ca57c58147d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 11 Jul 2024 14:42:57 +0800 Subject: [PATCH 049/106] feat: added models --- core/models/models/v2/notification_channel_v2.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index cd3172219..250d6e6cc 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -1,4 +1,8 @@ package models -type NotificationChannel struct { +type NotificationChannelV2 struct { + any `collection:"notification_channels"` + BaseModelV2[NotificationChannelV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` } From e66e4d67412b42b09f8af926b212e1efd7107fc8 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 11 Jul 2024 18:08:40 +0800 Subject: [PATCH 050/106] feat: updated models for notification --- core/constants/notification.go | 5 +++++ core/models/models/v2/notification_setting_v2.go | 4 +++- core/notification/service_v2.go | 12 ++++++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/constants/notification.go b/core/constants/notification.go index 2ce094b82..46f289bb7 100644 --- a/core/constants/notification.go +++ b/core/constants/notification.go @@ -6,3 +6,8 @@ const ( NotificationTriggerTaskEmptyResults = "task_empty_results" NotificationTriggerTaskNever = "task_never" ) + +const ( + NotificationTemplateModeRichText = "rich-text" + NotificationTemplateModeMarkdown = "markdown" +) diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index b07a01970..10a6c9c4d 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -9,7 +9,9 @@ type NotificationSettingV2 struct { Enabled bool `json:"enabled" bson:"enabled"` Global bool `json:"global" bson:"global"` Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` + TemplateMode string `json:"template_mode" bson:"template_mode"` + TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` + TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` TaskTrigger string `json:"task_trigger" bson:"task_trigger"` Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index a29784315..5a18e3666 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -48,7 +48,7 @@ func (svc *ServiceV2) initData() (err error) { Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] 爬虫任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, + TemplateMarkdown: `尊敬的 {{$.user.username}}, 请查看下面的任务数据。 @@ -83,7 +83,7 @@ func (svc *ServiceV2) initData() (err error) { Description: "This is the default mail notification. You can edit it with your own settings", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, + TemplateMarkdown: `Dear {{$.user.username}}, Please find the task data as below. @@ -118,7 +118,7 @@ Please find the task data as below. Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] 任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, + TemplateMarkdown: `尊敬的 {{$.user.username}}, 请查看下面的任务数据。 @@ -146,7 +146,7 @@ Please find the task data as below. Description: "This is the default mobile notification. You can edit it with your own settings", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, + TemplateMarkdown: `Dear {{$.user.username}}, Please find the task data as below. @@ -208,7 +208,7 @@ func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, entity bson.M) ( } // content - content, err := parser.Parse(s.Template, entity) + content, err := parser.Parse(s.TemplateMarkdown, entity) if err != nil { log.Warnf("parsing 'content' error: %v", err) } @@ -238,7 +238,7 @@ func (svc *ServiceV2) SendMobile(s *models.NotificationSettingV2, entity bson.M) } // content - content, err := parser.Parse(s.Template, entity) + content, err := parser.Parse(s.TemplateMarkdown, entity) if err != nil { log.Warnf("parsing 'content' error: %v", err) } From 7649885b2489aa3792e3f38e71c2e62ba3f928d0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 11 Jul 2024 23:10:15 +0800 Subject: [PATCH 051/106] feat: updated model for notification settings --- core/models/models/v2/notification_setting_v2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index 10a6c9c4d..832117fbe 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -12,6 +12,7 @@ type NotificationSettingV2 struct { TemplateMode string `json:"template_mode" bson:"template_mode"` TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` + TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` TaskTrigger string `json:"task_trigger" bson:"task_trigger"` Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` From d0611b4567348f819cc93bcd3c73fb21fa9669b3 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 12 Jul 2024 18:00:19 +0800 Subject: [PATCH 052/106] refactor: removed unnecessary code --- core/apps/utils.go | 91 ------- core/grpc/server/task_server.go | 238 ----------------- core/node/service/master_service.go | 368 -------------------------- core/notification/models.go | 32 --- core/notification/service.go | 395 ---------------------------- core/notification/service_test.go | 19 -- 6 files changed, 1143 deletions(-) delete mode 100644 core/apps/utils.go delete mode 100644 core/grpc/server/task_server.go delete mode 100644 core/node/service/master_service.go delete mode 100644 core/notification/models.go delete mode 100644 core/notification/service.go delete mode 100644 core/notification/service_test.go diff --git a/core/apps/utils.go b/core/apps/utils.go deleted file mode 100644 index f0d279e3d..000000000 --- a/core/apps/utils.go +++ /dev/null @@ -1,91 +0,0 @@ -package apps - -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/color" - "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" - grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" - grpcserver "github.com/crawlab-team/crawlab/core/grpc/server" - modelsclient "github.com/crawlab-team/crawlab/core/models/client" - modelsservice "github.com/crawlab-team/crawlab/core/models/service" - nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/schedule" - "github.com/crawlab-team/crawlab/core/spider/admin" - "github.com/crawlab-team/crawlab/core/stats" - "github.com/crawlab-team/crawlab/core/task/handler" - "github.com/crawlab-team/crawlab/core/task/scheduler" - taskstats "github.com/crawlab-team/crawlab/core/task/stats" - "github.com/crawlab-team/crawlab/core/user" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" -) - -func Start(app App) { - start(app) -} - -func start(app App) { - app.Init() - go app.Start() - app.Wait() - app.Stop() -} - -func DefaultWait() { - utils.DefaultWait() -} - -func initModule(name string, fn func() error) (err error) { - if err := fn(); err != nil { - log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) - _ = trace.TraceError(err) - panic(err) - } - log.Info(fmt.Sprintf("initialized %s successfully", name)) - return nil -} - -func initApp(name string, app App) { - _ = initModule(name, func() error { - app.Init() - return nil - }) -} - -var injectors = []interface{}{ - modelsservice.GetService, - modelsclient.NewServiceDelegate, - modelsclient.NewNodeServiceDelegate, - modelsclient.NewSpiderServiceDelegate, - modelsclient.NewTaskServiceDelegate, - modelsclient.NewTaskStatServiceDelegate, - modelsclient.NewEnvironmentServiceDelegate, - grpcclient.NewClient, - grpcclient.NewPool, - grpcserver.NewModelDelegateServer, - grpcserver.NewModelBaseServiceServer, - grpcserver.NewNodeServer, - grpcserver.NewTaskServer, - grpcserver.NewMessageServer, - config.NewConfigPathService, - user.GetUserService, - schedule.GetScheduleService, - admin.GetSpiderAdminService, - stats.GetStatsService, - nodeconfig.GetNodeConfigService, - taskstats.GetTaskStatsService, - color.NewService, - scheduler.GetTaskSchedulerService, - handler.GetTaskHandlerService, -} - -func injectModules() { - c := container.GetContainer() - for _, injector := range injectors { - if err := c.Provide(injector); err != nil { - panic(err) - } - } -} diff --git a/core/grpc/server/task_server.go b/core/grpc/server/task_server.go deleted file mode 100644 index b6d085923..000000000 --- a/core/grpc/server/task_server.go +++ /dev/null @@ -1,238 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/notification" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" - "strings" -) - -type TaskServer struct { - grpc.UnimplementedTaskServiceServer - - // dependencies - modelSvc service.ModelService - cfgSvc interfaces.NodeConfigService - statsSvc interfaces.TaskStatsService - - // internals - server interfaces.GrpcServer -} - -// Subscribe to task stream when a task runner in a node starts -func (svr TaskServer) Subscribe(stream grpc.TaskService_SubscribeServer) (err error) { - for { - msg, err := stream.Recv() - utils.LogDebug(msg.String()) - if err == io.EOF { - return nil - } - if err != nil { - if strings.HasSuffix(err.Error(), "context canceled") { - return nil - } - trace.PrintError(err) - continue - } - switch msg.Code { - case grpc.StreamMessageCode_INSERT_DATA: - err = svr.handleInsertData(msg) - case grpc.StreamMessageCode_INSERT_LOGS: - err = svr.handleInsertLogs(msg) - default: - err = errors.ErrorGrpcInvalidCode - log.Errorf("invalid stream message code: %d", msg.Code) - continue - } - if err != nil { - log.Errorf("grpc error[%d]: %v", msg.Code, err) - } - } -} - -// Fetch tasks to be executed by a task handler -func (svr TaskServer) Fetch(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { - nodeKey := request.GetNodeKey() - if nodeKey == "" { - return nil, trace.TraceError(errors.ErrorGrpcInvalidNodeKey) - } - n, err := svr.modelSvc.GetNodeByKey(nodeKey, nil) - if err != nil { - return nil, trace.TraceError(err) - } - var tid primitive.ObjectID - opts := &mongo.FindOptions{ - Sort: bson.D{ - {"p", 1}, - {"_id", 1}, - }, - Limit: 1, - } - if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { - // get task queue item assigned to this node - tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": n.Id}, opts, n.Id) - if err != nil { - return err - } - if !tid.IsZero() { - return nil - } - - // get task queue item assigned to any node (random mode) - tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": nil}, opts, n.Id) - if !tid.IsZero() { - return nil - } - if err != nil { - return err - } - return nil - }); err != nil { - return nil, err - } - return HandleSuccessWithData(tid) -} - -func (svr TaskServer) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { - svc := notification.GetService() - var t = new(models.Task) - if err := json.Unmarshal(request.Data, t); err != nil { - return nil, trace.TraceError(err) - } - t, err = svr.modelSvc.GetTaskById(t.Id) - if err != nil { - return nil, trace.TraceError(err) - } - td, err := json.Marshal(t) - if err != nil { - return nil, trace.TraceError(err) - } - var e bson.M - if err := json.Unmarshal(td, &e); err != nil { - return nil, trace.TraceError(err) - } - ts, err := svr.modelSvc.GetTaskStatById(t.Id) - if err != nil { - return nil, trace.TraceError(err) - } - settings, _, err := svc.GetSettingList(bson.M{ - "enabled": true, - }, nil, nil) - if err != nil { - return nil, trace.TraceError(err) - } - for _, s := range settings { - switch s.TaskTrigger { - case constants.NotificationTriggerTaskFinish: - if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { - _ = svc.Send(s, e) - } - case constants.NotificationTriggerTaskError: - if t.Status == constants.TaskStatusError || t.Status == constants.TaskStatusAbnormal { - _ = svc.Send(s, e) - } - case constants.NotificationTriggerTaskEmptyResults: - if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { - if ts.ResultCount == 0 { - _ = svc.Send(s, e) - } - } - case constants.NotificationTriggerTaskNever: - } - } - return nil, nil -} - -func (svr TaskServer) handleInsertData(msg *grpc.StreamMessage) (err error) { - data, err := svr.deserialize(msg) - if err != nil { - return err - } - var records []interface{} - for _, d := range data.Records { - res, ok := d[constants.TaskKey] - if ok { - switch res.(type) { - case string: - id, err := primitive.ObjectIDFromHex(res.(string)) - if err == nil { - d[constants.TaskKey] = id - } - } - } - records = append(records, d) - } - return svr.statsSvc.InsertData(data.TaskId, records...) -} - -func (svr TaskServer) handleInsertLogs(msg *grpc.StreamMessage) (err error) { - data, err := svr.deserialize(msg) - if err != nil { - return err - } - return svr.statsSvc.InsertLogs(data.TaskId, data.Logs...) -} - -func (svr TaskServer) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { - var tq models.TaskQueueItem - if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Find(query, opts).One(&tq); err != nil { - if err == mongo2.ErrNoDocuments { - return tid, nil - } - return tid, trace.TraceError(err) - } - t, err := svr.modelSvc.GetTaskById(tq.Id) - if err == nil { - t.NodeId = nid - _ = delegate.NewModelDelegate(t).Save() - } - _ = delegate.NewModelDelegate(&tq).Delete() - return tq.Id, nil -} - -func (svr TaskServer) deserialize(msg *grpc.StreamMessage) (data entity.StreamMessageTaskData, err error) { - if err := json.Unmarshal(msg.Data, &data); err != nil { - return data, trace.TraceError(err) - } - if data.TaskId.IsZero() { - return data, trace.TraceError(errors.ErrorGrpcInvalidType) - } - return data, nil -} - -func NewTaskServer() (res *TaskServer, err error) { - // task server - svr := &TaskServer{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - statsSvc interfaces.TaskStatsService, - cfgSvc interfaces.NodeConfigService, - ) { - svr.modelSvc = modelSvc - svr.statsSvc = statsSvc - svr.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } - - return svr, nil -} diff --git a/core/node/service/master_service.go b/core/node/service/master_service.go deleted file mode 100644 index 3ade5da19..000000000 --- a/core/node/service/master_service.go +++ /dev/null @@ -1,368 +0,0 @@ -package service - -import ( - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/notification" - "github.com/crawlab-team/crawlab/core/system" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" -) - -type MasterService struct { - // dependencies - modelSvc service.ModelService - cfgSvc interfaces.NodeConfigService - server interfaces.GrpcServer - schedulerSvc interfaces.TaskSchedulerService - handlerSvc interfaces.TaskHandlerService - scheduleSvc interfaces.ScheduleService - notificationSvc *notification.Service - spiderAdminSvc interfaces.SpiderAdminService - systemSvc *system.Service - - // settings - cfgPath string - address interfaces.Address - monitorInterval time.Duration - stopOnError bool -} - -func (svc *MasterService) Init() (err error) { - // do nothing - return nil -} - -func (svc *MasterService) Start() { - // create indexes - common.CreateIndexes() - - // start grpc server - if err := svc.server.Start(); err != nil { - panic(err) - } - - // register to db - if err := svc.Register(); err != nil { - panic(err) - } - - // start monitoring worker nodes - go svc.Monitor() - - // start task handler - go svc.handlerSvc.Start() - - // start task scheduler - go svc.schedulerSvc.Start() - - // start schedule service - go svc.scheduleSvc.Start() - - // start notification service - go svc.notificationSvc.Start() - - // start spider admin service - go svc.spiderAdminSvc.Start() - - // wait for quit signal - svc.Wait() - - // stop - svc.Stop() -} - -func (svc *MasterService) Wait() { - utils.DefaultWait() -} - -func (svc *MasterService) Stop() { - _ = svc.server.Stop() - log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey()) -} - -func (svc *MasterService) Monitor() { - log.Infof("master[%s] monitoring started", svc.GetConfigService().GetNodeKey()) - for { - if err := svc.monitor(); err != nil { - trace.PrintError(err) - if svc.stopOnError { - log.Errorf("master[%s] monitor error, now stopping...", svc.GetConfigService().GetNodeKey()) - svc.Stop() - return - } - } - - time.Sleep(svc.monitorInterval) - } -} - -func (svc *MasterService) GetConfigService() (cfgSvc interfaces.NodeConfigService) { - return svc.cfgSvc -} - -func (svc *MasterService) GetConfigPath() (path string) { - return svc.cfgPath -} - -func (svc *MasterService) SetConfigPath(path string) { - svc.cfgPath = path -} - -func (svc *MasterService) GetAddress() (address interfaces.Address) { - return svc.address -} - -func (svc *MasterService) SetAddress(address interfaces.Address) { - svc.address = address -} - -func (svc *MasterService) SetMonitorInterval(duration time.Duration) { - svc.monitorInterval = duration -} - -func (svc *MasterService) Register() (err error) { - nodeKey := svc.GetConfigService().GetNodeKey() - nodeName := svc.GetConfigService().GetNodeName() - node, err := svc.modelSvc.GetNodeByKey(nodeKey, nil) - if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { - // not exists - log.Infof("master[%s] does not exist in db", nodeKey) - node := &models.Node{ - Key: nodeKey, - Name: nodeName, - MaxRunners: config.DefaultConfigOptions.MaxRunners, - IsMaster: true, - Status: constants.NodeStatusOnline, - Enabled: true, - Active: true, - ActiveTs: time.Now(), - } - if viper.GetInt("task.handler.maxRunners") > 0 { - node.MaxRunners = viper.GetInt("task.handler.maxRunners") - } - nodeD := delegate.NewModelNodeDelegate(node) - if err := nodeD.Add(); err != nil { - return err - } - log.Infof("added master[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) - return nil - } else if err == nil { - // exists - log.Infof("master[%s] exists in db", nodeKey) - nodeD := delegate.NewModelNodeDelegate(node) - if err := nodeD.UpdateStatusOnline(); err != nil { - return err - } - log.Infof("updated master[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) - return nil - } else { - // error - return err - } -} - -func (svc *MasterService) StopOnError() { - svc.stopOnError = true -} - -func (svc *MasterService) GetServer() (svr interfaces.GrpcServer) { - return svc.server -} - -func (svc *MasterService) monitor() (err error) { - // update master node status in db - if err := svc.updateMasterNodeStatus(); err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - return nil - } - return err - } - - // all worker nodes - nodes, err := svc.getAllWorkerNodes() - if err != nil { - return err - } - - // error flag - isErr := false - - // iterate all nodes - for _, n := range nodes { - // subscribe - if err := svc.subscribeNode(&n); err != nil { - isErr = true - continue - } - - // ping client - if err := svc.pingNodeClient(&n); err != nil { - isErr = true - continue - } - - // update node available runners - if err := svc.updateNodeAvailableRunners(&n); err != nil { - isErr = true - continue - } - } - - if isErr { - return trace.TraceError(errors.ErrorNodeMonitorError) - } - - return nil -} - -func (svc *MasterService) getAllWorkerNodes() (nodes []models.Node, err error) { - query := bson.M{ - "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self - "active": true, // active - } - nodes, err = svc.modelSvc.GetNodeList(query, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - return nil, nil - } - return nil, trace.TraceError(err) - } - return nodes, nil -} - -func (svc *MasterService) updateMasterNodeStatus() (err error) { - nodeKey := svc.GetConfigService().GetNodeKey() - node, err := svc.modelSvc.GetNodeByKey(nodeKey, nil) - if err != nil { - return err - } - nodeD := delegate.NewModelNodeDelegate(node) - return nodeD.UpdateStatusOnline() -} - -func (svc *MasterService) setWorkerNodeOffline(n interfaces.Node) (err error) { - return delegate.NewModelNodeDelegate(n).UpdateStatusOffline() -} - -func (svc *MasterService) subscribeNode(n interfaces.Node) (err error) { - _, err = svc.server.GetSubscribe("node:" + n.GetKey()) - if err != nil { - log.Errorf("cannot subscribe worker node[%s]: %v", n.GetKey(), err) - if err := svc.setWorkerNodeOffline(n); err != nil { - return trace.TraceError(err) - } - return trace.TraceError(err) - } - return nil -} - -func (svc *MasterService) pingNodeClient(n interfaces.Node) (err error) { - if err := svc.server.SendStreamMessage("node:"+n.GetKey(), grpc.StreamMessageCode_PING); err != nil { - log.Errorf("cannot ping worker node client[%s]: %v", n.GetKey(), err) - if err := svc.setWorkerNodeOffline(n); err != nil { - return trace.TraceError(err) - } - return trace.TraceError(err) - } - return nil -} - -func (svc *MasterService) updateNodeAvailableRunners(n interfaces.Node) (err error) { - query := bson.M{ - "node_id": n.GetId(), - "status": constants.TaskStatusRunning, - } - runningTasksCount, err := mongo.GetMongoCol(interfaces.ModelColNameTask).Count(query) - if err != nil { - return trace.TraceError(err) - } - n.SetAvailableRunners(n.GetMaxRunners() - runningTasksCount) - return delegate.NewModelDelegate(n).Save() -} - -func NewMasterService(opts ...Option) (res interfaces.NodeMasterService, err error) { - // master service - svc := &MasterService{ - cfgPath: config2.GetConfigPath(), - monitorInterval: 15 * time.Second, - stopOnError: false, - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // server options - var serverOpts []server.Option - if svc.address != nil { - serverOpts = append(serverOpts, server.WithAddress(svc.address)) - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - modelSvc service.ModelService, - server interfaces.GrpcServer, - schedulerSvc interfaces.TaskSchedulerService, - handlerSvc interfaces.TaskHandlerService, - scheduleSvc interfaces.ScheduleService, - spiderAdminSvc interfaces.SpiderAdminService, - ) { - svc.cfgSvc = cfgSvc - svc.modelSvc = modelSvc - svc.server = server - svc.schedulerSvc = schedulerSvc - svc.handlerSvc = handlerSvc - svc.scheduleSvc = scheduleSvc - svc.spiderAdminSvc = spiderAdminSvc - }); err != nil { - return nil, err - } - - // notification service - svc.notificationSvc = notification.GetService() - - // system service - svc.systemSvc = system.GetService() - - // init - if err := svc.Init(); err != nil { - return nil, err - } - - return svc, nil -} - -func ProvideMasterService(path string, opts ...Option) func() (interfaces.NodeMasterService, error) { - // path - if path == "" || path == config2.GetConfigPath() { - if viper.GetString("config.path") != "" { - path = viper.GetString("config.path") - } else { - path = config2.GetConfigPath() - } - } - opts = append(opts, WithConfigPath(path)) - - return func() (interfaces.NodeMasterService, error) { - return NewMasterService(opts...) - } -} diff --git a/core/notification/models.go b/core/notification/models.go deleted file mode 100644 index efb554486..000000000 --- a/core/notification/models.go +++ /dev/null @@ -1,32 +0,0 @@ -package notification - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type NotificationSetting struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` -} - -type NotificationSettingMail struct { - Server string `json:"server" bson:"server"` - Port string `json:"port,omitempty" bson:"port,omitempty"` - User string `json:"user,omitempty" bson:"user,omitempty"` - Password string `json:"password,omitempty" bson:"password,omitempty"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` - To string `json:"to,omitempty" bson:"to,omitempty"` - Cc string `json:"cc,omitempty" bson:"cc,omitempty"` -} - -type NotificationSettingMobile struct { - Webhook string `json:"webhook" bson:"webhook"` -} diff --git a/core/notification/service.go b/core/notification/service.go deleted file mode 100644 index 0f3bf7bfc..000000000 --- a/core/notification/service.go +++ /dev/null @@ -1,395 +0,0 @@ -package notification - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - mongo2 "github.com/crawlab-team/crawlab/db/mongo" - parser "github.com/crawlab-team/crawlab/template-parser" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" -) - -type Service struct { - col *mongo2.Col // notification settings - modelSvc service.ModelService -} - -func (svc *Service) Init() (err error) { - if !utils.IsPro() { - return nil - } - - return nil -} - -func (svc *Service) Start() (err error) { - // initialize data - if err := svc.initData(); err != nil { - return err - } - - return nil -} - -func (svc *Service) Stop() (err error) { - return nil -} - -func (svc *Service) initData() (err error) { - total, err := svc.col.Count(nil) - if err != nil { - return err - } - if total > 0 { - return nil - } - - // data to initialize - settings := []NotificationSetting{ - { - Id: primitive.NewObjectID(), - Type: TypeMail, - Enabled: true, - Name: "任务通知（邮件）", - Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 爬虫任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -|键|值| -|:-:|:--| -|任务状态|{{$.status}}| -|任务优先级|{{$.priority}}| -|任务模式|{{$.mode}}| -|执行命令|{{$.cmd}}| -|执行参数|{{$.param}}| -|错误信息|{{$.error}}| -|节点|{{$.node.name}}| -|爬虫|{{$.spider.name}}| -|项目|{{$.spider.project.name}}| -|定时任务|{{$.schedule.name}}| -|结果数|{{$.:task_stat.result_count}}| -|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| -|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| -|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| -|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMail, - Enabled: true, - Name: "Task Change (Mail)", - Description: "This is the default mail notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -|Key|Value| -|:-:|:--| -|Task Status|{{$.status}}| -|Task Priority|{{$.priority}}| -|Task Mode|{{$.mode}}| -|Task Command|{{$.cmd}}| -|Task Params|{{$.param}}| -|Error Message|{{$.error}}| -|Node|{{$.node.name}}| -|Spider|{{$.spider.name}}| -|Project|{{$.spider.project.name}}| -|Schedule|{{$.schedule.name}}| -|Result Count|{{$.:task_stat.result_count}}| -|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| -|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| -|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| -|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMobile, - Enabled: true, - Name: "任务通知（移动端）", - Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -- **任务状态**: {{$.status}} -- **任务优先级**: {{$.priority}} -- **任务模式**: {{$.mode}} -- **执行命令**: {{$.cmd}} -- **执行参数**: {{$.param}} -- **错误信息**: {{$.error}} -- **节点**: {{$.node.name}} -- **爬虫**: {{$.spider.name}} -- **项目**: {{$.spider.project.name}} -- **定时任务**: {{$.schedule.name}} -- **结果数**: {{$.:task_stat.result_count}} -- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} -- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} -- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: NotificationSettingMobile{}, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMobile, - Enabled: true, - Name: "Task Change (Mobile)", - Description: "This is the default mobile notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskError, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -- **Task Status**: {{$.status}} -- **Task Priority**: {{$.priority}} -- **Task Mode**: {{$.mode}} -- **Task Command**: {{$.cmd}} -- **Task Params**: {{$.param}} -- **Error Message**: {{$.error}} -- **Node**: {{$.node.name}} -- **Spider**: {{$.spider.name}} -- **Project**: {{$.spider.project.name}} -- **Schedule**: {{$.schedule.name}} -- **Result Count**: {{$.:task_stat.result_count}} -- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} -- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} -- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: NotificationSettingMobile{}, - }, - } - var data []interface{} - for _, s := range settings { - data = append(data, s) - } - _, err = svc.col.InsertMany(data) - if err != nil { - return err - } - return nil -} - -func (svc *Service) Send(s NotificationSetting, entity bson.M) (err error) { - switch s.Type { - case TypeMail: - return svc.SendMail(s, entity) - case TypeMobile: - return svc.SendMobile(s, entity) - } - return nil -} - -func (svc *Service) SendMail(s NotificationSetting, entity bson.M) (err error) { - // to - to, err := parser.Parse(s.Mail.To, entity) - if err != nil { - log.Warnf("parsing 'to' error: %v", err) - } - if to == "" { - return nil - } - - // cc - cc, err := parser.Parse(s.Mail.Cc, entity) - if err != nil { - log.Warnf("parsing 'cc' error: %v", err) - } - - // title - title, err := parser.Parse(s.Title, entity) - if err != nil { - log.Warnf("parsing 'title' error: %v", err) - } - - // content - content, err := parser.Parse(s.Template, entity) - if err != nil { - log.Warnf("parsing 'content' error: %v", err) - } - - // send mail - if err := SendMail(&models.NotificationSettingMail{ - Server: s.Mail.Server, - Port: s.Mail.Port, - User: s.Mail.User, - Password: s.Mail.Password, - SenderEmail: s.Mail.SenderEmail, - SenderIdentity: s.Mail.SenderIdentity, - To: s.Mail.To, - Cc: s.Mail.Cc, - }, to, cc, title, content); err != nil { - return err - } - - return nil -} - -func (svc *Service) SendMobile(s NotificationSetting, entity bson.M) (err error) { - // webhook - webhook, err := parser.Parse(s.Mobile.Webhook, entity) - if err != nil { - log.Warnf("parsing 'webhook' error: %v", err) - } - if webhook == "" { - return nil - } - - // title - title, err := parser.Parse(s.Title, entity) - if err != nil { - log.Warnf("parsing 'title' error: %v", err) - } - - // content - content, err := parser.Parse(s.Template, entity) - if err != nil { - log.Warnf("parsing 'content' error: %v", err) - } - - // send - if err := SendMobileNotification(webhook, title, content); err != nil { - return err - } - - return nil -} - -func (svc *Service) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []NotificationSetting, total int, err error) { - // options - var options *mongo2.FindOptions - if pagination != nil || sort != nil { - options = new(mongo2.FindOptions) - if pagination != nil { - options.Skip = pagination.Size * (pagination.Page - 1) - options.Limit = pagination.Size - } - if sort != nil { - options.Sort = sort - } - } - - // get list - var list []NotificationSetting - if err := svc.col.Find(query, options).All(&list); err != nil { - if err.Error() == mongo.ErrNoDocuments.Error() { - return nil, 0, nil - } else { - return nil, 0, err - } - } - - // total count - total, err = svc.col.Count(query) - if err != nil { - return nil, 0, err - } - - return list, total, nil -} - -func (svc *Service) GetSetting(id primitive.ObjectID) (res *NotificationSetting, err error) { - var s NotificationSetting - if err := svc.col.FindId(id).One(&s); err != nil { - return nil, err - } - return &s, nil -} - -func (svc *Service) PosSetting(s *NotificationSetting) (err error) { - s.Id = primitive.NewObjectID() - if _, err := svc.col.Insert(s); err != nil { - return err - } - return nil -} - -func (svc *Service) PutSetting(id primitive.ObjectID, s NotificationSetting) (err error) { - if err := svc.col.ReplaceId(id, s); err != nil { - return err - } - - return nil -} - -func (svc *Service) DeleteSetting(id primitive.ObjectID) (err error) { - if err := svc.col.DeleteId(id); err != nil { - return err - } - - return nil -} - -func (svc *Service) EnableSetting(id primitive.ObjectID) (err error) { - return svc._toggleSettingFunc(true)(id) -} - -func (svc *Service) DisableSetting(id primitive.ObjectID) (err error) { - return svc._toggleSettingFunc(false)(id) -} - -func (svc *Service) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { - return func(id primitive.ObjectID) (err error) { - var s NotificationSetting - if err := svc.col.FindId(id).One(&s); err != nil { - return err - } - s.Enabled = value - if err := svc.col.ReplaceId(id, s); err != nil { - return err - } - return nil - } -} - -func NewService() *Service { - // service - svc := &Service{ - col: mongo2.GetMongoCol(SettingsColName), - } - - // model service - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - svc.modelSvc = modelSvc - - if err := svc.Init(); err != nil { - panic(err) - } - - return svc -} - -var _service *Service - -func GetService() *Service { - if _service == nil { - _service = NewService() - } - return _service -} diff --git a/core/notification/service_test.go b/core/notification/service_test.go deleted file mode 100644 index 20e094784..000000000 --- a/core/notification/service_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package notification - -import ( - "net/http" - "testing" - "time" -) - -func TestService_sendMobile(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - time.Sleep(1 * time.Second) - - data := map[string]interface{}{ - "task_id": T.TestTask.GetId().Hex(), - } - e.POST("/send/mobile").WithJSON(data). - Expect().Status(http.StatusOK) -} From 3a03ac63dc642b0d09f8738d5b45c9526d3e2b23 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 12 Jul 2024 20:05:14 +0800 Subject: [PATCH 053/106] fix: compiling issue --- core/apps/api_v2.go | 3 +- core/apps/utils.go | 40 ++++++++++++++++++++++++ core/grpc/server/options.go | 43 -------------------------- core/node/service/master_service_v2.go | 6 ---- 4 files changed, 42 insertions(+), 50 deletions(-) create mode 100644 core/apps/utils.go delete mode 100644 core/grpc/server/options.go diff --git a/core/apps/api_v2.go b/core/apps/api_v2.go index 966b1a087..16fe102c6 100644 --- a/core/apps/api_v2.go +++ b/core/apps/api_v2.go @@ -7,6 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" "github.com/spf13/viper" "net" @@ -73,7 +74,7 @@ func (app *ApiV2) Start() { } func (app *ApiV2) Wait() { - DefaultWait() + utils.DefaultWait() } func (app *ApiV2) Stop() { diff --git a/core/apps/utils.go b/core/apps/utils.go new file mode 100644 index 000000000..25e48c5e4 --- /dev/null +++ b/core/apps/utils.go @@ -0,0 +1,40 @@ +package apps + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/trace" +) + +func Start(app App) { + start(app) +} + +func start(app App) { + app.Init() + go app.Start() + app.Wait() + app.Stop() +} + +func DefaultWait() { + utils.DefaultWait() +} + +func initModule(name string, fn func() error) (err error) { + if err := fn(); err != nil { + log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) + _ = trace.TraceError(err) + panic(err) + } + log.Info(fmt.Sprintf("initialized %s successfully", name)) + return nil +} + +func initApp(name string, app App) { + _ = initModule(name, func() error { + app.Init() + return nil + }) +} diff --git a/core/grpc/server/options.go b/core/grpc/server/options.go deleted file mode 100644 index afae5dbae..000000000 --- a/core/grpc/server/options.go +++ /dev/null @@ -1,43 +0,0 @@ -package server - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" -) - -type Option func(svr interfaces.GrpcServer) - -func WithConfigPath(path string) Option { - return func(svr interfaces.GrpcServer) { - svr.SetConfigPath(path) - } -} - -func WithAddress(address interfaces.Address) Option { - return func(svr interfaces.GrpcServer) { - svr.SetAddress(address) - } -} - -type NodeServerOption func(svr *NodeServer) - -func WithServerNodeServerService(server interfaces.GrpcServer) NodeServerOption { - return func(svr *NodeServer) { - svr.server = server - } -} - -type TaskServerOption func(svr *TaskServer) - -func WithServerTaskServerService(server interfaces.GrpcServer) TaskServerOption { - return func(svr *TaskServer) { - svr.server = server - } -} - -type MessageServerOption func(svr *MessageServer) - -func WithServerMessageServerService(server interfaces.GrpcServer) MessageServerOption { - return func(svr *MessageServer) { - svr.server = server - } -} diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 73f9cf344..196a74489 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -327,12 +327,6 @@ func newMasterServiceV2() (res *MasterServiceV2, err error) { stopOnError: false, } - // server options - var serverOpts []server.Option - if svc.address != nil { - serverOpts = append(serverOpts, server.WithAddress(svc.address)) - } - // node config service svc.cfgSvc = config.GetNodeConfigService() From 16ac26ef0cdc9647516452439889364a6ed3c53a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 13 Jul 2024 10:03:24 +0800 Subject: [PATCH 054/106] feat: Add template field to NotificationSettingV2 model --- core/models/models/v2/notification_setting_v2.go | 1 + core/notification/service_v2.go | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index 832117fbe..b19adfa9a 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -9,6 +9,7 @@ type NotificationSettingV2 struct { Enabled bool `json:"enabled" bson:"enabled"` Global bool `json:"global" bson:"global"` Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template" bson:"template"` TemplateMode string `json:"template_mode" bson:"template_mode"` TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 5a18e3666..3bf0b5b17 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -48,7 +48,7 @@ func (svc *ServiceV2) initData() (err error) { Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] 爬虫任务更新: {{$.status}}", - TemplateMarkdown: `尊敬的 {{$.user.username}}, + Template: `尊敬的 {{$.user.username}}, 请查看下面的任务数据。 @@ -83,7 +83,7 @@ func (svc *ServiceV2) initData() (err error) { Description: "This is the default mail notification. You can edit it with your own settings", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] Task Update: {{$.status}}", - TemplateMarkdown: `Dear {{$.user.username}}, + Template: `Dear {{$.user.username}}, Please find the task data as below. @@ -118,7 +118,7 @@ Please find the task data as below. Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] 任务更新: {{$.status}}", - TemplateMarkdown: `尊敬的 {{$.user.username}}, + Template: `尊敬的 {{$.user.username}}, 请查看下面的任务数据。 @@ -146,7 +146,7 @@ Please find the task data as below. Description: "This is the default mobile notification. You can edit it with your own settings", TaskTrigger: constants.NotificationTriggerTaskFinish, Title: "[Crawlab] Task Update: {{$.status}}", - TemplateMarkdown: `Dear {{$.user.username}}, + Template: `Dear {{$.user.username}}, Please find the task data as below. From 54cf19f679999b35e99de09bc8f4ce6b4693579e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 13 Jul 2024 10:53:38 +0800 Subject: [PATCH 055/106] feat: Add trigger fields to NotificationSettingV2 model --- core/constants/notification.go | 4 ++++ core/models/models/v2/notification_setting_v2.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/core/constants/notification.go b/core/constants/notification.go index 46f289bb7..eb2b8f256 100644 --- a/core/constants/notification.go +++ b/core/constants/notification.go @@ -5,6 +5,10 @@ const ( NotificationTriggerTaskError = "task_error" NotificationTriggerTaskEmptyResults = "task_empty_results" NotificationTriggerTaskNever = "task_never" + NotificationTriggerNodeStatusChange = "node_status_change" + NotificationTriggerNodeOnline = "node_online" + NotificationTriggerNodeOffline = "node_offline" + NotificationTriggerNodeNever = "node_never" ) const ( diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index b19adfa9a..b56ed3a71 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -15,6 +15,8 @@ type NotificationSettingV2 struct { TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + TriggerTarget string `json:"trigger_target" bson:"trigger_target"` + Trigger string `json:"trigger" bson:"trigger"` Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` } From 9073f1ca0297237971bebf6fa4843d80b4fff804 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 13 Jul 2024 11:08:56 +0800 Subject: [PATCH 056/106] refactor: Remove unused code and update dependencies --- core/apps/utils.go | 7 -- core/cmd/root.go | 5 - core/config/version.go | 12 -- core/fs/service_v2_test.go | 115 ++++++++++++------ .../server/model_base_service_v2_server.go | 28 ++--- core/models/client/model_service_v2_test.go | 105 +++++++++------- frontend/index.html | 1 - frontend/public/simplemde/simplemde.js | 15 --- 8 files changed, 154 insertions(+), 134 deletions(-) delete mode 100644 core/config/version.go delete mode 100644 frontend/public/simplemde/simplemde.js diff --git a/core/apps/utils.go b/core/apps/utils.go index 25e48c5e4..a524b4675 100644 --- a/core/apps/utils.go +++ b/core/apps/utils.go @@ -31,10 +31,3 @@ func initModule(name string, fn func() error) (err error) { log.Info(fmt.Sprintf("initialized %s successfully", name)) return nil } - -func initApp(name string, app App) { - _ = initModule(name, func() error { - app.Init() - return nil - }) -} diff --git a/core/cmd/root.go b/core/cmd/root.go index d9a11d2f2..108e016b1 100644 --- a/core/cmd/root.go +++ b/core/cmd/root.go @@ -23,11 +23,6 @@ func Execute() error { return rootCmd.Execute() } -// GetRootCmd get rootCmd instance -func GetRootCmd() *cobra.Command { - return rootCmd -} - func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "c", "", "Use Custom Config File") } diff --git a/core/config/version.go b/core/config/version.go deleted file mode 100644 index 4bcbf0661..000000000 --- a/core/config/version.go +++ /dev/null @@ -1,12 +0,0 @@ -package config - -import "strings" - -const Version = "v0.6.3" - -func GetVersion() (v string) { - if strings.HasPrefix(Version, "v") { - return Version - } - return "v" + Version -} diff --git a/core/fs/service_v2_test.go b/core/fs/service_v2_test.go index 09191305d..f3006f5db 100644 --- a/core/fs/service_v2_test.go +++ b/core/fs/service_v2_test.go @@ -1,7 +1,7 @@ package fs import ( - "io/ioutil" + "github.com/apex/log" "os" "path/filepath" "testing" @@ -10,20 +10,25 @@ import ( ) func TestServiceV2_List(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() testDir := filepath.Join(rootDir, "dir") - os.Mkdir(testDir, 0755) - ioutil.WriteFile(filepath.Join(testDir, "file1.txt"), []byte("hello world"), 0644) - ioutil.WriteFile(filepath.Join(testDir, "file2.txt"), []byte("hello again"), 0644) + _ = os.Mkdir(testDir, 0755) + _ = os.WriteFile(filepath.Join(testDir, "file1.txt"), []byte("hello world"), 0644) + _ = os.WriteFile(filepath.Join(testDir, "file2.txt"), []byte("hello again"), 0644) subDir := filepath.Join(testDir, "subdir") - os.Mkdir(subDir, 0755) - ioutil.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644) - os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion + _ = os.Mkdir(subDir, 0755) + _ = os.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644) + _ = os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion svc := NewFsServiceV2(rootDir) @@ -56,14 +61,19 @@ func TestServiceV2_List(t *testing.T) { } func TestServiceV2_GetFile(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() expectedContent := []byte("hello world") - ioutil.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644) + _ = os.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644) svc := NewFsServiceV2(rootDir) @@ -75,14 +85,19 @@ func TestServiceV2_GetFile(t *testing.T) { } func TestServiceV2_Delete(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() filePath := filepath.Join(rootDir, "file.txt") - ioutil.WriteFile(filePath, []byte("hello world"), 0644) + _ = os.WriteFile(filePath, []byte("hello world"), 0644) svc := NewFsServiceV2(rootDir) @@ -98,11 +113,16 @@ func TestServiceV2_Delete(t *testing.T) { } func TestServiceV2_CreateDir(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) @@ -118,11 +138,16 @@ func TestServiceV2_CreateDir(t *testing.T) { } func TestServiceV2_Save(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) @@ -133,22 +158,27 @@ func TestServiceV2_Save(t *testing.T) { } // Verify the file was saved - data, err := ioutil.ReadFile(filepath.Join(rootDir, "newFile.txt")) + data, err := os.ReadFile(filepath.Join(rootDir, "newFile.txt")) assert.NoError(t, err) assert.Equal(t, "Hello, world!", string(data)) } func TestServiceV2_Rename(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a file to rename - ioutil.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644) + _ = os.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644) // Rename the file err = svc.Rename("oldName.txt", "newName.txt") @@ -162,16 +192,21 @@ func TestServiceV2_Rename(t *testing.T) { } func TestServiceV2_RenameDir(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a directory to rename - os.Mkdir(filepath.Join(rootDir, "oldName"), 0755) + _ = os.Mkdir(filepath.Join(rootDir, "oldName"), 0755) // Rename the directory err = svc.Rename("oldName", "newName") @@ -185,16 +220,21 @@ func TestServiceV2_RenameDir(t *testing.T) { } func TestServiceV2_Copy(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a file to copy - ioutil.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644) + _ = os.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644) // Copy the file err = svc.Copy("source.txt", "copy.txt") @@ -203,23 +243,28 @@ func TestServiceV2_Copy(t *testing.T) { } // Verify the file was copied - data, err := ioutil.ReadFile(filepath.Join(rootDir, "copy.txt")) + data, err := os.ReadFile(filepath.Join(rootDir, "copy.txt")) assert.NoError(t, err) assert.Equal(t, "Hello, world!", string(data)) } func TestServiceV2_CopyDir(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a directory to copy - os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755) - ioutil.WriteFile(filepath.Join(rootDir, "sourceDir", "file.txt"), []byte("Hello, world!"), 0644) + _ = os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755) + _ = os.WriteFile(filepath.Join(rootDir, "sourceDir", "file.txt"), []byte("Hello, world!"), 0644) // Copy the directory err = svc.Copy("sourceDir", "copyDir") @@ -232,7 +277,7 @@ func TestServiceV2_CopyDir(t *testing.T) { assert.NoError(t, err) // Verify the file inside the directory was copied - data, err := ioutil.ReadFile(filepath.Join(rootDir, "copyDir", "file.txt")) + data, err := os.ReadFile(filepath.Join(rootDir, "copyDir", "file.txt")) assert.NoError(t, err) assert.Equal(t, "Hello, world!", string(data)) } diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index eefd389a6..7f356ec4f 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -63,7 +63,7 @@ type ModelBaseServiceServerV2 struct { grpc.UnimplementedModelBaseServiceV2Server } -func (svr ModelBaseServiceServerV2) GetById(ctx context.Context, req *grpc.ModelServiceV2GetByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) GetById(_ context.Context, req *grpc.ModelServiceV2GetByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -76,7 +76,7 @@ func (svr ModelBaseServiceServerV2) GetById(ctx context.Context, req *grpc.Model return HandleSuccessWithData(data) } -func (svr ModelBaseServiceServerV2) GetOne(ctx context.Context, req *grpc.ModelServiceV2GetOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) GetOne(_ context.Context, req *grpc.ModelServiceV2GetOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -95,7 +95,7 @@ func (svr ModelBaseServiceServerV2) GetOne(ctx context.Context, req *grpc.ModelS return HandleSuccessWithData(data) } -func (svr ModelBaseServiceServerV2) GetMany(ctx context.Context, req *grpc.ModelServiceV2GetManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) GetMany(_ context.Context, req *grpc.ModelServiceV2GetManyRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -114,7 +114,7 @@ func (svr ModelBaseServiceServerV2) GetMany(ctx context.Context, req *grpc.Model return HandleSuccessWithData(data) } -func (svr ModelBaseServiceServerV2) DeleteById(ctx context.Context, req *grpc.ModelServiceV2DeleteByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) DeleteById(_ context.Context, req *grpc.ModelServiceV2DeleteByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -127,7 +127,7 @@ func (svr ModelBaseServiceServerV2) DeleteById(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) DeleteOne(ctx context.Context, req *grpc.ModelServiceV2DeleteOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) DeleteOne(_ context.Context, req *grpc.ModelServiceV2DeleteOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -141,7 +141,7 @@ func (svr ModelBaseServiceServerV2) DeleteOne(ctx context.Context, req *grpc.Mod return HandleSuccess() } -func (svr ModelBaseServiceServerV2) DeleteMany(ctx context.Context, req *grpc.ModelServiceV2DeleteManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) DeleteMany(_ context.Context, req *grpc.ModelServiceV2DeleteManyRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -155,7 +155,7 @@ func (svr ModelBaseServiceServerV2) DeleteMany(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) UpdateById(ctx context.Context, req *grpc.ModelServiceV2UpdateByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) UpdateById(_ context.Context, req *grpc.ModelServiceV2UpdateByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -173,7 +173,7 @@ func (svr ModelBaseServiceServerV2) UpdateById(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) UpdateOne(ctx context.Context, req *grpc.ModelServiceV2UpdateOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) UpdateOne(_ context.Context, req *grpc.ModelServiceV2UpdateOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -192,7 +192,7 @@ func (svr ModelBaseServiceServerV2) UpdateOne(ctx context.Context, req *grpc.Mod return HandleSuccess() } -func (svr ModelBaseServiceServerV2) UpdateMany(ctx context.Context, req *grpc.ModelServiceV2UpdateManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) UpdateMany(_ context.Context, req *grpc.ModelServiceV2UpdateManyRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -211,7 +211,7 @@ func (svr ModelBaseServiceServerV2) UpdateMany(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) ReplaceById(ctx context.Context, req *grpc.ModelServiceV2ReplaceByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) ReplaceById(_ context.Context, req *grpc.ModelServiceV2ReplaceByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -231,7 +231,7 @@ func (svr ModelBaseServiceServerV2) ReplaceById(ctx context.Context, req *grpc.M return HandleSuccess() } -func (svr ModelBaseServiceServerV2) ReplaceOne(ctx context.Context, req *grpc.ModelServiceV2ReplaceOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) ReplaceOne(_ context.Context, req *grpc.ModelServiceV2ReplaceOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -252,7 +252,7 @@ func (svr ModelBaseServiceServerV2) ReplaceOne(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) InsertOne(ctx context.Context, req *grpc.ModelServiceV2InsertOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) InsertOne(_ context.Context, req *grpc.ModelServiceV2InsertOneRequest) (res *grpc.Response, err error) { model := GetOneInstanceModel(req.ModelType) modelType := reflect.TypeOf(model) modelValuePtr := reflect.New(modelType).Interface() @@ -268,7 +268,7 @@ func (svr ModelBaseServiceServerV2) InsertOne(ctx context.Context, req *grpc.Mod return HandleSuccessWithData(r.InsertedID) } -func (svr ModelBaseServiceServerV2) InsertMany(ctx context.Context, req *grpc.ModelServiceV2InsertManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) InsertMany(_ context.Context, req *grpc.ModelServiceV2InsertManyRequest) (res *grpc.Response, err error) { model := GetOneInstanceModel(req.ModelType) modelType := reflect.TypeOf(model) modelsSliceType := reflect.SliceOf(modelType) @@ -290,7 +290,7 @@ func (svr ModelBaseServiceServerV2) InsertMany(ctx context.Context, req *grpc.Mo return HandleSuccessWithData(r.InsertedIDs) } -func (svr ModelBaseServiceServerV2) Count(ctx context.Context, req *grpc.ModelServiceV2CountRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) Count(_ context.Context, req *grpc.ModelServiceV2CountRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { diff --git a/core/models/client/model_service_v2_test.go b/core/models/client/model_service_v2_test.go index 7bd5ac5fd..6738af4ae 100644 --- a/core/models/client/model_service_v2_test.go +++ b/core/models/client/model_service_v2_test.go @@ -2,6 +2,7 @@ package client_test import ( "context" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models/v2" @@ -28,13 +29,27 @@ func teardownTestDB() { db.Drop(context.Background()) } +func startSvr(svr *server.GrpcServerV2) { + err := svr.Start() + if err != nil { + log.Errorf("failed to start grpc server: %v", err) + } +} + +func stopSvr(svr *server.GrpcServerV2) { + err := svr.Stop() + if err != nil { + log.Errorf("failed to stop grpc server: %v", err) + } +} + func TestModelServiceV2_GetById(t *testing.T) { setupTestDB() defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -45,7 +60,7 @@ func TestModelServiceV2_GetById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -61,8 +76,8 @@ func TestModelServiceV2_GetOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -73,7 +88,7 @@ func TestModelServiceV2_GetOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -89,8 +104,8 @@ func TestModelServiceV2_GetMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -101,7 +116,7 @@ func TestModelServiceV2_GetMany(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -118,8 +133,8 @@ func TestModelServiceV2_DeleteById(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -130,7 +145,7 @@ func TestModelServiceV2_DeleteById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -148,8 +163,8 @@ func TestModelServiceV2_DeleteOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -160,7 +175,7 @@ func TestModelServiceV2_DeleteOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -178,8 +193,8 @@ func TestModelServiceV2_DeleteMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -190,7 +205,7 @@ func TestModelServiceV2_DeleteMany(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -208,8 +223,8 @@ func TestModelServiceV2_UpdateById(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -220,7 +235,7 @@ func TestModelServiceV2_UpdateById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -238,8 +253,8 @@ func TestModelServiceV2_UpdateOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -250,7 +265,7 @@ func TestModelServiceV2_UpdateOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -268,8 +283,8 @@ func TestModelServiceV2_UpdateMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m1 := TestModel{ Name: "Test Name", @@ -284,7 +299,7 @@ func TestModelServiceV2_UpdateMany(t *testing.T) { require.Nil(t, err) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -302,8 +317,8 @@ func TestModelServiceV2_ReplaceById(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -314,7 +329,7 @@ func TestModelServiceV2_ReplaceById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -333,8 +348,8 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -345,7 +360,7 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -364,10 +379,10 @@ func TestModelServiceV2_InsertOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -388,25 +403,25 @@ func TestModelServiceV2_InsertMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() clientSvc := client.NewModelServiceV2[TestModel]() - models := []TestModel{ + testModels := []TestModel{ {Name: "Test Name 1"}, {Name: "Test Name 2"}, } - ids, err := clientSvc.InsertMany(models) + ids, err := clientSvc.InsertMany(testModels) require.Nil(t, err) for i, id := range ids { res, err := clientSvc.GetById(id) require.Nil(t, err) - assert.Equal(t, res.Name, models[i].Name) + assert.Equal(t, res.Name, testModels[i].Name) } } @@ -415,8 +430,8 @@ func TestModelServiceV2_Count(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) modelSvc := service.NewModelServiceV2[TestModel]() for i := 0; i < 5; i++ { @@ -427,7 +442,7 @@ func TestModelServiceV2_Count(t *testing.T) { } time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() diff --git a/frontend/index.html b/frontend/index.html index c42f06c83..ccaeab84e 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -12,7 +12,6 @@ - ` + +func GetTheme() string { + return defaultTheme +} diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go deleted file mode 100644 index 32c7e5bff..000000000 --- a/core/task/handler/runner.go +++ /dev/null @@ -1,691 +0,0 @@ -package handler - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "github.com/apex/log" - "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - fs2 "github.com/crawlab-team/crawlab/core/fs" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/sys_exec" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/shirou/gopsutil/process" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "time" -) - -type Runner struct { - // dependencies - svc interfaces.TaskHandlerService // task handler service - fsSvc interfaces.FsServiceV2 // task fs service - hookSvc interfaces.TaskHookService // task hook service - - // settings - subscribeTimeout time.Duration - bufferSize int - - // internals - cmd *exec.Cmd // process command instance - pid int // process id - tid primitive.ObjectID // task id - t interfaces.Task // task model.Task - s interfaces.Spider // spider model.Spider - ch chan constants.TaskSignal // channel to communicate between Service and Runner - err error // standard process error - envs []models.Env // environment variables - cwd string // working directory - c interfaces.GrpcClient // grpc client - sub grpc.TaskService_SubscribeClient // grpc task service stream client - - // log internals - scannerStdout *bufio.Reader - scannerStderr *bufio.Reader - logBatchSize int -} - -func (r *Runner) Init() (err error) { - // update task - if err := r.updateTask("", nil); err != nil { - return err - } - - // start grpc client - if !r.c.IsStarted() { - r.c.Start() - } - - // working directory - workspacePath := viper.GetString("workspace") - r.cwd = filepath.Join(workspacePath, r.s.GetId().Hex()) - - // sync files from master - if !utils.IsMaster() { - if err := r.syncFiles(); err != nil { - return err - } - } - - // grpc task service stream client - if err := r.initSub(); err != nil { - return err - } - - // pre actions - if r.hookSvc != nil { - if err := r.hookSvc.PreActions(r.t, r.s, r.fsSvc, r.svc); err != nil { - return err - } - } - - return nil -} - -func (r *Runner) Run() (err error) { - // log task started - log.Infof("task[%s] started", r.tid.Hex()) - - // configure cmd - r.configureCmd() - - // configure environment variables - r.configureEnv() - - // configure logging - r.configureLogging() - - // start process - if err := r.cmd.Start(); err != nil { - return r.updateTask(constants.TaskStatusError, err) - } - - // start logging - go r.startLogging() - - // process id - if r.cmd.Process == nil { - return r.updateTask(constants.TaskStatusError, constants.ErrNotExists) - } - r.pid = r.cmd.Process.Pid - r.t.SetPid(r.pid) - - // update task status (processing) - if err := r.updateTask(constants.TaskStatusRunning, nil); err != nil { - return err - } - - // wait for process to finish - go r.wait() - - // start health check - go r.startHealthCheck() - - // declare task status - status := "" - - // wait for signal - signal := <-r.ch - switch signal { - case constants.TaskSignalFinish: - err = nil - status = constants.TaskStatusFinished - case constants.TaskSignalCancel: - err = constants.ErrTaskCancelled - status = constants.TaskStatusCancelled - case constants.TaskSignalError: - err = r.err - status = constants.TaskStatusError - case constants.TaskSignalLost: - err = constants.ErrTaskLost - status = constants.TaskStatusError - default: - err = constants.ErrInvalidSignal - status = constants.TaskStatusError - } - - // update task status - if err := r.updateTask(status, err); err != nil { - return err - } - - // post actions - if r.hookSvc != nil { - if err := r.hookSvc.PostActions(r.t, r.s, r.fsSvc, r.svc); err != nil { - return err - } - } - - return err -} - -func (r *Runner) Cancel() (err error) { - // kill process - opts := &sys_exec.KillProcessOptions{ - Timeout: r.svc.GetCancelTimeout(), - Force: true, - } - if err := sys_exec.KillProcess(r.cmd, opts); err != nil { - return err - } - - // make sure the process does not exist - op := func() error { - if exists, _ := process.PidExists(int32(r.pid)); exists { - return errors.ErrorTaskProcessStillExists - } - return nil - } - ctx, cancel := context.WithTimeout(context.Background(), r.svc.GetExitWatchDuration()) - defer cancel() - b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) - if err := backoff.Retry(op, b); err != nil { - return trace.TraceError(errors.ErrorTaskUnableToCancel) - } - - return nil -} - -// CleanUp clean up task runner -func (r *Runner) CleanUp() (err error) { - return nil -} - -func (r *Runner) SetSubscribeTimeout(timeout time.Duration) { - r.subscribeTimeout = timeout -} - -func (r *Runner) GetTaskId() (id primitive.ObjectID) { - return r.tid -} - -func (r *Runner) configureCmd() { - var cmdStr string - - // customized spider - if r.t.GetCmd() == "" { - cmdStr = r.s.GetCmd() - } else { - cmdStr = r.t.GetCmd() - } - - // parameters - if r.t.GetParam() != "" { - cmdStr += " " + r.t.GetParam() - } else if r.s.GetParam() != "" { - cmdStr += " " + r.s.GetParam() - } - - // get cmd instance - r.cmd, _ = sys_exec.BuildCmd(cmdStr) - - // set working directory - r.cmd.Dir = r.cwd - - // configure pgid to allow killing sub processes - //sys_exec.SetPgid(r.cmd) -} - -func (r *Runner) configureLogging() { - // set stdout reader - stdout, _ := r.cmd.StdoutPipe() - r.scannerStdout = bufio.NewReaderSize(stdout, r.bufferSize) - - // set stderr reader - stderr, _ := r.cmd.StderrPipe() - r.scannerStderr = bufio.NewReaderSize(stderr, r.bufferSize) -} - -func (r *Runner) startLogging() { - // start reading stdout - go r.startLoggingReaderStdout() - - // start reading stderr - go r.startLoggingReaderStderr() -} - -func (r *Runner) startLoggingReaderStdout() { - for { - line, err := r.scannerStdout.ReadString(byte('\n')) - if err != nil { - break - } - line = strings.TrimSuffix(line, "\n") - r.writeLogLines([]string{line}) - } -} - -func (r *Runner) startLoggingReaderStderr() { - for { - line, err := r.scannerStderr.ReadString(byte('\n')) - if err != nil { - break - } - line = strings.TrimSuffix(line, "\n") - r.writeLogLines([]string{line}) - } -} - -func (r *Runner) startHealthCheck() { - if r.cmd.ProcessState == nil || r.cmd.ProcessState.Exited() { - return - } - for { - exists, _ := process.PidExists(int32(r.pid)) - if !exists { - // process lost - r.ch <- constants.TaskSignalLost - return - } - time.Sleep(1 * time.Second) - } -} - -func (r *Runner) configureEnv() { - // 默认把Node.js的全局node_modules加入环境变量 - envPath := os.Getenv("PATH") - nodePath := "/usr/lib/node_modules" - if !strings.Contains(envPath, nodePath) { - _ = os.Setenv("PATH", nodePath+":"+envPath) - } - _ = os.Setenv("NODE_PATH", nodePath) - - // default envs - r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex()) - if viper.GetString("grpc.address") != "" { - r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address")) - } - if viper.GetString("grpc.authKey") != "" { - r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey")) - } else { - r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+constants.DefaultGrpcAuthKey) - } - - // global environment variables - envs, err := r.svc.GetModelEnvironmentService().GetEnvironmentList(nil, nil) - if err != nil { - trace.PrintError(err) - return - } - for _, env := range envs { - r.cmd.Env = append(r.cmd.Env, env.GetKey()+"="+env.GetValue()) - } -} - -func (r *Runner) syncFiles() (err error) { - masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.GetId().Hex()) - workspacePath := viper.GetString("workspace") - workerDir := filepath.Join(workspacePath, r.s.GetId().Hex()) - - // get file list from master - resp, err := http.Get(masterURL + "/scan") - if err != nil { - fmt.Println("Error getting file list from master:", err) - return trace.TraceError(err) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - fmt.Println("Error reading response body:", err) - return trace.TraceError(err) - } - var masterFiles map[string]entity.FsFileInfo - err = json.Unmarshal(body, &masterFiles) - if err != nil { - fmt.Println("Error unmarshaling JSON:", err) - return trace.TraceError(err) - } - - // create a map for master files - masterFilesMap := make(map[string]entity.FsFileInfo) - for _, file := range masterFiles { - masterFilesMap[file.Path] = file - } - - // create worker directory if not exists - if _, err := os.Stat(workerDir); os.IsNotExist(err) { - if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { - fmt.Println("Error creating worker directory:", err) - return trace.TraceError(err) - } - } - - // get file list from worker - workerFiles, err := utils.ScanDirectory(workerDir) - if err != nil { - fmt.Println("Error scanning worker directory:", err) - return trace.TraceError(err) - } - - // set up wait group and error channel - var wg sync.WaitGroup - errCh := make(chan error, 1) - - // delete files that are deleted on master node - for path, workerFile := range workerFiles { - if _, exists := masterFilesMap[path]; !exists { - fmt.Println("Deleting file:", path) - err := os.Remove(workerFile.FullPath) - if err != nil { - fmt.Println("Error deleting file:", err) - } - } - } - - // download files that are new or modified on master node - for path, masterFile := range masterFilesMap { - workerFile, exists := workerFiles[path] - if !exists || masterFile.Hash != workerFile.Hash { - wg.Add(1) - go func(path string, masterFile entity.FsFileInfo) { - defer wg.Done() - logrus.Infof("File needs to be synchronized: %s", path) - err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) - if err != nil { - logrus.Errorf("Error downloading file: %v", err) - select { - case errCh <- err: - default: - } - } - }(path, masterFile) - } - } - - wg.Wait() - close(errCh) - if err := <-errCh; err != nil { - return err - } - - return nil -} - -func (r *Runner) downloadFile(url string, filePath string) error { - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - out, err := os.Create(filePath) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, resp.Body) - return err -} - -// wait for process to finish and send task signal (constants.TaskSignal) -// to task runner's channel (Runner.ch) according to exit code -func (r *Runner) wait() { - // wait for process to finish - if err := r.cmd.Wait(); err != nil { - exitError, ok := err.(*exec.ExitError) - if !ok { - r.ch <- constants.TaskSignalError - return - } - exitCode := exitError.ExitCode() - if exitCode == -1 { - // cancel error - r.ch <- constants.TaskSignalCancel - return - } - - // standard error - r.err = err - r.ch <- constants.TaskSignalError - return - } - - // success - r.ch <- constants.TaskSignalFinish -} - -// updateTask update and get updated info of task (Runner.t) -func (r *Runner) updateTask(status string, e error) (err error) { - if r.t != nil && status != "" { - // update task status - r.t.SetStatus(status) - if e != nil { - r.t.SetError(e.Error()) - } - if r.svc.GetNodeConfigService().IsMaster() { - if err := delegate.NewModelDelegate(r.t).Save(); err != nil { - return err - } - } else { - if err := client.NewModelDelegate(r.t, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { - return err - } - } - - // send notification - go r.sendNotification() - - // update stats - go func() { - r._updateTaskStat(status) - r._updateSpiderStat(status) - }() - } - - // get task - r.t, err = r.svc.GetTaskById(r.tid) - if err != nil { - return err - } - - return nil -} - -func (r *Runner) initSub() (err error) { - r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) - if err != nil { - return trace.TraceError(err) - } - return nil -} - -func (r *Runner) writeLogLines(lines []string) { - data, err := json.Marshal(&entity.StreamMessageTaskData{ - TaskId: r.tid, - Logs: lines, - }) - if err != nil { - trace.PrintError(err) - return - } - msg := &grpc.StreamMessage{ - Code: grpc.StreamMessageCode_INSERT_LOGS, - Data: data, - } - if err := r.sub.Send(msg); err != nil { - trace.PrintError(err) - return - } -} - -func (r *Runner) _updateTaskStat(status string) { - ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) - if err != nil { - trace.PrintError(err) - return - } - switch status { - case constants.TaskStatusPending: - // do nothing - case constants.TaskStatusRunning: - ts.SetStartTs(time.Now()) - ts.SetWaitDuration(ts.GetStartTs().Sub(ts.GetCreateTs()).Milliseconds()) - case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: - ts.SetEndTs(time.Now()) - ts.SetRuntimeDuration(ts.GetEndTs().Sub(ts.GetStartTs()).Milliseconds()) - ts.SetTotalDuration(ts.GetEndTs().Sub(ts.GetCreateTs()).Milliseconds()) - } - if r.svc.GetNodeConfigService().IsMaster() { - if err := delegate.NewModelDelegate(ts).Save(); err != nil { - trace.PrintError(err) - return - } - } else { - if err := client.NewModelDelegate(ts, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { - trace.PrintError(err) - return - } - } -} - -func (r *Runner) sendNotification() { - data, err := json.Marshal(r.t) - if err != nil { - trace.PrintError(err) - return - } - req := &grpc.Request{ - NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), - Data: data, - } - _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) - if err != nil { - trace.PrintError(err) - return - } -} - -func (r *Runner) _updateSpiderStat(status string) { - // task stat - ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) - if err != nil { - trace.PrintError(err) - return - } - - // update - var update bson.M - switch status { - case constants.TaskStatusPending, constants.TaskStatusRunning: - update = bson.M{ - "$set": bson.M{ - "last_task_id": r.tid, // last task id - }, - "$inc": bson.M{ - "tasks": 1, // task count - "wait_duration": ts.GetWaitDuration(), // wait duration - }, - } - case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: - update = bson.M{ - "$inc": bson.M{ - "results": ts.GetResultCount(), // results - "runtime_duration": ts.GetRuntimeDuration() / 1000, // runtime duration - "total_duration": ts.GetTotalDuration() / 1000, // total duration - }, - } - default: - trace.PrintError(errors.ErrorTaskInvalidType) - return - } - - // perform update - if r.svc.GetNodeConfigService().IsMaster() { - if err := mongo.GetMongoCol(interfaces.ModelColNameSpiderStat).UpdateId(r.s.GetId(), update); err != nil { - trace.PrintError(err) - return - } - } else { - modelSvc, err := client.NewBaseServiceDelegate( - client.WithBaseServiceModelId(interfaces.ModelIdSpiderStat), - client.WithBaseServiceConfigPath(r.svc.GetConfigPath()), - ) - if err != nil { - trace.PrintError(err) - return - } - if err := modelSvc.UpdateById(r.s.GetId(), update); err != nil { - trace.PrintError(err) - return - } - } - -} - -func NewTaskRunner(id primitive.ObjectID, svc interfaces.TaskHandlerService, opts ...RunnerOption) (r2 interfaces.TaskRunner, err error) { - // validate options - if id.IsZero() { - return nil, constants.ErrInvalidOptions - } - - // runner - r := &Runner{ - subscribeTimeout: 30 * time.Second, - bufferSize: 1024 * 1024, - svc: svc, - tid: id, - ch: make(chan constants.TaskSignal), - logBatchSize: 20, - } - - // apply options - for _, opt := range opts { - opt(r) - } - - // task - r.t, err = svc.GetTaskById(id) - if err != nil { - return nil, err - } - - // spider - r.s, err = svc.GetSpiderById(r.t.GetSpiderId()) - if err != nil { - return nil, err - } - - // task fs service - r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.GetId().Hex())) - - // dependency injection - if err := container.GetContainer().Invoke(func( - c interfaces.GrpcClient, - ) { - r.c = c - }); err != nil { - return nil, trace.TraceError(err) - } - - _ = container.GetContainer().Invoke(func(hookSvc interfaces.TaskHookService) { - r.hookSvc = hookSvc - }) - - // initialize task runner - if err := r.Init(); err != nil { - return r, err - } - - return r, nil -} diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 7fec5e18c..b4baac24c 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -605,17 +605,13 @@ func (r *RunnerV2) _updateTaskStat(status string) { } func (r *RunnerV2) sendNotification() { - data, err := json.Marshal(r.t) - if err != nil { - trace.PrintError(err) - return - } - req := &grpc.Request{ + req := &grpc.TaskServiceSendNotificationRequest{ NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), - Data: data, + TaskId: r.tid.Hex(), } - _, err = r.c.TaskClient.SendNotification(context.Background(), req) + _, err := r.c.TaskClient.SendNotification(context.Background(), req) if err != nil { + log.Errorf("Error sending notification: %v", err) trace.PrintError(err) return } diff --git a/core/task/handler/service.go b/core/task/handler/service.go deleted file mode 100644 index 69a99884a..000000000 --- a/core/task/handler/service.go +++ /dev/null @@ -1,506 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - errors2 "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "sync" - "time" -) - -type Service struct { - // dependencies - interfaces.TaskBaseService - cfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - clientModelSvc interfaces.GrpcClientModelService - clientModelNodeSvc interfaces.GrpcClientModelNodeService - clientModelSpiderSvc interfaces.GrpcClientModelSpiderService - clientModelTaskSvc interfaces.GrpcClientModelTaskService - clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService - clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService - c interfaces.GrpcClient // grpc client - - // settings - //maxRunners int - exitWatchDuration time.Duration - reportInterval time.Duration - fetchInterval time.Duration - fetchTimeout time.Duration - cancelTimeout time.Duration - - // internals variables - stopped bool - mu sync.Mutex - runners sync.Map // pool of task runners started - syncLocks sync.Map // files sync locks map of task runners -} - -func (svc *Service) Start() { - // Initialize gRPC if not started - if !svc.c.IsStarted() { - svc.c.Start() - } - - go svc.ReportStatus() - go svc.Fetch() -} - -func (svc *Service) Run(taskId primitive.ObjectID) (err error) { - return svc.run(taskId) -} - -func (svc *Service) Reset() { - svc.mu.Lock() - defer svc.mu.Unlock() -} - -func (svc *Service) Cancel(taskId primitive.ObjectID) (err error) { - r, err := svc.getRunner(taskId) - if err != nil { - return err - } - if err := r.Cancel(); err != nil { - return err - } - return nil -} - -func (svc *Service) Fetch() { - for { - // wait - time.Sleep(svc.fetchInterval) - - // current node - n, err := svc.GetCurrentNode() - if err != nil { - continue - } - - // skip if node is not active or enabled - if !n.GetActive() || !n.GetEnabled() { - continue - } - - // validate if there are available runners - if svc.getRunnerCount() >= n.GetMaxRunners() { - continue - } - - // stop - if svc.stopped { - return - } - - // fetch task - tid, err := svc.fetch() - if err != nil { - trace.PrintError(err) - continue - } - - // skip if no task id - if tid.IsZero() { - continue - } - - // run task - if err := svc.run(tid); err != nil { - trace.PrintError(err) - t, err := svc.GetTaskById(tid) - if err == nil && t.GetStatus() != constants.TaskStatusCancelled { - t.SetError(err.Error()) - _ = svc.SaveTask(t, constants.TaskStatusError) - continue - } - continue - } - } -} - -func (svc *Service) ReportStatus() { - for { - if svc.stopped { - return - } - - // report handler status - if err := svc.reportStatus(); err != nil { - trace.PrintError(err) - } - - // wait - time.Sleep(svc.reportInterval) - } -} - -func (svc *Service) IsSyncLocked(path string) (ok bool) { - _, ok = svc.syncLocks.Load(path) - return ok -} - -func (svc *Service) LockSync(path string) { - svc.syncLocks.Store(path, true) -} - -func (svc *Service) UnlockSync(path string) { - svc.syncLocks.Delete(path) -} - -//func (svc *Service) GetMaxRunners() (maxRunners int) { -// return svc.maxRunners -//} -// -//func (svc *Service) SetMaxRunners(maxRunners int) { -// svc.maxRunners = maxRunners -//} - -func (svc *Service) GetExitWatchDuration() (duration time.Duration) { - return svc.exitWatchDuration -} - -func (svc *Service) SetExitWatchDuration(duration time.Duration) { - svc.exitWatchDuration = duration -} - -func (svc *Service) GetFetchInterval() (interval time.Duration) { - return svc.fetchInterval -} - -func (svc *Service) SetFetchInterval(interval time.Duration) { - svc.fetchInterval = interval -} - -func (svc *Service) GetReportInterval() (interval time.Duration) { - return svc.reportInterval -} - -func (svc *Service) SetReportInterval(interval time.Duration) { - svc.reportInterval = interval -} - -func (svc *Service) GetCancelTimeout() (timeout time.Duration) { - return svc.cancelTimeout -} - -func (svc *Service) SetCancelTimeout(timeout time.Duration) { - svc.cancelTimeout = timeout -} - -func (svc *Service) GetModelService() (modelSvc interfaces.GrpcClientModelService) { - return svc.clientModelSvc -} - -func (svc *Service) GetModelSpiderService() (modelSpiderSvc interfaces.GrpcClientModelSpiderService) { - return svc.clientModelSpiderSvc -} - -func (svc *Service) GetModelTaskService() (modelTaskSvc interfaces.GrpcClientModelTaskService) { - return svc.clientModelTaskSvc -} - -func (svc *Service) GetModelTaskStatService() (modelTaskSvc interfaces.GrpcClientModelTaskStatService) { - return svc.clientModelTaskStatSvc -} - -func (svc *Service) GetModelEnvironmentService() (modelTaskSvc interfaces.GrpcClientModelEnvironmentService) { - return svc.clientModelEnvironmentSvc -} - -func (svc *Service) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) { - return svc.cfgSvc -} - -func (svc *Service) GetCurrentNode() (n interfaces.Node, err error) { - // node key - nodeKey := svc.cfgSvc.GetNodeKey() - - // current node - if svc.cfgSvc.IsMaster() { - n, err = svc.modelSvc.GetNodeByKey(nodeKey, nil) - } else { - n, err = svc.clientModelNodeSvc.GetNodeByKey(nodeKey) - } - if err != nil { - return nil, err - } - - return n, nil -} - -func (svc *Service) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) { - if svc.cfgSvc.IsMaster() { - t, err = svc.modelSvc.GetTaskById(id) - } else { - t, err = svc.clientModelTaskSvc.GetTaskById(id) - } - if err != nil { - return nil, err - } - - return t, nil -} - -func (svc *Service) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) { - if svc.cfgSvc.IsMaster() { - s, err = svc.modelSvc.GetSpiderById(id) - } else { - s, err = svc.clientModelSpiderSvc.GetSpiderById(id) - } - if err != nil { - return nil, err - } - - return s, nil -} - -func (svc *Service) getRunners() (runners []*Runner) { - svc.mu.Lock() - defer svc.mu.Unlock() - svc.runners.Range(func(key, value interface{}) bool { - r := value.(Runner) - runners = append(runners, &r) - return true - }) - return runners -} - -func (svc *Service) getRunnerCount() (count int) { - n, err := svc.GetCurrentNode() - if err != nil { - trace.PrintError(err) - return - } - query := bson.M{ - "node_id": n.GetId(), - "status": constants.TaskStatusRunning, - } - if svc.cfgSvc.IsMaster() { - count, err = svc.modelSvc.GetBaseService(interfaces.ModelIdTask).Count(query) - if err != nil { - trace.PrintError(err) - return - } - } else { - count, err = svc.clientModelTaskSvc.Count(query) - if err != nil { - trace.PrintError(err) - return - } - } - return count -} - -func (svc *Service) getRunner(taskId primitive.ObjectID) (r interfaces.TaskRunner, err error) { - log.Debugf("[TaskHandlerService] getRunner: taskId[%v]", taskId) - v, ok := svc.runners.Load(taskId) - if !ok { - return nil, trace.TraceError(errors2.ErrorTaskNotExists) - } - switch v.(type) { - case interfaces.TaskRunner: - r = v.(interfaces.TaskRunner) - default: - return nil, trace.TraceError(errors2.ErrorModelInvalidType) - } - return r, nil -} - -func (svc *Service) addRunner(taskId primitive.ObjectID, r interfaces.TaskRunner) { - log.Debugf("[TaskHandlerService] addRunner: taskId[%v]", taskId) - svc.runners.Store(taskId, r) -} - -func (svc *Service) deleteRunner(taskId primitive.ObjectID) { - log.Debugf("[TaskHandlerService] deleteRunner: taskId[%v]", taskId) - svc.runners.Delete(taskId) -} - -func (svc *Service) saveTask(t interfaces.Task, status string) (err error) { - // normalize status - if status == "" { - status = constants.TaskStatusPending - } - - // set task status - t.SetStatus(status) - - // attempt to get task from database - _, err = svc.clientModelTaskSvc.GetTaskById(t.GetId()) - if err != nil { - // if task does not exist, add to database - if err == mongo.ErrNoDocuments { - if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Add(); err != nil { - return err - } - return nil - } else { - return err - } - } else { - // otherwise, update - if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Save(); err != nil { - return err - } - return nil - } -} - -func (svc *Service) reportStatus() (err error) { - // current node - n, err := svc.GetCurrentNode() - if err != nil { - return err - } - - // available runners of handler - ar := n.GetMaxRunners() - svc.getRunnerCount() - - // set available runners - n.SetAvailableRunners(ar) - - // save node - if svc.cfgSvc.IsMaster() { - err = delegate.NewModelDelegate(n).Save() - } else { - err = client.NewModelDelegate(n, client.WithDelegateConfigPath(svc.GetConfigPath())).Save() - } - if err != nil { - return err - } - - return nil -} - -func (svc *Service) fetch() (tid primitive.ObjectID, err error) { - ctx, cancel := context.WithTimeout(context.Background(), svc.fetchTimeout) - defer cancel() - res, err := svc.c.GetTaskClient().Fetch(ctx, svc.c.NewRequest(nil)) - if err != nil { - return tid, trace.TraceError(err) - } - if err := json.Unmarshal(res.Data, &tid); err != nil { - return tid, trace.TraceError(err) - } - return tid, nil -} - -func (svc *Service) run(taskId primitive.ObjectID) (err error) { - // attempt to get runner from pool - _, ok := svc.runners.Load(taskId) - if ok { - return trace.TraceError(errors2.ErrorTaskAlreadyExists) - } - - // create a new task runner - r, err := NewTaskRunner(taskId, svc) - if err != nil { - return trace.TraceError(err) - } - - // add runner to pool - svc.addRunner(taskId, r) - - // create a goroutine to run task - go func() { - // delete runner from pool - defer svc.deleteRunner(r.GetTaskId()) - defer func(r interfaces.TaskRunner) { - err := r.CleanUp() - if err != nil { - log.Errorf("task[%s] clean up error: %v", r.GetTaskId().Hex(), err) - } - }(r) - // run task process (blocking) - // error or finish after task runner ends - if err := r.Run(); err != nil { - switch { - case errors.Is(err, constants.ErrTaskError): - log.Errorf("task[%s] finished with error: %v", r.GetTaskId().Hex(), err) - case errors.Is(err, constants.ErrTaskCancelled): - log.Errorf("task[%s] cancelled", r.GetTaskId().Hex()) - default: - log.Errorf("task[%s] finished with unknown error: %v", r.GetTaskId().Hex(), err) - } - } - log.Infof("task[%s] finished", r.GetTaskId().Hex()) - }() - - return nil -} - -func NewTaskHandlerService() (svc2 interfaces.TaskHandlerService, err error) { - // base service - baseSvc, err := task.NewBaseService() - if err != nil { - return nil, trace.TraceError(err) - } - - // service - svc := &Service{ - TaskBaseService: baseSvc, - exitWatchDuration: 60 * time.Second, - fetchInterval: 1 * time.Second, - fetchTimeout: 15 * time.Second, - reportInterval: 5 * time.Second, - cancelTimeout: 5 * time.Second, - mu: sync.Mutex{}, - runners: sync.Map{}, - syncLocks: sync.Map{}, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - modelSvc service.ModelService, - clientModelSvc interfaces.GrpcClientModelService, - clientModelNodeSvc interfaces.GrpcClientModelNodeService, - clientModelSpiderSvc interfaces.GrpcClientModelSpiderService, - clientModelTaskSvc interfaces.GrpcClientModelTaskService, - clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService, - clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService, - c interfaces.GrpcClient, - ) { - svc.cfgSvc = cfgSvc - svc.modelSvc = modelSvc - svc.clientModelSvc = clientModelSvc - svc.clientModelNodeSvc = clientModelNodeSvc - svc.clientModelSpiderSvc = clientModelSpiderSvc - svc.clientModelTaskSvc = clientModelTaskSvc - svc.clientModelTaskStatSvc = clientModelTaskStatSvc - svc.clientModelEnvironmentSvc = clientModelEnvironmentSvc - svc.c = c - }); err != nil { - return nil, trace.TraceError(err) - } - - log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) - - return svc, nil -} - -var _service interfaces.TaskHandlerService - -func GetTaskHandlerService() (svr interfaces.TaskHandlerService, err error) { - if _service != nil { - return _service, nil - } - _service, err = NewTaskHandlerService() - if err != nil { - return nil, err - } - return _service, nil -} diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index be7c2725c..28c2c73c7 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -245,11 +245,11 @@ func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, return s, nil } -func (svc *ServiceV2) getRunners() (runners []*Runner) { +func (svc *ServiceV2) getRunners() (runners []*RunnerV2) { svc.mu.Lock() defer svc.mu.Unlock() svc.runners.Range(func(key, value interface{}) bool { - r := value.(Runner) + r := value.(RunnerV2) runners = append(runners, &r) return true }) diff --git a/grpc/proto/services/task_service.proto b/grpc/proto/services/task_service.proto index f004e5651..6b67caf0b 100644 --- a/grpc/proto/services/task_service.proto +++ b/grpc/proto/services/task_service.proto @@ -7,8 +7,13 @@ import "entity/stream_message.proto"; package grpc; option go_package = ".;grpc"; +message TaskServiceSendNotificationRequest { + string node_key = 1; + string task_id = 2; +} + service TaskService { rpc Subscribe(stream StreamMessage) returns (Response){}; rpc Fetch(Request) returns (Response){}; - rpc SendNotification(Request) returns (Response){}; + rpc SendNotification(TaskServiceSendNotificationRequest) returns (Response){}; } diff --git a/grpc/task_service.pb.go b/grpc/task_service.pb.go index 5d03cbb33..5aa365a99 100644 --- a/grpc/task_service.pb.go +++ b/grpc/task_service.pb.go @@ -10,6 +10,7 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" + sync "sync" ) const ( @@ -19,6 +20,61 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type TaskServiceSendNotificationRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + TaskId string `protobuf:"bytes,2,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` +} + +func (x *TaskServiceSendNotificationRequest) Reset() { + *x = TaskServiceSendNotificationRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_services_task_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TaskServiceSendNotificationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TaskServiceSendNotificationRequest) ProtoMessage() {} + +func (x *TaskServiceSendNotificationRequest) ProtoReflect() protoreflect.Message { + mi := &file_services_task_service_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TaskServiceSendNotificationRequest.ProtoReflect.Descriptor instead. +func (*TaskServiceSendNotificationRequest) Descriptor() ([]byte, []int) { + return file_services_task_service_proto_rawDescGZIP(), []int{0} +} + +func (x *TaskServiceSendNotificationRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *TaskServiceSendNotificationRequest) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + var File_services_task_service_proto protoreflect.FileDescriptor var file_services_task_service_proto_rawDesc = []byte{ @@ -28,33 +84,54 @@ var file_services_task_service_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xa2, 0x01, - 0x0a, 0x0b, 0x54, 0x61, 0x73, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, - 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, - 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x28, 0x01, 0x12, 0x28, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x0d, 0x2e, 0x67, - 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, - 0x10, 0x53, 0x65, 0x6e, 0x64, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x58, 0x0a, + 0x22, 0x54, 0x61, 0x73, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x65, 0x6e, 0x64, + 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x17, + 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x32, 0xbd, 0x01, 0x0a, 0x0b, 0x54, 0x61, 0x73, 0x6b, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, + 0x72, 0x69, 0x62, 0x65, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x28, 0x0a, + 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, 0x4e, + 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x65, + 0x6e, 0x64, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, + 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_services_task_service_proto_rawDescOnce sync.Once + file_services_task_service_proto_rawDescData = file_services_task_service_proto_rawDesc +) + +func file_services_task_service_proto_rawDescGZIP() []byte { + file_services_task_service_proto_rawDescOnce.Do(func() { + file_services_task_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_services_task_service_proto_rawDescData) + }) + return file_services_task_service_proto_rawDescData } +var file_services_task_service_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_services_task_service_proto_goTypes = []any{ - (*StreamMessage)(nil), // 0: grpc.StreamMessage - (*Request)(nil), // 1: grpc.Request - (*Response)(nil), // 2: grpc.Response + (*TaskServiceSendNotificationRequest)(nil), // 0: grpc.TaskServiceSendNotificationRequest + (*StreamMessage)(nil), // 1: grpc.StreamMessage + (*Request)(nil), // 2: grpc.Request + (*Response)(nil), // 3: grpc.Response } var file_services_task_service_proto_depIdxs = []int32{ - 0, // 0: grpc.TaskService.Subscribe:input_type -> grpc.StreamMessage - 1, // 1: grpc.TaskService.Fetch:input_type -> grpc.Request - 1, // 2: grpc.TaskService.SendNotification:input_type -> grpc.Request - 2, // 3: grpc.TaskService.Subscribe:output_type -> grpc.Response - 2, // 4: grpc.TaskService.Fetch:output_type -> grpc.Response - 2, // 5: grpc.TaskService.SendNotification:output_type -> grpc.Response + 1, // 0: grpc.TaskService.Subscribe:input_type -> grpc.StreamMessage + 2, // 1: grpc.TaskService.Fetch:input_type -> grpc.Request + 0, // 2: grpc.TaskService.SendNotification:input_type -> grpc.TaskServiceSendNotificationRequest + 3, // 3: grpc.TaskService.Subscribe:output_type -> grpc.Response + 3, // 4: grpc.TaskService.Fetch:output_type -> grpc.Response + 3, // 5: grpc.TaskService.SendNotification:output_type -> grpc.Response 3, // [3:6] is the sub-list for method output_type 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name @@ -70,18 +147,33 @@ func file_services_task_service_proto_init() { file_entity_request_proto_init() file_entity_response_proto_init() file_entity_stream_message_proto_init() + if !protoimpl.UnsafeEnabled { + file_services_task_service_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*TaskServiceSendNotificationRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_services_task_service_proto_rawDesc, NumEnums: 0, - NumMessages: 0, + NumMessages: 1, NumExtensions: 0, NumServices: 1, }, GoTypes: file_services_task_service_proto_goTypes, DependencyIndexes: file_services_task_service_proto_depIdxs, + MessageInfos: file_services_task_service_proto_msgTypes, }.Build() File_services_task_service_proto = out.File file_services_task_service_proto_rawDesc = nil diff --git a/grpc/task_service_grpc.pb.go b/grpc/task_service_grpc.pb.go index 6cb274dd0..8ae3a4693 100644 --- a/grpc/task_service_grpc.pb.go +++ b/grpc/task_service_grpc.pb.go @@ -30,7 +30,7 @@ const ( type TaskServiceClient interface { Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) - SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + SendNotification(ctx context.Context, in *TaskServiceSendNotificationRequest, opts ...grpc.CallOption) (*Response, error) } type taskServiceClient struct { @@ -86,7 +86,7 @@ func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc return out, nil } -func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { +func (c *taskServiceClient) SendNotification(ctx context.Context, in *TaskServiceSendNotificationRequest, opts ...grpc.CallOption) (*Response, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Response) err := c.cc.Invoke(ctx, TaskService_SendNotification_FullMethodName, in, out, cOpts...) @@ -102,7 +102,7 @@ func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, o type TaskServiceServer interface { Subscribe(TaskService_SubscribeServer) error Fetch(context.Context, *Request) (*Response, error) - SendNotification(context.Context, *Request) (*Response, error) + SendNotification(context.Context, *TaskServiceSendNotificationRequest) (*Response, error) mustEmbedUnimplementedTaskServiceServer() } @@ -116,7 +116,7 @@ func (UnimplementedTaskServiceServer) Subscribe(TaskService_SubscribeServer) err func (UnimplementedTaskServiceServer) Fetch(context.Context, *Request) (*Response, error) { return nil, status.Errorf(codes.Unimplemented, "method Fetch not implemented") } -func (UnimplementedTaskServiceServer) SendNotification(context.Context, *Request) (*Response, error) { +func (UnimplementedTaskServiceServer) SendNotification(context.Context, *TaskServiceSendNotificationRequest) (*Response, error) { return nil, status.Errorf(codes.Unimplemented, "method SendNotification not implemented") } func (UnimplementedTaskServiceServer) mustEmbedUnimplementedTaskServiceServer() {} @@ -177,7 +177,7 @@ func _TaskService_Fetch_Handler(srv interface{}, ctx context.Context, dec func(i } func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Request) + in := new(TaskServiceSendNotificationRequest) if err := dec(in); err != nil { return nil, err } @@ -189,7 +189,7 @@ func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, FullMethod: TaskService_SendNotification_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(TaskServiceServer).SendNotification(ctx, req.(*Request)) + return srv.(TaskServiceServer).SendNotification(ctx, req.(*TaskServiceSendNotificationRequest)) } return interceptor(ctx, in, info, handler) } From a5104663e8fafed9909802b244be39631ebe64ce Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 17 Jul 2024 17:39:07 +0800 Subject: [PATCH 062/106] refactor: Add description, provider, and settings fields to NotificationChannelV2 model --- .../models/v2/notification_channel_v2.go | 11 ++++++++ core/notification/constants.go | 28 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 250d6e6cc..035baeb0d 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -4,5 +4,16 @@ type NotificationChannelV2 struct { any `collection:"notification_channels"` BaseModelV2[NotificationChannelV2] `bson:",inline"` Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` Type string `json:"type" bson:"type"` + Provider string `json:"provider" bson:"provider"` + MailSettings struct { + SMTPServer string `json:"smtp_server" bson:"smtp_server"` + SMTPPort string `json:"smtp_port" bson:"smtp_port"` + SMTPFromEmailAddress string `json:"smtp_from_email_address" bson:"smtp_from_email_address"` + SMTPFromEmailPassword string `json:"smtp_from_email_password" bson:"smtp_from_email_password"` + } `json:"mail_settings,omitempty" bson:"mail_settings,omitempty"` + IMSettings struct { + Webhook string `json:"webhook" bson:"webhook"` + } `json:"im_settings,omitempty" bson:"im_settings,omitempty"` } diff --git a/core/notification/constants.go b/core/notification/constants.go index d3f56b721..a69491c3a 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -6,5 +6,31 @@ const ( ) const ( - SettingsColName = "notification_settings" + ChannelMailProviderGmail = "gmail" + ChannelMailProviderOutlook = "outlook" + ChannelMailProviderYahoo = "yahoo" + ChannelMailProviderHotmail = "hotmail" + ChannelMailProviderAol = "aol" + ChannelMailProviderZoho = "zoho" + ChannelMailProviderYandex = "yandex" + ChannelMailProviderICloud = "icloud" + ChannelMailProviderQQMailProvider = "qq" + ChannelMailProvider163 = "163" + ChannelMailProvider126 = "126" + ChannelMailProviderSina = "sina" + ChannelMailProviderSohu = "sohu" + ChannelMailProvider21CN = "21cn" + ChannelMailProviderTencent = "tencent" + ChannelMailProviderHuawei = "huawei" + ChannelMailProviderAliyun = "aliyun" + + ChannelIMProviderSlack = "slack" // https://api.slack.com/messaging/webhooks + ChannelIMProviderMSTeams = "ms_teams" // https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cjavascript + ChannelIMProviderTelegram = "telegram" // https://core.telegram.org/bots/api + ChannelIMProviderDiscord = "discord" // https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks + ChannelIMProviderWhatsappBusiness = "whatsapp_business" // https://developers.facebook.com/docs/whatsapp/cloud-api/guides/send-messages + ChannelIMProviderFacebookMessenger = "facebook_messenger" // https://developers.facebook.com/docs/messenger-platform/send-messages + ChannelIMProviderWechatWork = "wechat_work" // https://developer.work.weixin.qq.com/document/path/91770 + ChannelIMProviderDingtalk = "dingtalk" // https://open.dingtalk.com/document/orgapp/custom-robot-access + ChannelIMProviderLark = "lark" // https://www.larksuite.com/hc/en-US/articles/099698615114-use-webhook-triggers ) From 821383a677bed7702e2ee557c0bdb4db1677a737 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 18 Jul 2024 00:05:48 +0800 Subject: [PATCH 063/106] refactor: Update SendNotification function to handle old and new settings triggers --- .../server/model_base_service_v2_server.go | 1 + .../models/v2/notification_channel_v2.go | 10 +- .../models/v2/notification_setting_v2.go | 55 +++-- core/node/service/master_service_v2.go | 31 +-- core/notification/constants.go | 4 +- core/notification/{mobile.go => im.go} | 11 +- core/notification/mail.go | 42 ++-- core/notification/service_v2.go | 189 +++--------------- core/spider/admin/service_v2.go | 4 - 9 files changed, 96 insertions(+), 251 deletions(-) rename core/notification/{mobile.go => im.go} (70%) diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 7f356ec4f..59ef7d115 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -27,6 +27,7 @@ var ( *new(models2.GitV2), *new(models2.MetricV2), *new(models2.NodeV2), + *new(models2.NotificationChannelV2), *new(models2.NotificationSettingV2), *new(models2.PermissionV2), *new(models2.ProjectV2), diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 035baeb0d..2f328b79b 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -3,15 +3,15 @@ package models type NotificationChannelV2 struct { any `collection:"notification_channels"` BaseModelV2[NotificationChannelV2] `bson:",inline"` + Type string `json:"type" bson:"type"` Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` - Type string `json:"type" bson:"type"` Provider string `json:"provider" bson:"provider"` MailSettings struct { - SMTPServer string `json:"smtp_server" bson:"smtp_server"` - SMTPPort string `json:"smtp_port" bson:"smtp_port"` - SMTPFromEmailAddress string `json:"smtp_from_email_address" bson:"smtp_from_email_address"` - SMTPFromEmailPassword string `json:"smtp_from_email_password" bson:"smtp_from_email_password"` + SMTPServer string `json:"smtp_server" bson:"smtp_server"` + SMTPPort string `json:"smtp_port" bson:"smtp_port"` + SMTPUser string `json:"smtp_from_email_address" bson:"smtp_from_email_address"` + SMTPPassword string `json:"smtp_from_email_password" bson:"smtp_from_email_password"` } `json:"mail_settings,omitempty" bson:"mail_settings,omitempty"` IMSettings struct { Webhook string `json:"webhook" bson:"webhook"` diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index 7164a5050..6f36c3ec3 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -1,38 +1,33 @@ package models +import "go.mongodb.org/mongo-driver/bson/primitive" + type NotificationSettingV2 struct { any `collection:"notification_settings"` BaseModelV2[NotificationSettingV2] `bson:",inline"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template" bson:"template"` - TemplateMode string `json:"template_mode" bson:"template_mode"` - TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` - TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` - TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` - TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - TriggerTarget string `json:"trigger_target" bson:"trigger_target"` - Trigger string `json:"trigger" bson:"trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` -} + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` -type NotificationSettingMail struct { - Server string `json:"server" bson:"server"` - Port string `json:"port,omitempty" bson:"port,omitempty"` - User string `json:"user,omitempty" bson:"user,omitempty"` - Password string `json:"password,omitempty" bson:"password,omitempty"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` - To string `json:"to,omitempty" bson:"to,omitempty"` - Cc string `json:"cc,omitempty" bson:"cc,omitempty"` -} + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template" bson:"template"` + TemplateMode string `json:"template_mode" bson:"template_mode"` + TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` + TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` + TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` + TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty"` + + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + TriggerTarget string `json:"trigger_target" bson:"trigger_target"` + Trigger string `json:"trigger" bson:"trigger"` + + HasMail bool `json:"has_mail" bson:"has_mail"` + SenderEmail string `json:"sender_email" bson:"sender_email"` + SenderName string `json:"sender_name" bson:"sender_name"` + MailTo string `json:"mail_to" bson:"mail_to"` + MailCc string `json:"mail_cc" bson:"mail_cc"` + MailBcc string `json:"mail_bcc" bson:"mail_bcc"` -type NotificationSettingMobile struct { - Webhook string `json:"webhook" bson:"webhook"` + ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` + Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` } diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 1487547b2..87f9c85dd 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -12,9 +12,7 @@ import ( models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/notification" "github.com/crawlab-team/crawlab/core/schedule" - "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/system" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/task/scheduler" @@ -30,14 +28,12 @@ import ( type MasterServiceV2 struct { // dependencies - cfgSvc interfaces.NodeConfigService - server *server.GrpcServerV2 - schedulerSvc *scheduler.ServiceV2 - handlerSvc *handler.ServiceV2 - scheduleSvc *schedule.ServiceV2 - notificationSvc *notification.ServiceV2 - spiderAdminSvc *admin.ServiceV2 - systemSvc *system.ServiceV2 + cfgSvc interfaces.NodeConfigService + server *server.GrpcServerV2 + schedulerSvc *scheduler.ServiceV2 + handlerSvc *handler.ServiceV2 + scheduleSvc *schedule.ServiceV2 + systemSvc *system.ServiceV2 // settings cfgPath string @@ -77,12 +73,6 @@ func (svc *MasterServiceV2) Start() { // start schedule service go svc.scheduleSvc.Start() - // start notification service - go svc.notificationSvc.Start() - - // start spider admin service - go svc.spiderAdminSvc.Start() - // wait for quit signal svc.Wait() @@ -361,15 +351,6 @@ func newMasterServiceV2() (res *MasterServiceV2, err error) { return nil, err } - // notification service - svc.notificationSvc = notification.GetNotificationServiceV2() - - // spider admin service - svc.spiderAdminSvc, err = admin.GetSpiderAdminServiceV2() - if err != nil { - return nil, err - } - // system service svc.systemSvc = system.GetSystemServiceV2() diff --git a/core/notification/constants.go b/core/notification/constants.go index a69491c3a..b3904af8a 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -1,8 +1,8 @@ package notification const ( - TypeMail = "mail" - TypeMobile = "mobile" + TypeMail = "mail" + TypeIM = "im" ) const ( diff --git a/core/notification/mobile.go b/core/notification/im.go similarity index 70% rename from core/notification/mobile.go rename to core/notification/im.go index 90a238705..3650a4dbe 100644 --- a/core/notification/mobile.go +++ b/core/notification/im.go @@ -2,6 +2,7 @@ package notification import ( "errors" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/trace" "github.com/imroc/req" "strings" @@ -12,7 +13,9 @@ type ResBody struct { ErrMsg string `json:"errmsg"` } -func SendMobileNotification(webhook string, title string, content string) error { +func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) error { + // TODO: compatibility with different IM providers + // request header header := req.Header{ "Content-Type": "application/json; charset=utf-8", @@ -22,7 +25,7 @@ func SendMobileNotification(webhook string, title string, content string) error data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ - "title": title, + "title": s.Title, "text": content, "content": content, }, @@ -32,7 +35,7 @@ func SendMobileNotification(webhook string, title string, content string) error }, "text": content, } - if strings.Contains(strings.ToLower(webhook), "feishu") { + if strings.Contains(strings.ToLower(ch.IMSettings.Webhook), "feishu") { data = req.Param{ "msg_type": "text", "content": req.Param{ @@ -42,7 +45,7 @@ func SendMobileNotification(webhook string, title string, content string) error } // perform request - res, err := req.Post(webhook, header, req.BodyJSON(&data)) + res, err := req.Post(ch.IMSettings.Webhook, header, req.BodyJSON(&data)) if err != nil { return trace.TraceError(err) } diff --git a/core/notification/mail.go b/core/notification/mail.go index 289381aa8..118ca4aec 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -14,23 +14,31 @@ import ( "strings" ) -func SendMail(m *models.NotificationSettingMail, to, cc, title, content string) error { +func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc, title, content string) error { + // mail settings + ms := ch.MailSettings + // config - port, _ := strconv.Atoi(m.Port) - password := m.Password - SMTPUser := m.User + port, err := strconv.Atoi(ms.SMTPPort) + if err != nil { + log.Errorf("failed to convert SMTP port to int: %v", err) + trace.PrintError(err) + return err + } smtpConfig := smtpAuthentication{ - Server: m.Server, + Server: ms.SMTPServer, Port: port, - SenderEmail: m.SenderEmail, - SenderIdentity: m.SenderIdentity, - SMTPPassword: password, - SMTPUser: SMTPUser, + SenderIdentity: s.SenderName, + SenderEmail: s.SenderEmail, + SMTPUser: ms.SMTPUser, + SMTPPassword: ms.SMTPPassword, } + options := sendOptions{ + Subject: title, To: to, Cc: cc, - Subject: title, + Bcc: bcc, } // convert html to text @@ -80,9 +88,10 @@ type smtpAuthentication struct { // sendOptions are options for sending an email type sendOptions struct { - To string Subject string + To string Cc string + Bcc string } // send email @@ -99,14 +108,6 @@ func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, t return errors.New("SMTP user is empty") } - if smtpConfig.SenderIdentity == "" { - return errors.New("SMTP sender identity is empty") - } - - if smtpConfig.SenderEmail == "" { - return errors.New("SMTP sender email is empty") - } - if options.To == "" { return errors.New("no receiver emails configured") } @@ -134,6 +135,9 @@ func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, t if options.Cc != "" { m.SetHeader("Cc", getRecipientList(options.Cc)...) } + if options.Bcc != "" { + m.SetHeader("Bcc", getRecipientList(options.Bcc)...) + } m.SetBody("text/plain", txtBody) m.AddAlternative("text/html", htmlBody) diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 44118333d..ded20c93f 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -7,6 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" + "go.mongodb.org/mongo-driver/bson/primitive" "regexp" "strings" "sync" @@ -15,181 +16,45 @@ import ( type ServiceV2 struct { } -func (svc *ServiceV2) Start() { - // initialize data - if err := svc.initData(); err != nil { - log.Errorf("[NotificationServiceV2] initializing data error: %v", err) - return - } -} - -func (svc *ServiceV2) initData() (err error) { - total, err := service.NewModelServiceV2[models.NotificationSettingV2]().Count(nil) - if err != nil { - return err - } - if total > 0 { - return nil - } - - // data to initialize - settings := []models.NotificationSettingV2{ - { - Type: TypeMail, - Enabled: true, - Name: "任务通知（邮件）", - Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 爬虫任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -|键|值| -|:-:|:--| -|任务状态|{{$.status}}| -|任务优先级|{{$.priority}}| -|任务模式|{{$.mode}}| -|执行命令|{{$.cmd}}| -|执行参数|{{$.param}}| -|错误信息|{{$.error}}| -|节点|{{$.node.name}}| -|爬虫|{{$.spider.name}}| -|项目|{{$.spider.project.name}}| -|定时任务|{{$.schedule.name}}| -|结果数|{{$.:task_stat.result_count}}| -|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| -|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| -|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| -|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: models.NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Type: TypeMail, - Enabled: true, - Name: "Task Change (Mail)", - Description: "This is the default mail notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -|Key|Value| -|:-:|:--| -|Task Status|{{$.status}}| -|Task Priority|{{$.priority}}| -|Task Mode|{{$.mode}}| -|Task Command|{{$.cmd}}| -|Task Params|{{$.param}}| -|Error Message|{{$.error}}| -|Node|{{$.node.name}}| -|Spider|{{$.spider.name}}| -|Project|{{$.spider.project.name}}| -|Schedule|{{$.schedule.name}}| -|Result Count|{{$.:task_stat.result_count}}| -|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| -|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| -|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| -|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: models.NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Type: TypeMobile, - Enabled: true, - Name: "任务通知（移动端）", - Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -- **任务状态**: {{$.status}} -- **任务优先级**: {{$.priority}} -- **任务模式**: {{$.mode}} -- **执行命令**: {{$.cmd}} -- **执行参数**: {{$.param}} -- **错误信息**: {{$.error}} -- **节点**: {{$.node.name}} -- **爬虫**: {{$.spider.name}} -- **项目**: {{$.spider.project.name}} -- **定时任务**: {{$.schedule.name}} -- **结果数**: {{$.:task_stat.result_count}} -- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} -- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} -- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: models.NotificationSettingMobile{}, - }, - { - Type: TypeMobile, - Enabled: true, - Name: "Task Change (Mobile)", - Description: "This is the default mobile notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -- **Task Status**: {{$.status}} -- **Task Priority**: {{$.priority}} -- **Task Mode**: {{$.mode}} -- **Task Command**: {{$.cmd}} -- **Task Params**: {{$.param}} -- **Error Message**: {{$.error}} -- **Node**: {{$.node.name}} -- **Spider**: {{$.spider.name}} -- **Project**: {{$.spider.project.name}} -- **Schedule**: {{$.schedule.name}} -- **Result Count**: {{$.:task_stat.result_count}} -- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} -- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} -- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: models.NotificationSettingMobile{}, - }, - } - _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertMany(settings) - if err != nil { - return err - } - return nil -} - func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { content := svc.getContent(s, args...) - switch s.Type { - case TypeMail: - svc.SendMail(s, content) - case TypeMobile: - svc.SendMobile(s, content) + + wg := sync.WaitGroup{} + wg.Add(len(s.ChannelIds)) + for _, chId := range s.ChannelIds { + go func(chId primitive.ObjectID) { + defer wg.Done() + ch, err := service.NewModelServiceV2[models.NotificationChannelV2]().GetById(chId) + if err != nil { + log.Errorf("[NotificationServiceV2] get channel error: %v", err) + return + } + switch ch.Type { + case TypeMail: + svc.SendMail(s, ch, content) + case TypeIM: + svc.SendIM(s, ch, content) + } + }(chId) } + wg.Wait() } -func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, content string) { +func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) { // TODO: parse to/cc/bcc + mailTo := s.MailTo + mailCc := s.MailCc + mailBcc := s.MailBcc // send mail - err := SendMail(&s.Mail, s.Mail.To, s.Mail.Cc, s.Title, content) + err := SendMail(s, ch, mailTo, mailCc, mailBcc, s.Title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mail error: %v", err) } } -func (svc *ServiceV2) SendMobile(s *models.NotificationSettingV2, content string) { - err := SendMobileNotification(s.Mobile.Webhook, s.Title, content) +func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) { + err := SendIMNotification(s, ch, content) if err != nil { log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index cf821201d..543fe4535 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -29,10 +29,6 @@ type ServiceV2 struct { cfgPath string } -func (svc *ServiceV2) Start() (err error) { - return nil -} - func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // spider s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) From 029c5d1448e13aacb8aa60ff7bc72b6994574dda Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 18 Jul 2024 17:41:54 +0800 Subject: [PATCH 064/106] refactor: Update NotificationChannelV2 model to simplify mail and IM settings --- core/models/models/v2/notification_channel_v2.go | 14 +++++--------- core/notification/constants.go | 3 --- core/notification/im.go | 4 ++-- core/notification/mail.go | 11 ++++------- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 2f328b79b..0b86a6b84 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -7,13 +7,9 @@ type NotificationChannelV2 struct { Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` Provider string `json:"provider" bson:"provider"` - MailSettings struct { - SMTPServer string `json:"smtp_server" bson:"smtp_server"` - SMTPPort string `json:"smtp_port" bson:"smtp_port"` - SMTPUser string `json:"smtp_from_email_address" bson:"smtp_from_email_address"` - SMTPPassword string `json:"smtp_from_email_password" bson:"smtp_from_email_password"` - } `json:"mail_settings,omitempty" bson:"mail_settings,omitempty"` - IMSettings struct { - Webhook string `json:"webhook" bson:"webhook"` - } `json:"im_settings,omitempty" bson:"im_settings,omitempty"` + SMTPServer string `json:"smtp_server" bson:"smtp_server"` + SMTPPort string `json:"smtp_port" bson:"smtp_port"` + SMTPUsername string `json:"smtp_username" bson:"smtp_username"` + SMTPPassword string `json:"smtp_password" bson:"smtp_password"` + WebhookUrl string `json:"webhook_url" bson:"webhook_url"` } diff --git a/core/notification/constants.go b/core/notification/constants.go index b3904af8a..383b04dc6 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -19,10 +19,7 @@ const ( ChannelMailProvider126 = "126" ChannelMailProviderSina = "sina" ChannelMailProviderSohu = "sohu" - ChannelMailProvider21CN = "21cn" ChannelMailProviderTencent = "tencent" - ChannelMailProviderHuawei = "huawei" - ChannelMailProviderAliyun = "aliyun" ChannelIMProviderSlack = "slack" // https://api.slack.com/messaging/webhooks ChannelIMProviderMSTeams = "ms_teams" // https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cjavascript diff --git a/core/notification/im.go b/core/notification/im.go index 3650a4dbe..85d9cac53 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -35,7 +35,7 @@ func SendIMNotification(s *models.NotificationSettingV2, ch *models.Notification }, "text": content, } - if strings.Contains(strings.ToLower(ch.IMSettings.Webhook), "feishu") { + if strings.Contains(strings.ToLower(ch.WebhookUrl), "feishu") { data = req.Param{ "msg_type": "text", "content": req.Param{ @@ -45,7 +45,7 @@ func SendIMNotification(s *models.NotificationSettingV2, ch *models.Notification } // perform request - res, err := req.Post(ch.IMSettings.Webhook, header, req.BodyJSON(&data)) + res, err := req.Post(ch.WebhookUrl, header, req.BodyJSON(&data)) if err != nil { return trace.TraceError(err) } diff --git a/core/notification/mail.go b/core/notification/mail.go index 118ca4aec..b35764f11 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -15,23 +15,20 @@ import ( ) func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc, title, content string) error { - // mail settings - ms := ch.MailSettings - // config - port, err := strconv.Atoi(ms.SMTPPort) + port, err := strconv.Atoi(ch.SMTPPort) if err != nil { log.Errorf("failed to convert SMTP port to int: %v", err) trace.PrintError(err) return err } smtpConfig := smtpAuthentication{ - Server: ms.SMTPServer, + Server: ch.SMTPServer, Port: port, SenderIdentity: s.SenderName, SenderEmail: s.SenderEmail, - SMTPUser: ms.SMTPUser, - SMTPPassword: ms.SMTPPassword, + SMTPUser: ch.SMTPUsername, + SMTPPassword: ch.SMTPPassword, } options := sendOptions{ From b33c61a893e2aa768148e98aaca8b79ca7168528 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 21 Jul 2024 23:34:18 +0800 Subject: [PATCH 065/106] refactor: Update mail provider constants and add new provider for exmail --- core/notification/constants.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/core/notification/constants.go b/core/notification/constants.go index 383b04dc6..810cee543 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -6,20 +6,15 @@ const ( ) const ( - ChannelMailProviderGmail = "gmail" - ChannelMailProviderOutlook = "outlook" - ChannelMailProviderYahoo = "yahoo" - ChannelMailProviderHotmail = "hotmail" - ChannelMailProviderAol = "aol" - ChannelMailProviderZoho = "zoho" - ChannelMailProviderYandex = "yandex" - ChannelMailProviderICloud = "icloud" - ChannelMailProviderQQMailProvider = "qq" - ChannelMailProvider163 = "163" - ChannelMailProvider126 = "126" - ChannelMailProviderSina = "sina" - ChannelMailProviderSohu = "sohu" - ChannelMailProviderTencent = "tencent" + ChannelMailProviderGmail = "gmail" + ChannelMailProviderOutlook = "outlook" + ChannelMailProviderYahoo = "yahoo" + ChannelMailProviderICloud = "icloud" + ChannelMailProviderAol = "aol" + ChannelMailProviderZoho = "zoho" + ChannelMailProviderQQ = "qq" + ChannelMailProvider163 = "163" + ChannelMailProviderExmail = "exmail" ChannelIMProviderSlack = "slack" // https://api.slack.com/messaging/webhooks ChannelIMProviderMSTeams = "ms_teams" // https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cjavascript From c893b5452e02aa5cc8c7b0f87fe9a408101cc1e1 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 22 Jul 2024 17:34:55 +0800 Subject: [PATCH 066/106] refactor: Update SMTPPort type to int in NotificationChannelV2 model --- core/models/models/v2/notification_channel_v2.go | 2 +- core/notification/mail.go | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 0b86a6b84..61698de58 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -8,7 +8,7 @@ type NotificationChannelV2 struct { Description string `json:"description" bson:"description"` Provider string `json:"provider" bson:"provider"` SMTPServer string `json:"smtp_server" bson:"smtp_server"` - SMTPPort string `json:"smtp_port" bson:"smtp_port"` + SMTPPort int `json:"smtp_port" bson:"smtp_port"` SMTPUsername string `json:"smtp_username" bson:"smtp_username"` SMTPPassword string `json:"smtp_password" bson:"smtp_password"` WebhookUrl string `json:"webhook_url" bson:"webhook_url"` diff --git a/core/notification/mail.go b/core/notification/mail.go index b35764f11..5a6eeb19f 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -10,21 +10,14 @@ import ( "net/mail" "regexp" "runtime/debug" - "strconv" "strings" ) func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc, title, content string) error { // config - port, err := strconv.Atoi(ch.SMTPPort) - if err != nil { - log.Errorf("failed to convert SMTP port to int: %v", err) - trace.PrintError(err) - return err - } smtpConfig := smtpAuthentication{ Server: ch.SMTPServer, - Port: port, + Port: ch.SMTPPort, SenderIdentity: s.SenderName, SenderEmail: s.SenderEmail, SMTPUser: ch.SMTPUsername, From 9ffdd3f1cd2eb31efcab31cebf858ba4a64c544c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 23 Jul 2024 12:22:59 +0800 Subject: [PATCH 067/106] refactor: Remove unused code and update models and functions for notification channels and settings --- .../server/model_base_service_v2_server.go | 1 + .../models/v2/notification_request_v2.go | 12 + core/models/models/v2/schedule_v2.go | 1 - core/models/models/v2/task_v2.go | 2 - core/notification/im.go | 51 ++++ core/notification/mail.go | 5 +- core/notification/mail_theme.go | 8 - core/notification/mail_theme_flat.go | 287 ------------------ core/notification/payload.go | 8 - core/notification/service_v2.go | 105 ++++++- core/spider/admin/service_v2.go | 3 - core/task/base.go | 97 ------ core/task/scheduler/service.go | 264 ---------------- core/task/stats/service.go | 155 ---------- 14 files changed, 168 insertions(+), 831 deletions(-) create mode 100644 core/models/models/v2/notification_request_v2.go delete mode 100644 core/notification/mail_theme.go delete mode 100644 core/notification/mail_theme_flat.go delete mode 100644 core/notification/payload.go delete mode 100644 core/task/base.go delete mode 100644 core/task/scheduler/service.go delete mode 100644 core/task/stats/service.go diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 59ef7d115..90eec10dc 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -28,6 +28,7 @@ var ( *new(models2.MetricV2), *new(models2.NodeV2), *new(models2.NotificationChannelV2), + *new(models2.NotificationRequestV2), *new(models2.NotificationSettingV2), *new(models2.PermissionV2), *new(models2.ProjectV2), diff --git a/core/models/models/v2/notification_request_v2.go b/core/models/models/v2/notification_request_v2.go new file mode 100644 index 000000000..1130cc928 --- /dev/null +++ b/core/models/models/v2/notification_request_v2.go @@ -0,0 +1,12 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationRequestV2 struct { + any `collection:"notification_requests"` + BaseModelV2[NotificationRequestV2] `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` +} diff --git a/core/models/models/v2/schedule_v2.go b/core/models/models/v2/schedule_v2.go index a52f1b920..4d8cf42cf 100644 --- a/core/models/models/v2/schedule_v2.go +++ b/core/models/models/v2/schedule_v2.go @@ -19,5 +19,4 @@ type ScheduleV2 struct { NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` Priority int `json:"priority" bson:"priority"` Enabled bool `json:"enabled" bson:"enabled"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/models/v2/task_v2.go b/core/models/models/v2/task_v2.go index 3473b6dba..d3f2b178f 100644 --- a/core/models/models/v2/task_v2.go +++ b/core/models/models/v2/task_v2.go @@ -2,7 +2,6 @@ package models import ( "go.mongodb.org/mongo-driver/bson/primitive" - "time" ) type TaskV2 struct { @@ -26,5 +25,4 @@ type TaskV2 struct { SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` UserId primitive.ObjectID `json:"-" bson:"-"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` } diff --git a/core/notification/im.go b/core/notification/im.go index 85d9cac53..74856f812 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -2,6 +2,7 @@ package notification import ( "errors" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/trace" "github.com/imroc/req" @@ -15,6 +16,10 @@ type ResBody struct { func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) error { // TODO: compatibility with different IM providers + switch ch.Provider { + case ChannelIMProviderLark: + return sendImLark(ch, content) + } // request header header := req.Header{ @@ -63,3 +68,49 @@ func SendIMNotification(s *models.NotificationSettingV2, ch *models.Notification return nil } + +func getIMRequestHeader() req.Header { + return req.Header{ + "Content-Type": "application/json; charset=utf-8", + } +} + +func performIMRequest(webhookUrl string, data req.Param) error { + // perform request + res, err := req.Post(webhookUrl, getIMRequestHeader(), req.BodyJSON(&data)) + if err != nil { + log.Errorf("IM request error: %v", err) + return err + } + + // parse response + var resBody ResBody + if err := res.ToJSON(&resBody); err != nil { + log.Errorf("Parsing IM response error: %v", err) + return err + } + + // validate response code + if resBody.ErrCode != 0 { + log.Errorf("IM response error: %v", resBody.ErrMsg) + return errors.New(resBody.ErrMsg) + } + + return nil +} + +func sendImLark(ch *models.NotificationChannelV2, content string) error { + // request header + data := req.Param{ + "msg_type": "interactive", + "card": req.Param{ + "elements": []req.Param{ + { + "tag": "markdown", + "content": content, + }, + }, + }, + } + return performIMRequest(ch.WebhookUrl, data) +} diff --git a/core/notification/mail.go b/core/notification/mail.go index 5a6eeb19f..549cc0e6b 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -9,7 +9,6 @@ import ( "gopkg.in/gomail.v2" "net/mail" "regexp" - "runtime/debug" "strings" ) @@ -44,8 +43,8 @@ func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, // send the email if err := send(smtpConfig, options, content, text); err != nil { - log.Errorf(err.Error()) - debug.PrintStack() + log.Errorf("failed to send email: %v", err) + trace.PrintError(err) return err } diff --git a/core/notification/mail_theme.go b/core/notification/mail_theme.go deleted file mode 100644 index 9266ddd8c..000000000 --- a/core/notification/mail_theme.go +++ /dev/null @@ -1,8 +0,0 @@ -package notification - -import "github.com/matcornic/hermes/v2" - -type MailTheme interface { - hermes.Theme - GetStyle() string -} diff --git a/core/notification/mail_theme_flat.go b/core/notification/mail_theme_flat.go deleted file mode 100644 index 6edb480f2..000000000 --- a/core/notification/mail_theme_flat.go +++ /dev/null @@ -1,287 +0,0 @@ -package notification - -// MailThemeFlat is a theme -type MailThemeFlat struct{} - -// Name returns the name of the flat theme -func (dt *MailThemeFlat) Name() string { - return "flat" -} - -// HTMLTemplate returns a Golang template that will generate an HTML email. -func (dt *MailThemeFlat) HTMLTemplate() string { - return ` - - - - - - - - - - - - - - - -` -} - -// PlainTextTemplate returns a Golang template that will generate an plain text email. -func (dt *MailThemeFlat) PlainTextTemplate() string { - return `{{ with .Email.Body.Intros }} - {{ range $line := . }} -

{{ $line }}

- {{ end }} -{{ end }} -{{ if (ne .Email.Body.FreeMarkdown "") }} - {{ .Email.Body.FreeMarkdown.ToHTML }} -{{ else }} - {{ with .Email.Body.Dictionary }} -
    - {{ range $entry := . }} -
  • {{ $entry.Key }}: {{ $entry.Value }}
    • - {{ end }} -
- {{ end }} - {{ with .Email.Body.Table }} - {{ $data := .Data }} - {{ $columns := .Columns }} - {{ if gt (len $data) 0 }} - - - {{ $col := index $data 0 }} - {{ range $entry := $col }} - - {{ end }} - - {{ range $row := $data }} - - {{ range $cell := $row }} - - {{ end }} - - {{ end }} -
{{ $entry.Key }}
- {{ $cell.Value }} -
- {{ end }} - {{ end }} - {{ with .Email.Body.Actions }} - {{ range $action := . }} -

{{ $action.Instructions }} {{ $action.Button.Link }}

- {{ end }} - {{ end }} -{{ end }} -{{ with .Email.Body.Outros }} - {{ range $line := . }} -

{{ $line }}

- {{ end }} -{{ end }} -

{{.Email.Body.Signature}},
{{.Hermes.Product.Name}} - {{.Hermes.Product.Link}}

- -

{{.Hermes.Product.Copyright}}

-` -} - -func (dt *MailThemeFlat) GetStyle() string { - return ` - -` -} diff --git a/core/notification/payload.go b/core/notification/payload.go deleted file mode 100644 index 7337ebe0f..000000000 --- a/core/notification/payload.go +++ /dev/null @@ -1,8 +0,0 @@ -package notification - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type SendPayload struct { - TaskId primitive.ObjectID `json:"task_id"` - Data string `json:"data"` -} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index ded20c93f..2f2118ac2 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -11,6 +11,7 @@ import ( "regexp" "strings" "sync" + "time" ) type ServiceV2 struct { @@ -91,14 +92,108 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Id.Hex()) case "status": content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Status) + case "cmd": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Cmd) + case "param": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Param) + case "error": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Error) + case "pid": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Task.Pid)) + case "type": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Type) + case "mode": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Mode) case "priority": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Task.Priority)) + case "created_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.CreatedAt.Format(time.DateTime)) + case "updated_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.UpdatedAt.Format(time.DateTime)) + } + + case "spider": + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Id.Hex()) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Name) + case "type": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Type) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Description) case "mode": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Mode) + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Mode) case "cmd": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Cmd) + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Cmd) case "param": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Param) + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Param) + case "priority": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Spider.Priority)) + case "created_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.CreatedAt.Format(time.DateTime)) + case "updated_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.UpdatedAt.Format(time.DateTime)) + } + + case "node": + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Id.Hex()) + case "key": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Key) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Name) + case "ip": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Ip) + case "port": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Port) + case "hostname": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Hostname) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Description) + case "status": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Status) + case "enabled": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.Enabled)) + case "active": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.Active)) + case "active_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.ActiveAt.Format("2006-01-02 15:04:05")) + case "available_runners": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.AvailableRunners)) + case "max_runners": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.MaxRunners)) + case "created_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.CreatedAt.Format(time.DateTime)) + case "updated_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.UpdatedAt.Format(time.DateTime)) + } + + case "schedule": + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Id.Hex()) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Name) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Description) + case "cron": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Cron) + case "cmd": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Cmd) + case "param": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Param) + case "mode": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Mode) + case "priority": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Schedule.Priority)) + case "enabled": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Schedule.Enabled)) + case "created_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.CreatedAt.Format(time.DateTime)) + case "updated_at": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.UpdatedAt.Format(time.DateTime)) } } } @@ -153,6 +248,10 @@ func (svc *ServiceV2) parseTemplateVariables(template string) (variables []entit return variables } +func (svc *ServiceV2) getUserById(id primitive.ObjectID) { + +} + func newNotificationServiceV2() *ServiceV2 { return &ServiceV2{} } diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index 543fe4535..b3c7136aa 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -15,7 +15,6 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "sync" - "time" ) type ServiceV2 struct { @@ -51,7 +50,6 @@ func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.Spider ScheduleId: opts.ScheduleId, Priority: opts.Priority, UserId: opts.UserId, - CreateTs: time.Now(), } mainTask.SetId(primitive.NewObjectID()) @@ -88,7 +86,6 @@ func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.Spider ScheduleId: opts.ScheduleId, Priority: opts.Priority, UserId: opts.UserId, - CreateTs: time.Now(), } t.SetId(primitive.NewObjectID()) t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) diff --git a/core/task/base.go b/core/task/base.go deleted file mode 100644 index 90de0ac22..000000000 --- a/core/task/base.go +++ /dev/null @@ -1,97 +0,0 @@ -package task - -import ( - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" -) - -type BaseService struct { - // dependencies - interfaces.WithConfigPath - modelSvc service.ModelService - - // internals - stopped bool -} - -func (svc *BaseService) Init() error { - // implement me - return nil -} - -func (svc *BaseService) Start() { - // implement me -} - -func (svc *BaseService) Wait() { - utils.DefaultWait() -} - -func (svc *BaseService) Stop() { - svc.stopped = true -} - -// SaveTask deprecated -func (svc *BaseService) SaveTask(t interfaces.Task, status string) (err error) { - // normalize status - if status == "" { - status = constants.TaskStatusPending - } - - // set task status - t.SetStatus(status) - - // attempt to get task from database - _, err = svc.modelSvc.GetTaskById(t.GetId()) - if err != nil { - // if task does not exist, add to database - if err == mongo.ErrNoDocuments { - if err := delegate.NewModelDelegate(t).Add(); err != nil { - return err - } - return nil - } else { - return err - } - } else { - // otherwise, update - if err := delegate.NewModelDelegate(t).Save(); err != nil { - return err - } - return nil - } -} - -func (svc *BaseService) IsStopped() (res bool) { - return svc.stopped -} - -func (svc *BaseService) GetQueue(nodeId primitive.ObjectID) (queue string) { - if nodeId.IsZero() { - return fmt.Sprintf("%s", constants.TaskListQueuePrefixPublic) - } else { - return fmt.Sprintf("%s:%s", constants.TaskListQueuePrefixNodes, nodeId.Hex()) - } -} - -func NewBaseService() (svc2 interfaces.TaskBaseService, err error) { - svc := &BaseService{} - - // dependency injection - if err := container.GetContainer().Invoke(func(cfgPath interfaces.WithConfigPath, modelSvc service.ModelService) { - svc.WithConfigPath = cfgPath - svc.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - return svc, nil -} diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go deleted file mode 100644 index 3644aa172..000000000 --- a/core/task/scheduler/service.go +++ /dev/null @@ -1,264 +0,0 @@ -package scheduler - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" -) - -type Service struct { - // dependencies - interfaces.TaskBaseService - nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - svr interfaces.GrpcServer - handlerSvc interfaces.TaskHandlerService - - // settings - interval time.Duration -} - -func (svc *Service) Start() { - go svc.initTaskStatus() - go svc.cleanupTasks() - svc.Wait() - svc.Stop() -} - -func (svc *Service) Enqueue(t interfaces.Task) (t2 interfaces.Task, err error) { - // set task status - t.SetStatus(constants.TaskStatusPending) - - // user - var u *models.User - if !t.GetUserId().IsZero() { - u, _ = svc.modelSvc.GetUserById(t.GetUserId()) - } - - // add task - if err = delegate.NewModelDelegate(t, u).Add(); err != nil { - return nil, err - } - - // task queue item - tq := &models.TaskQueueItem{ - Id: t.GetId(), - Priority: t.GetPriority(), - NodeId: t.GetNodeId(), - } - - // task stat - ts := &models.TaskStat{ - Id: t.GetId(), - CreateTs: time.Now(), - } - - // enqueue task - _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Insert(tq) - if err != nil { - return nil, trace.TraceError(err) - } - - // add task stat - _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Insert(ts) - if err != nil { - return nil, trace.TraceError(err) - } - - // success - return t, nil -} - -func (svc *Service) Cancel(id primitive.ObjectID, args ...interface{}) (err error) { - // task - t, err := svc.modelSvc.GetTaskById(id) - if err != nil { - return trace.TraceError(err) - } - - // initial status - initialStatus := t.Status - - // set task status as "cancelled" - _ = svc.SaveTask(t, constants.TaskStatusCancelled) - - // set status of pending tasks as "cancelled" and remove from task item queue - if initialStatus == constants.TaskStatusPending { - // remove from task item queue - if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).DeleteId(t.GetId()); err != nil { - return trace.TraceError(err) - } - return nil - } - - // whether task is running on master node - isMasterTask, err := svc.isMasterNode(t) - if err != nil { - // when error, force status being set as "cancelled" - return svc.SaveTask(t, constants.TaskStatusCancelled) - } - - // node - n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) - if err != nil { - return trace.TraceError(err) - } - - if isMasterTask { - // cancel task on master - if err := svc.handlerSvc.Cancel(id); err != nil { - return trace.TraceError(err) - } - // cancel success - return nil - } else { - // send to cancel task on worker nodes - if err := svc.svr.SendStreamMessageWithData("node:"+n.GetKey(), grpc.StreamMessageCode_CANCEL_TASK, t); err != nil { - return trace.TraceError(err) - } - // cancel success - return nil - } -} - -func (svc *Service) SetInterval(interval time.Duration) { - svc.interval = interval -} - -// initTaskStatus initialize task status of existing tasks -func (svc *Service) initTaskStatus() { - // set status of running tasks as TaskStatusAbnormal - runningTasks, err := svc.modelSvc.GetTaskList(bson.M{ - "status": bson.M{ - "$in": []string{ - constants.TaskStatusPending, - constants.TaskStatusRunning, - }, - }, - }, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - return - } - trace.PrintError(err) - } - for _, t := range runningTasks { - go func(t *models.Task) { - if err := svc.SaveTask(t, constants.TaskStatusAbnormal); err != nil { - trace.PrintError(err) - } - }(&t) - } - if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskQueue).DeleteList(nil); err != nil { - return - } -} - -func (svc *Service) isMasterNode(t *models.Task) (ok bool, err error) { - if t.GetNodeId().IsZero() { - return false, trace.TraceError(errors.ErrorTaskNoNodeId) - } - n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) - if err != nil { - if err == mongo2.ErrNoDocuments { - return false, trace.TraceError(errors.ErrorTaskNodeNotFound) - } - return false, trace.TraceError(err) - } - return n.IsMaster, nil -} - -func (svc *Service) cleanupTasks() { - for { - // task stats over 30 days ago - taskStats, err := svc.modelSvc.GetTaskStatList(bson.M{ - "create_ts": bson.M{ - "$lt": time.Now().Add(-30 * 24 * time.Hour), - }, - }, nil) - if err != nil { - time.Sleep(30 * time.Minute) - continue - } - - // task ids - var ids []primitive.ObjectID - for _, ts := range taskStats { - ids = append(ids, ts.Id) - } - - if len(ids) > 0 { - // remove tasks - if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTask).DeleteList(bson.M{ - "_id": bson.M{"$in": ids}, - }); err != nil { - trace.PrintError(err) - } - - // remove task stats - if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskStat).DeleteList(bson.M{ - "_id": bson.M{"$in": ids}, - }); err != nil { - trace.PrintError(err) - } - } - - time.Sleep(30 * time.Minute) - } -} - -func NewTaskSchedulerService() (svc2 interfaces.TaskSchedulerService, err error) { - // base service - baseSvc, err := task.NewBaseService() - if err != nil { - return nil, trace.TraceError(err) - } - - // service - svc := &Service{ - TaskBaseService: baseSvc, - interval: 5 * time.Second, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - nodeCfgSvc interfaces.NodeConfigService, - modelSvc service.ModelService, - svr interfaces.GrpcServer, - handlerSvc interfaces.TaskHandlerService, - ) { - svc.nodeCfgSvc = nodeCfgSvc - svc.modelSvc = modelSvc - svc.svr = svr - svc.handlerSvc = handlerSvc - }); err != nil { - return nil, err - } - - return svc, nil -} - -var svc interfaces.TaskSchedulerService - -func GetTaskSchedulerService() (svr interfaces.TaskSchedulerService, err error) { - if svc != nil { - return svc, nil - } - svc, err = NewTaskSchedulerService() - if err != nil { - return nil, err - } - return svc, nil -} diff --git a/core/task/stats/service.go b/core/task/stats/service.go deleted file mode 100644 index b6964c297..000000000 --- a/core/task/stats/service.go +++ /dev/null @@ -1,155 +0,0 @@ -package stats - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/crawlab/core/task/log" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "sync" - "time" -) - -type Service struct { - // dependencies - interfaces.TaskBaseService - nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - - // internals - mu sync.Mutex - resultServices sync.Map - rsTtl time.Duration - logDriver log.Driver -} - -func (svc *Service) Init() (err error) { - go svc.cleanup() - return nil -} - -func (svc *Service) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { - resultSvc, err := svc.getResultService(id) - if err != nil { - return err - } - if err := resultSvc.Insert(records...); err != nil { - return err - } - go svc.updateTaskStats(id, len(records)) - return nil -} - -func (svc *Service) InsertLogs(id primitive.ObjectID, logs ...string) (err error) { - return svc.logDriver.WriteLines(id.Hex(), logs) -} - -func (svc *Service) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { - // atomic operation - svc.mu.Lock() - defer svc.mu.Unlock() - - // attempt to get from cache - res, _ := svc.resultServices.Load(id.Hex()) - if res != nil { - // hit in cache - resultSvc, ok := res.(interfaces.ResultService) - resultSvc.SetTime(time.Now()) - if ok { - return resultSvc, nil - } - } - - // task - t, err := svc.modelSvc.GetTaskById(id) - if err != nil { - return nil, err - } - - // result service - resultSvc, err = result.GetResultService(t.SpiderId) - if err != nil { - return nil, err - } - - // store in cache - svc.resultServices.Store(id.Hex(), resultSvc) - - return resultSvc, nil -} - -func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) { - _ = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).UpdateId(id, bson.M{ - "$inc": bson.M{ - "result_count": resultCount, - }, - }) -} - -func (svc *Service) cleanup() { - for { - // atomic operation - svc.mu.Lock() - - svc.resultServices.Range(func(key, value interface{}) bool { - rs := value.(interfaces.ResultService) - if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { - svc.resultServices.Delete(key) - } - return true - }) - - svc.mu.Unlock() - - time.Sleep(10 * time.Minute) - } -} - -func NewTaskStatsService() (svc2 interfaces.TaskStatsService, err error) { - // base service - baseSvc, err := task.NewBaseService() - if err != nil { - return nil, trace.TraceError(err) - } - - // service - svc := &Service{ - mu: sync.Mutex{}, - TaskBaseService: baseSvc, - resultServices: sync.Map{}, - } - - // dependency injection - if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService) { - svc.nodeCfgSvc = nodeCfgSvc - svc.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // log driver - svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) - if err != nil { - return nil, err - } - - return svc, nil -} - -var _service interfaces.TaskStatsService - -func GetTaskStatsService() (svr interfaces.TaskStatsService, err error) { - if _service != nil { - return _service, nil - } - _service, err = NewTaskStatsService() - if err != nil { - return nil, err - } - return _service, nil -} From 7f9fac0fc0415d31d448e5930dbd8705640645ae Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 23 Jul 2024 17:24:40 +0800 Subject: [PATCH 068/106] feat: updated notification settings and channels --- core/go.mod | 47 +-------- core/go.sum | 19 ++++ core/models/models/v2/node_v2.go | 1 - core/models/models/v2/spider_v2.go | 1 - core/models/models/v2/task_stat_v2.go | 2 - core/notification/im.go | 14 ++- core/notification/service_v2.go | 131 ++++++++++++++++++++------ core/spider/admin/service_v2.go | 74 +++++---------- core/task/handler/runner_v2.go | 16 ++-- core/task/scheduler/service_v2.go | 16 ++-- go.work.sum | 4 +- 11 files changed, 176 insertions(+), 149 deletions(-) diff --git a/core/go.mod b/core/go.mod index 06e62076b..8e7030702 100644 --- a/core/go.mod +++ b/core/go.mod @@ -6,7 +6,6 @@ replace ( github.com/crawlab-team/crawlab/db => ../db github.com/crawlab-team/crawlab/fs => ../fs github.com/crawlab-team/crawlab/grpc => ../grpc - github.com/crawlab-team/crawlab/template-parser => ../template-parser github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs ) @@ -18,13 +17,11 @@ require ( github.com/crawlab-team/crawlab/db v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/fs v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/grpc v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/trace v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/vcs v0.0.0-20240701075920-023ba275664a github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 - github.com/gavv/httpexpect/v2 v2.16.0 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 @@ -32,13 +29,11 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-uuid v1.0.3 github.com/imroc/req v0.3.2 - github.com/matcornic/hermes/v2 v2.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 github.com/segmentio/kafka-go v0.4.39 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/sirupsen/logrus v1.9.0 github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.19.0 @@ -57,16 +52,10 @@ require ( ) require ( - github.com/Masterminds/semver v1.4.2 // indirect - github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/PuerkitoBio/goquery v1.9.2 // indirect - github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect - github.com/ajg/form v1.5.1 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/aokoli/goutils v1.0.1 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -76,8 +65,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/fatih/structs v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -89,28 +76,22 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-querystring v1.1.0 // indirect + github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect - github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.2.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect - github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.2 // indirect - github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect @@ -120,26 +101,19 @@ require ( github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.4 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect - github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/olekukonko/tablewriter v0.0.1 // indirect github.com/onsi/gomega v1.30.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sanity-io/litter v1.5.5 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect @@ -148,28 +122,17 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.34.0 // indirect - github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 // indirect - github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - github.com/yudai/gojsondiff v1.0.0 // indirect - github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect @@ -192,8 +155,6 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - moul.io/http2curl/v2 v2.3.0 // indirect ) diff --git a/core/go.sum b/core/go.sum index 9bfd5e460..5fd1eb825 100644 --- a/core/go.sum +++ b/core/go.sum @@ -69,6 +69,7 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= @@ -83,6 +84,7 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -112,6 +114,7 @@ github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -289,6 +292,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 h1:saBP362Qm7zDdDXqv61kI4rzhmLFq3Z1gx34xpl6cWE= +github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -393,6 +398,7 @@ github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/imroc/req v0.3.2 h1:M/JkeU6RPmX+WYvT2vaaOL0K+q8ufL5LxwvJc4xeB4o= +github.com/imroc/req v0.3.2/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -409,6 +415,7 @@ github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8 github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -426,20 +433,24 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -467,6 +478,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -556,6 +568,7 @@ github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -763,6 +776,7 @@ go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQc go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= +go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -842,6 +856,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -924,6 +939,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1043,6 +1059,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1051,6 +1068,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1066,6 +1084,7 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/core/models/models/v2/node_v2.go b/core/models/models/v2/node_v2.go index 6d5a84421..968b9bc9e 100644 --- a/core/models/models/v2/node_v2.go +++ b/core/models/models/v2/node_v2.go @@ -10,7 +10,6 @@ type NodeV2 struct { Key string `json:"key" bson:"key"` Name string `json:"name" bson:"name"` Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` Mac string `json:"mac" bson:"mac"` Hostname string `json:"hostname" bson:"hostname"` Description string `json:"description" bson:"description"` diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go index ac3bf3007..24b4cb059 100644 --- a/core/models/models/v2/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -8,7 +8,6 @@ type SpiderV2 struct { any `collection:"spiders"` BaseModelV2[SpiderV2] `bson:",inline"` Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id ColName string `json:"col_name,omitempty" bson:"-"` // data collection name DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id diff --git a/core/models/models/v2/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go index 5456946d4..65459244e 100644 --- a/core/models/models/v2/task_stat_v2.go +++ b/core/models/models/v2/task_stat_v2.go @@ -7,12 +7,10 @@ import ( type TaskStatV2 struct { any `collection:"task_stats"` BaseModelV2[TaskStatV2] `bson:",inline"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` } diff --git a/core/notification/im.go b/core/notification/im.go index 74856f812..12f0162ba 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -14,11 +14,11 @@ type ResBody struct { ErrMsg string `json:"errmsg"` } -func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) error { +func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) error { // TODO: compatibility with different IM providers switch ch.Provider { case ChannelIMProviderLark: - return sendImLark(ch, content) + return sendImLark(ch, title, content) } // request header @@ -30,7 +30,7 @@ func SendIMNotification(s *models.NotificationSettingV2, ch *models.Notification data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ - "title": s.Title, + "title": title, "text": content, "content": content, }, @@ -99,11 +99,17 @@ func performIMRequest(webhookUrl string, data req.Param) error { return nil } -func sendImLark(ch *models.NotificationChannelV2, content string) error { +func sendImLark(ch *models.NotificationChannelV2, title, content string) error { // request header data := req.Param{ "msg_type": "interactive", "card": req.Param{ + "header": req.Param{ + "title": req.Param{ + "tag": "plain_text", + "content": title, + }, + }, "elements": []req.Param{ { "tag": "markdown", diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 2f2118ac2..343d3023b 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -7,6 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gomarkdown/markdown" "go.mongodb.org/mongo-driver/bson/primitive" "regexp" "strings" @@ -18,6 +19,7 @@ type ServiceV2 struct { } func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { + title := s.Title content := svc.getContent(s, args...) wg := sync.WaitGroup{} @@ -32,30 +34,30 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { } switch ch.Type { case TypeMail: - svc.SendMail(s, ch, content) + svc.SendMail(s, ch, title, content) case TypeIM: - svc.SendIM(s, ch, content) + svc.SendIM(s, ch, title, content) } }(chId) } wg.Wait() } -func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) { +func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { // TODO: parse to/cc/bcc mailTo := s.MailTo mailCc := s.MailCc mailBcc := s.MailBcc // send mail - err := SendMail(s, ch, mailTo, mailCc, mailBcc, s.Title, content) + err := SendMail(s, ch, mailTo, mailCc, mailBcc, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mail error: %v", err) } } -func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) { - err := SendIMNotification(s, ch, content) +func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { + err := SendIMNotification(s, ch, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } @@ -68,7 +70,9 @@ func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, args ...any) ( switch s.TemplateMode { case constants.NotificationTemplateModeMarkdown: variables := svc.parseTemplateVariables(s.TemplateMarkdown) - return svc.getTaskContent(s.TemplateMarkdown, variables, vd) + content = svc.getTaskContent(s.TemplateMarkdown, variables, vd) + content = svc.convertMarkdownToHtml(content) + return content case constants.NotificationTemplateModeRichText: variables := svc.parseTemplateVariables(s.TemplateRichText) return svc.getTaskContent(s.TemplateRichText, variables, vd) @@ -87,6 +91,10 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific for _, v := range variables { switch v.Category { case "task": + if vd.Task == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Id.Hex()) @@ -106,20 +114,46 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Mode) case "priority": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Task.Priority)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Task.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Task.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Task.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Task.UpdatedBy)) + } + + case "task_stat": + if vd.TaskStat == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "start_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.TaskStat.StartTs)) + case "end_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.TaskStat.EndTs)) + case "wait_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.WaitDuration/1000)) + case "runtime_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.RuntimeDuration/1000)) + case "total_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.TotalDuration/1000)) + case "result_count": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.TaskStat.ResultCount)) } case "spider": + if vd.Spider == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Id.Hex()) case "name": content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Name) - case "type": - content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Type) case "description": content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Description) case "mode": @@ -130,13 +164,21 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Param) case "priority": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Spider.Priority)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Spider.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Spider.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Spider.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Spider.UpdatedBy)) } case "node": + if vd.Node == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Id.Hex()) @@ -146,8 +188,8 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Name) case "ip": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Ip) - case "port": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Port) + case "mac": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Mac) case "hostname": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Hostname) case "description": @@ -159,18 +201,26 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific case "active": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.Active)) case "active_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.ActiveAt.Format("2006-01-02 15:04:05")) + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.ActiveAt)) case "available_runners": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.AvailableRunners)) case "max_runners": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.MaxRunners)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Node.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Node.UpdatedBy)) } case "schedule": + if vd.Schedule == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Id.Hex()) @@ -190,10 +240,14 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Schedule.Priority)) case "enabled": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Schedule.Enabled)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Schedule.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Schedule.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.UpdatedBy)) } } } @@ -248,8 +302,27 @@ func (svc *ServiceV2) parseTemplateVariables(template string) (variables []entit return variables } -func (svc *ServiceV2) getUserById(id primitive.ObjectID) { +func (svc *ServiceV2) getUsernameById(id primitive.ObjectID) (username string) { + if id.IsZero() { + return "" + } + u, err := service.NewModelServiceV2[models.UserV2]().GetById(id) + if err != nil { + log.Errorf("[NotificationServiceV2] get user error: %v", err) + return "" + } + return u.Username +} + +func (svc *ServiceV2) getFormattedTime(t time.Time) (res string) { + if t.IsZero() { + return "N/A" + } + return t.Local().Format(time.DateTime) +} +func (svc *ServiceV2) convertMarkdownToHtml(content string) (html string) { + return string(markdown.ToHTML([]byte(content), nil, nil)) } func newNotificationServiceV2() *ServiceV2 { diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index b3c7136aa..8fcba6c9c 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -41,7 +41,7 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task - mainTask := &models2.TaskV2{ + t := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, NodeIds: opts.NodeIds, @@ -49,66 +49,38 @@ func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.Spider Param: opts.Param, ScheduleId: opts.ScheduleId, Priority: opts.Priority, - UserId: opts.UserId, } - mainTask.SetId(primitive.NewObjectID()) + t.SetId(primitive.NewObjectID()) // normalize - if mainTask.Mode == "" { - mainTask.Mode = s.Mode + if t.Mode == "" { + t.Mode = s.Mode } - if mainTask.NodeIds == nil { - mainTask.NodeIds = s.NodeIds + if t.NodeIds == nil { + t.NodeIds = s.NodeIds } - if mainTask.Cmd == "" { - mainTask.Cmd = s.Cmd + if t.Cmd == "" { + t.Cmd = s.Cmd } - if mainTask.Param == "" { - mainTask.Param = s.Param + if t.Param == "" { + t.Param = s.Param } - if mainTask.Priority == 0 { - mainTask.Priority = s.Priority + if t.Priority == 0 { + t.Priority = s.Priority } - if svc.isMultiTask(opts) { - // multi tasks - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - for _, nodeId := range nodeIds { - t := &models2.TaskV2{ - SpiderId: s.Id, - Mode: opts.Mode, - Cmd: opts.Cmd, - Param: opts.Param, - NodeId: nodeId, - ScheduleId: opts.ScheduleId, - Priority: opts.Priority, - UserId: opts.UserId, - } - t.SetId(primitive.NewObjectID()) - t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.Id) - } - } else { - // single task - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - if len(nodeIds) > 0 { - mainTask.NodeId = nodeIds[0] - } - t2, err := svc.schedulerSvc.Enqueue(mainTask, opts.UserId) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.Id) + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + if len(nodeIds) > 0 { + t.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) + if err != nil { + return nil, err } + taskIds = append(taskIds, t2.Id) return taskIds, nil } diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index b4baac24c..3788661fe 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -522,14 +522,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { } } + // update stats + r._updateTaskStat(status) + r._updateSpiderStat(status) + // send notification go r.sendNotification() - - // update stats - go func() { - r._updateTaskStat(status) - r._updateSpiderStat(status) - }() } // get task @@ -579,15 +577,15 @@ func (r *RunnerV2) _updateTaskStat(status string) { // do nothing case constants.TaskStatusRunning: ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: if ts.StartTs.IsZero() { ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } ts.EndTs = time.Now() ts.RuntimeDuration = ts.EndTs.Sub(ts.StartTs).Milliseconds() - ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() + ts.TotalDuration = ts.EndTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index 81d5907e7..ef4d7ac78 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -39,12 +39,12 @@ func (svc *ServiceV2) Start() { func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) { // set task status t.Status = constants.TaskStatusPending - t.SetCreatedBy(by) + t.SetCreated(by) t.SetUpdated(by) // add task taskModelSvc := service.NewModelServiceV2[models2.TaskV2]() - _, err = taskModelSvc.InsertOne(*t) + id, err := taskModelSvc.InsertOne(*t) if err != nil { return nil, err } @@ -54,13 +54,15 @@ func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *mod Priority: t.Priority, NodeId: t.NodeId, } - tq.SetId(t.Id) + tq.SetId(id) + tq.SetCreated(by) + tq.SetUpdated(by) // task stat - ts := models2.TaskStatV2{ - CreateTs: time.Now(), - } - ts.SetId(t.Id) + ts := models2.TaskStatV2{} + ts.SetId(id) + ts.SetCreated(by) + ts.SetUpdated(by) // enqueue task _, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq) diff --git a/go.work.sum b/go.work.sum index b43210105..99ae07af1 100644 --- a/go.work.sum +++ b/go.work.sum @@ -582,6 +582,7 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fasthttp/websocket v1.4.3-rc.6 h1:omHqsl8j+KXpmzRjF8bmzOSYJ8GnS0E3efi1wYT+niY= github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CACNqNvCqQCoty/Lc= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -607,7 +608,6 @@ github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -723,7 +723,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= @@ -765,6 +764,7 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= From 13c6a20ee3c38abf05e5f42c65d3ad2830380186 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 23 Jul 2024 17:45:52 +0800 Subject: [PATCH 069/106] feat: Add Slack as a new IM provider for notification channels --- core/notification/im.go | 13 +++++++++++++ core/notification/service_v2.go | 16 +++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/core/notification/im.go b/core/notification/im.go index 12f0162ba..6fb55629c 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -19,6 +19,8 @@ func SendIMNotification(s *models.NotificationSettingV2, ch *models.Notification switch ch.Provider { case ChannelIMProviderLark: return sendImLark(ch, title, content) + case ChannelIMProviderSlack: + return sendImSlack(ch, title, content) } // request header @@ -120,3 +122,14 @@ func sendImLark(ch *models.NotificationChannelV2, title, content string) error { } return performIMRequest(ch.WebhookUrl, data) } + +func sendImSlack(ch *models.NotificationChannelV2, title, content string) error { + // request header + data := req.Param{ + "blocks": []req.Param{ + {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, + {"type": "section", "text": req.Param{"type": "mrkdwn", "text": content}}, + }, + } + return performIMRequest(ch.WebhookUrl, data) +} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 343d3023b..c41c3a262 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -20,7 +20,6 @@ type ServiceV2 struct { func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { title := s.Title - content := svc.getContent(s, args...) wg := sync.WaitGroup{} wg.Add(len(s.ChannelIds)) @@ -32,6 +31,7 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { log.Errorf("[NotificationServiceV2] get channel error: %v", err) return } + content := svc.getContent(s, ch, args...) switch ch.Type { case TypeMail: svc.SendMail(s, ch, title, content) @@ -63,7 +63,7 @@ func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.Notific } } -func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, args ...any) (content string) { +func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, args ...any) (content string) { switch s.TriggerTarget { case constants.NotificationTriggerTargetTask: vd := svc.getTaskVariableData(args...) @@ -71,11 +71,17 @@ func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, args ...any) ( case constants.NotificationTemplateModeMarkdown: variables := svc.parseTemplateVariables(s.TemplateMarkdown) content = svc.getTaskContent(s.TemplateMarkdown, variables, vd) - content = svc.convertMarkdownToHtml(content) + if ch.Type == TypeMail { + content = svc.convertMarkdownToHtml(content) + } return content case constants.NotificationTemplateModeRichText: - variables := svc.parseTemplateVariables(s.TemplateRichText) - return svc.getTaskContent(s.TemplateRichText, variables, vd) + template := s.TemplateRichText + if ch.Type == TypeIM { + template = s.TemplateMarkdown + } + variables := svc.parseTemplateVariables(template) + return svc.getTaskContent(template, variables, vd) } case constants.NotificationTriggerTargetNode: From 923921c17ad4d05ea9cd006c33e5af6952f6d6ca Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 23 Jul 2024 23:43:33 +0800 Subject: [PATCH 070/106] refactor: updated go.work.sum --- go.work.sum | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/go.work.sum b/go.work.sum index 99ae07af1..0c5e3202d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -514,11 +514,9 @@ github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dX github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE= github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= @@ -541,7 +539,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -608,6 +605,7 @@ github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -700,22 +698,16 @@ github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHL github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= -github.com/imroc/req v0.3.2/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= -github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= -github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= -github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.18.0/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= -github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -723,6 +715,7 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= @@ -732,7 +725,6 @@ github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHz github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/knz/go-libedit v1.10.1 h1:0pHpWtx9vcvC0xGZqEQlQdfSQs7WRlAjuPvk3fOZDCo= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= @@ -769,7 +761,6 @@ github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= @@ -850,7 +841,6 @@ go.etcd.io/etcd/client/v2 v2.305.12 h1:0m4ovXYo1CHaA/Mp3X/Fak5sRNIWf01wk/X1/G3sG go.etcd.io/etcd/client/v2 v2.305.12/go.mod h1:aQ/yhsxMu+Oht1FOupSr60oBvcS9cKXHrzBpDsPTf9E= go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= -go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -903,7 +893,6 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -912,7 +901,6 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -940,7 +928,6 @@ golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -952,8 +939,6 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= From 7b1fa48fd9385c0ba480521675de85a309a8eecb Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 24 Jul 2024 17:00:35 +0800 Subject: [PATCH 071/106] feat: support notification for node --- core/grpc/server/node_server_v2.go | 11 ++++ core/grpc/server/task_server_v2.go | 4 ++ core/node/service/master_service_v2.go | 28 +++++++-- core/notification/entity.go | 2 +- core/notification/service_v2.go | 78 ++++++++++++++++++-------- 5 files changed, 92 insertions(+), 31 deletions(-) diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index 269ca7ef3..d1936e3eb 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -10,6 +10,8 @@ import ( "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/notification" + "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/grpc" errors2 "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" @@ -93,6 +95,7 @@ func (svr NodeServerV2) SendHeartbeat(_ context.Context, req *grpc.NodeServiceSe } return HandleError(err) } + oldStatus := node.Status // validate status if node.Status == constants.NodeStatusUnregistered { @@ -107,6 +110,14 @@ func (svr NodeServerV2) SendHeartbeat(_ context.Context, req *grpc.NodeServiceSe if err != nil { return HandleError(err) } + newStatus := node.Status + + // send notification if status changed + if utils.IsPro() { + if oldStatus != newStatus { + go notification.GetNotificationServiceV2().SendNodeNotification(node) + } + } return HandleSuccessWithData(node) } diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index 9cc28ed9c..d90760fce 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -110,6 +110,10 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo } func (svr TaskServerV2) SendNotification(_ context.Context, request *grpc.TaskServiceSendNotificationRequest) (response *grpc.Response, err error) { + if !utils.IsPro() { + return nil, nil + } + // task id taskId, err := primitive.ObjectIDFromHex(request.TaskId) if err != nil { diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 87f9c85dd..580a4f57d 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -12,6 +12,7 @@ import ( models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/notification" "github.com/crawlab-team/crawlab/core/schedule" "github.com/crawlab-team/crawlab/core/system" "github.com/crawlab-team/crawlab/core/task/handler" @@ -207,11 +208,12 @@ func (svc *MasterServiceV2) monitor() (err error) { wg.Add(len(workerNodes)) for _, n := range workerNodes { go func(n *models2.NodeV2) { + defer wg.Done() + // subscribe ok := svc.subscribeNode(n) if !ok { go svc.setWorkerNodeOffline(n) - wg.Done() return } @@ -219,19 +221,14 @@ func (svc *MasterServiceV2) monitor() (err error) { ok = svc.pingNodeClient(n) if !ok { go svc.setWorkerNodeOffline(n) - wg.Done() return } // update node available runners if err := svc.updateNodeAvailableRunners(n); err != nil { trace.PrintError(err) - wg.Done() return } - - // done - wg.Done() }(&n) } @@ -261,13 +258,24 @@ func (svc *MasterServiceV2) updateMasterNodeStatus() (err error) { if err != nil { return err } + oldStatus := node.Status + node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() + newStatus := node.Status + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } + + if utils.IsPro() { + if oldStatus != newStatus { + go svc.sendNotification(node) + } + } + return nil } @@ -280,6 +288,7 @@ func (svc *MasterServiceV2) setWorkerNodeOffline(node *models2.NodeV2) { if err != nil { trace.PrintError(err) } + svc.sendNotification(node) } func (svc *MasterServiceV2) subscribeNode(n *models2.NodeV2) (ok bool) { @@ -316,6 +325,13 @@ func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models2.NodeV2) (er return nil } +func (svc *MasterServiceV2) sendNotification(node *models2.NodeV2) { + if !utils.IsPro() { + return + } + go notification.GetNotificationServiceV2().SendNodeNotification(node) +} + func newMasterServiceV2() (res *MasterServiceV2, err error) { // master service svc := &MasterServiceV2{ diff --git a/core/notification/entity.go b/core/notification/entity.go index 40b69eec2..b7e410270 100644 --- a/core/notification/entity.go +++ b/core/notification/entity.go @@ -2,7 +2,7 @@ package notification import "github.com/crawlab-team/crawlab/core/models/models/v2" -type VariableDataTask struct { +type VariableData struct { Task *models.TaskV2 `json:"task"` TaskStat *models.TaskStatV2 `json:"task_stat"` Spider *models.SpiderV2 `json:"spider"` diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index c41c3a262..fae1ee167 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -7,7 +7,9 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/trace" "github.com/gomarkdown/markdown" + "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "regexp" "strings" @@ -64,35 +66,28 @@ func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.Notific } func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, args ...any) (content string) { - switch s.TriggerTarget { - case constants.NotificationTriggerTargetTask: - vd := svc.getTaskVariableData(args...) - switch s.TemplateMode { - case constants.NotificationTemplateModeMarkdown: - variables := svc.parseTemplateVariables(s.TemplateMarkdown) - content = svc.getTaskContent(s.TemplateMarkdown, variables, vd) - if ch.Type == TypeMail { - content = svc.convertMarkdownToHtml(content) - } - return content - case constants.NotificationTemplateModeRichText: - template := s.TemplateRichText - if ch.Type == TypeIM { - template = s.TemplateMarkdown - } - variables := svc.parseTemplateVariables(template) - return svc.getTaskContent(template, variables, vd) + vd := svc.getVariableData(args...) + switch s.TemplateMode { + case constants.NotificationTemplateModeMarkdown: + variables := svc.parseTemplateVariables(s.TemplateMarkdown) + content = svc.geContentWithVariables(s.TemplateMarkdown, variables, vd) + if ch.Type == TypeMail { + content = svc.convertMarkdownToHtml(content) } - - case constants.NotificationTriggerTargetNode: - // TODO: implement - + return content + case constants.NotificationTemplateModeRichText: + template := s.TemplateRichText + if ch.Type == TypeIM { + template = s.TemplateMarkdown + } + variables := svc.parseTemplateVariables(template) + return svc.geContentWithVariables(template, variables, vd) } return content } -func (svc *ServiceV2) getTaskContent(template string, variables []entity.NotificationVariable, vd VariableDataTask) (content string) { +func (svc *ServiceV2) geContentWithVariables(template string, variables []entity.NotificationVariable, vd VariableData) (content string) { content = template for _, v := range variables { switch v.Category { @@ -192,6 +187,8 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Key) case "name": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Name) + case "is_master": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.IsMaster)) case "ip": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Ip) case "mac": @@ -260,7 +257,7 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific return content } -func (svc *ServiceV2) getTaskVariableData(args ...any) (vd VariableDataTask) { +func (svc *ServiceV2) getVariableData(args ...any) (vd VariableData) { for _, arg := range args { switch arg.(type) { case *models.TaskV2: @@ -331,6 +328,39 @@ func (svc *ServiceV2) convertMarkdownToHtml(content string) (html string) { return string(markdown.ToHTML([]byte(content), nil, nil)) } +func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { + // arguments + var args []any + args = append(args, node) + + // settings + settings, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetMany(bson.M{ + "enabled": true, + "trigger_target": constants.NotificationTriggerTargetNode, + }, nil) + if err != nil { + log.Errorf("get notification settings error: %v", err) + trace.PrintError(err) + return + } + + for _, s := range settings { + // send notification + switch s.Trigger { + case constants.NotificationTriggerNodeStatusChange: + go svc.Send(&s, args...) + case constants.NotificationTriggerNodeOnline: + if node.Status == constants.NodeStatusOnline { + go svc.Send(&s, args...) + } + case constants.NotificationTriggerNodeOffline: + if node.Status == constants.NodeStatusOffline { + go svc.Send(&s, args...) + } + } + } +} + func newNotificationServiceV2() *ServiceV2 { return &ServiceV2{} } From 46839f103e75dd977404cd59f290f0163c449a2d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 24 Jul 2024 17:58:05 +0800 Subject: [PATCH 072/106] refactor: Update IM notification function to support DingTalk and Wechat Work providers --- core/notification/im.go | 66 +++++++++++++++++++++++++++++++-- core/notification/service_v2.go | 6 +-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/core/notification/im.go b/core/notification/im.go index 6fb55629c..63a16ea92 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -2,10 +2,12 @@ package notification import ( "errors" + "fmt" "github.com/apex/log" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/trace" "github.com/imroc/req" + "regexp" "strings" ) @@ -14,13 +16,17 @@ type ResBody struct { ErrMsg string `json:"errmsg"` } -func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) error { +func SendIMNotification(ch *models.NotificationChannelV2, title, content string) error { // TODO: compatibility with different IM providers switch ch.Provider { case ChannelIMProviderLark: return sendImLark(ch, title, content) case ChannelIMProviderSlack: return sendImSlack(ch, title, content) + case ChannelIMProviderDingtalk: + return sendImDingTalk(ch, title, content) + case ChannelIMProviderWechatWork: + return sendImWechatWork(ch, title, content) } // request header @@ -101,8 +107,40 @@ func performIMRequest(webhookUrl string, data req.Param) error { return nil } +func convertMarkdownToSlack(markdown string) string { + // Convert bold text + reBold := regexp.MustCompile(`\*\*(.*?)\*\*`) + slack := reBold.ReplaceAllString(markdown, `*$1*`) + + // Convert italic text + reItalic := regexp.MustCompile(`\*(.*?)\*`) + slack = reItalic.ReplaceAllString(slack, `_$1_`) + + // Convert links + reLink := regexp.MustCompile(`\[(.*?)\]\((.*?)\)`) + slack = reLink.ReplaceAllString(slack, `<$2|$1>`) + + // Convert inline code + reInlineCode := regexp.MustCompile("`(.*?)`") + slack = reInlineCode.ReplaceAllString(slack, "`$1`") + + // Convert unordered list + slack = strings.ReplaceAll(slack, "- ", "• ") + + // Convert ordered list + reOrderedList := regexp.MustCompile(`^\d+\. `) + slack = reOrderedList.ReplaceAllStringFunc(slack, func(s string) string { + return strings.Replace(s, ". ", ". ", 1) + }) + + // Convert blockquote + reBlockquote := regexp.MustCompile(`^> (.*)`) + slack = reBlockquote.ReplaceAllString(slack, `> $1`) + + return slack +} + func sendImLark(ch *models.NotificationChannelV2, title, content string) error { - // request header data := req.Param{ "msg_type": "interactive", "card": req.Param{ @@ -124,11 +162,31 @@ func sendImLark(ch *models.NotificationChannelV2, title, content string) error { } func sendImSlack(ch *models.NotificationChannelV2, title, content string) error { - // request header data := req.Param{ "blocks": []req.Param{ {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, - {"type": "section", "text": req.Param{"type": "mrkdwn", "text": content}}, + {"type": "section", "text": req.Param{"type": "mrkdwn", "text": convertMarkdownToSlack(content)}}, + }, + } + return performIMRequest(ch.WebhookUrl, data) +} + +func sendImDingTalk(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "title": title, + "text": fmt.Sprintf("# %s\n\n%s", title, content), + }, + } + return performIMRequest(ch.WebhookUrl, data) +} + +func sendImWechatWork(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "content": fmt.Sprintf("# %s\n\n%s", title, content), }, } return performIMRequest(ch.WebhookUrl, data) diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index fae1ee167..407e61e62 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -38,7 +38,7 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { case TypeMail: svc.SendMail(s, ch, title, content) case TypeIM: - svc.SendIM(s, ch, title, content) + svc.SendIM(ch, title, content) } }(chId) } @@ -58,8 +58,8 @@ func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.Notif } } -func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { - err := SendIMNotification(s, ch, title, content) +func (svc *ServiceV2) SendIM(ch *models.NotificationChannelV2, title, content string) { + err := SendIMNotification(ch, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } From 70cc86c4d3b9d67bd6b292dc04fee16904def5f3 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 25 Jul 2024 14:40:37 +0800 Subject: [PATCH 073/106] refactor: Update NotificationChannelV2 model to include Telegram notification settings --- .../models/v2/notification_channel_v2.go | 12 +- core/notification/constants.go | 16 +-- core/notification/im.go | 130 +++++++++++++++--- 3 files changed, 128 insertions(+), 30 deletions(-) diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 61698de58..955076187 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -7,9 +7,11 @@ type NotificationChannelV2 struct { Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` Provider string `json:"provider" bson:"provider"` - SMTPServer string `json:"smtp_server" bson:"smtp_server"` - SMTPPort int `json:"smtp_port" bson:"smtp_port"` - SMTPUsername string `json:"smtp_username" bson:"smtp_username"` - SMTPPassword string `json:"smtp_password" bson:"smtp_password"` - WebhookUrl string `json:"webhook_url" bson:"webhook_url"` + SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` + SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` + SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` + SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` + WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` + TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` + TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` } diff --git a/core/notification/constants.go b/core/notification/constants.go index 810cee543..6c483d4be 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -16,13 +16,11 @@ const ( ChannelMailProvider163 = "163" ChannelMailProviderExmail = "exmail" - ChannelIMProviderSlack = "slack" // https://api.slack.com/messaging/webhooks - ChannelIMProviderMSTeams = "ms_teams" // https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cjavascript - ChannelIMProviderTelegram = "telegram" // https://core.telegram.org/bots/api - ChannelIMProviderDiscord = "discord" // https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks - ChannelIMProviderWhatsappBusiness = "whatsapp_business" // https://developers.facebook.com/docs/whatsapp/cloud-api/guides/send-messages - ChannelIMProviderFacebookMessenger = "facebook_messenger" // https://developers.facebook.com/docs/messenger-platform/send-messages - ChannelIMProviderWechatWork = "wechat_work" // https://developer.work.weixin.qq.com/document/path/91770 - ChannelIMProviderDingtalk = "dingtalk" // https://open.dingtalk.com/document/orgapp/custom-robot-access - ChannelIMProviderLark = "lark" // https://www.larksuite.com/hc/en-US/articles/099698615114-use-webhook-triggers + ChannelIMProviderSlack = "slack" // https://api.slack.com/messaging/webhooks + ChannelIMProviderTelegram = "telegram" // https://core.telegram.org/bots/api + ChannelIMProviderDiscord = "discord" // https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks + ChannelIMProviderMSTeams = "ms_teams" // https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cjavascript + ChannelIMProviderWechatWork = "wechat_work" // https://developer.work.weixin.qq.com/document/path/91770 + ChannelIMProviderDingtalk = "dingtalk" // https://open.dingtalk.com/document/orgapp/custom-robot-access + ChannelIMProviderLark = "lark" // https://www.larksuite.com/hc/en-US/articles/099698615114-use-webhook-triggers ) diff --git a/core/notification/im.go b/core/notification/im.go index 63a16ea92..099c3b37f 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -27,6 +27,8 @@ func SendIMNotification(ch *models.NotificationChannelV2, title, content string) return sendImDingTalk(ch, title, content) case ChannelIMProviderWechatWork: return sendImWechatWork(ch, title, content) + case ChannelIMProviderTelegram: + return sendImTelegram(ch, title, content) } // request header @@ -83,28 +85,27 @@ func getIMRequestHeader() req.Header { } } -func performIMRequest(webhookUrl string, data req.Param) error { +func performIMRequest[T any](webhookUrl string, data req.Param) (resBody T, err error) { // perform request res, err := req.Post(webhookUrl, getIMRequestHeader(), req.BodyJSON(&data)) if err != nil { log.Errorf("IM request error: %v", err) - return err + return resBody, err } // parse response - var resBody ResBody if err := res.ToJSON(&resBody); err != nil { - log.Errorf("Parsing IM response error: %v", err) - return err - } - - // validate response code - if resBody.ErrCode != 0 { - log.Errorf("IM response error: %v", resBody.ErrMsg) - return errors.New(resBody.ErrMsg) + log.Warnf("Parsing IM response error: %v", err) + resText, err := res.ToString() + if err != nil { + log.Warnf("Converting response to string error: %v", err) + return resBody, err + } + log.Infof("IM response: %s", resText) + return resBody, nil } - return nil + return resBody, nil } func convertMarkdownToSlack(markdown string) string { @@ -140,6 +141,36 @@ func convertMarkdownToSlack(markdown string) string { return slack } +func convertMarkdownToTelegram(markdownText string) string { + // Combined regex to handle bold and italic + re := regexp.MustCompile(`(?m)(\*\*)(.*)(\*\*)|(__)(.*)(__)|(\*)(.*)(\*)|(_)(.*)(_)`) + markdownText = re.ReplaceAllStringFunc(markdownText, func(match string) string { + groups := re.FindStringSubmatch(match) + if groups[1] != "" || groups[4] != "" { + // Handle bold + return "*" + match[2:len(match)-2] + "*" + } else if groups[6] != "" || groups[9] != "" { + // Handle italic + return "_" + match[1:len(match)-1] + "_" + } else { + // No match + return match + } + }) + + // Convert unordered list + re = regexp.MustCompile(`(?m)^- (.*)`) + markdownText = re.ReplaceAllString(markdownText, "• $1") + + // Escape characters + escapeChars := []string{"#", "-", "."} + for _, c := range escapeChars { + markdownText = strings.ReplaceAll(markdownText, c, "\\"+c) + } + + return markdownText +} + func sendImLark(ch *models.NotificationChannelV2, title, content string) error { data := req.Param{ "msg_type": "interactive", @@ -158,7 +189,14 @@ func sendImLark(ch *models.NotificationChannelV2, title, content string) error { }, }, } - return performIMRequest(ch.WebhookUrl, data) + resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil } func sendImSlack(ch *models.NotificationChannelV2, title, content string) error { @@ -168,7 +206,14 @@ func sendImSlack(ch *models.NotificationChannelV2, title, content string) error {"type": "section", "text": req.Param{"type": "mrkdwn", "text": convertMarkdownToSlack(content)}}, }, } - return performIMRequest(ch.WebhookUrl, data) + resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil } func sendImDingTalk(ch *models.NotificationChannelV2, title string, content string) error { @@ -179,7 +224,14 @@ func sendImDingTalk(ch *models.NotificationChannelV2, title string, content stri "text": fmt.Sprintf("# %s\n\n%s", title, content), }, } - return performIMRequest(ch.WebhookUrl, data) + resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil } func sendImWechatWork(ch *models.NotificationChannelV2, title string, content string) error { @@ -189,5 +241,51 @@ func sendImWechatWork(ch *models.NotificationChannelV2, title string, content st "content": fmt.Sprintf("# %s\n\n%s", title, content), }, } - return performIMRequest(ch.WebhookUrl, data) + resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil +} + +func sendImTelegram(ch *models.NotificationChannelV2, title string, content string) error { + type ResBody struct { + Ok bool `json:"ok"` + Description string `json:"description"` + } + + // chat id + chatId := ch.TelegramChatId + if !strings.HasPrefix("@", ch.TelegramChatId) { + chatId = fmt.Sprintf("@%s", ch.TelegramChatId) + } + + // webhook url + webhookUrl := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", ch.TelegramBotToken) + + // original Markdown text + text := fmt.Sprintf("**%s**\n\n%s", title, content) + + // convert to Telegram MarkdownV2 + text = convertMarkdownToTelegram(text) + + // request data + data := req.Param{ + "chat_id": chatId, + "text": text, + "parse_mode": "MarkdownV2", + } + + // perform request + resBody, err := performIMRequest[ResBody](webhookUrl, data) + if err != nil { + return err + } + if !resBody.Ok { + return errors.New(resBody.Description) + } + return nil } From df563a5e2d7e5f73c85ae1d8e6a1774c90e9805f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 25 Jul 2024 16:38:37 +0800 Subject: [PATCH 074/106] feat: Add Discord and MSTeams as new IM providers for notification channels --- core/notification/im.go | 85 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/core/notification/im.go b/core/notification/im.go index 099c3b37f..f394eddde 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -20,15 +20,19 @@ func SendIMNotification(ch *models.NotificationChannelV2, title, content string) // TODO: compatibility with different IM providers switch ch.Provider { case ChannelIMProviderLark: - return sendImLark(ch, title, content) + return sendIMLark(ch, title, content) case ChannelIMProviderSlack: - return sendImSlack(ch, title, content) + return sendIMSlack(ch, title, content) case ChannelIMProviderDingtalk: - return sendImDingTalk(ch, title, content) + return sendIMDingTalk(ch, title, content) case ChannelIMProviderWechatWork: - return sendImWechatWork(ch, title, content) + return sendIMWechatWork(ch, title, content) case ChannelIMProviderTelegram: - return sendImTelegram(ch, title, content) + return sendIMTelegram(ch, title, content) + case ChannelIMProviderDiscord: + return sendIMDiscord(ch, title, content) + case ChannelIMProviderMSTeams: + return sendIMMSTeams(ch, title, content) } // request header @@ -93,6 +97,15 @@ func performIMRequest[T any](webhookUrl string, data req.Param) (resBody T, err return resBody, err } + // get response + response := res.Response() + + // check status code + if response.StatusCode >= 400 { + log.Errorf("IM response status code: %d", res.Response().StatusCode) + return resBody, errors.New(fmt.Sprintf("IM error response %d: %s", response.StatusCode, res.String())) + } + // parse response if err := res.ToJSON(&resBody); err != nil { log.Warnf("Parsing IM response error: %v", err) @@ -171,7 +184,7 @@ func convertMarkdownToTelegram(markdownText string) string { return markdownText } -func sendImLark(ch *models.NotificationChannelV2, title, content string) error { +func sendIMLark(ch *models.NotificationChannelV2, title, content string) error { data := req.Param{ "msg_type": "interactive", "card": req.Param{ @@ -199,7 +212,7 @@ func sendImLark(ch *models.NotificationChannelV2, title, content string) error { return nil } -func sendImSlack(ch *models.NotificationChannelV2, title, content string) error { +func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error { data := req.Param{ "blocks": []req.Param{ {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, @@ -216,7 +229,7 @@ func sendImSlack(ch *models.NotificationChannelV2, title, content string) error return nil } -func sendImDingTalk(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content string) error { data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ @@ -234,7 +247,7 @@ func sendImDingTalk(ch *models.NotificationChannelV2, title string, content stri return nil } -func sendImWechatWork(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content string) error { data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ @@ -251,7 +264,7 @@ func sendImWechatWork(ch *models.NotificationChannelV2, title string, content st return nil } -func sendImTelegram(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMTelegram(ch *models.NotificationChannelV2, title string, content string) error { type ResBody struct { Ok bool `json:"ok"` Description string `json:"description"` @@ -289,3 +302,55 @@ func sendImTelegram(ch *models.NotificationChannelV2, title string, content stri } return nil } + +func sendIMDiscord(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "embeds": []req.Param{ + { + "title": title, + "description": content, + }, + }, + } + resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil +} + +func sendIMMSTeams(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "type": "message", + "attachments": []req.Param{{ + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": nil, + "content": req.Param{ + "$schema": "https://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.2", + "body": []req.Param{ + { + "type": "TextBlock", + "text": title, + }, + { + "type": "TextBlock", + "text": content, + }, + }, + }, + }}, + } + resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil +} From eef3eea777d5e510c54c68ad3e7edc78a09eaac6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 25 Jul 2024 16:50:03 +0800 Subject: [PATCH 075/106] refactor: Refactor IM notification function to improve code structure and readability --- core/notification/im.go | 42 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/core/notification/im.go b/core/notification/im.go index f394eddde..3af47cd41 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -89,12 +89,12 @@ func getIMRequestHeader() req.Header { } } -func performIMRequest[T any](webhookUrl string, data req.Param) (resBody T, err error) { +func performIMRequest(webhookUrl string, data req.Param) (res *req.Resp, err error) { // perform request - res, err := req.Post(webhookUrl, getIMRequestHeader(), req.BodyJSON(&data)) + res, err = req.Post(webhookUrl, getIMRequestHeader(), req.BodyJSON(&data)) if err != nil { log.Errorf("IM request error: %v", err) - return resBody, err + return nil, err } // get response @@ -103,7 +103,16 @@ func performIMRequest[T any](webhookUrl string, data req.Param) (resBody T, err // check status code if response.StatusCode >= 400 { log.Errorf("IM response status code: %d", res.Response().StatusCode) - return resBody, errors.New(fmt.Sprintf("IM error response %d: %s", response.StatusCode, res.String())) + return nil, errors.New(fmt.Sprintf("IM error response %d: %s", response.StatusCode, res.String())) + } + + return res, nil +} + +func performIMRequestWithJson[T any](webhookUrl string, data req.Param) (resBody T, err error) { + res, err := performIMRequest(webhookUrl, data) + if err != nil { + return resBody, err } // parse response @@ -202,7 +211,7 @@ func sendIMLark(ch *models.NotificationChannelV2, title, content string) error { }, }, } - resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) if err != nil { return err } @@ -219,7 +228,7 @@ func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error {"type": "section", "text": req.Param{"type": "mrkdwn", "text": convertMarkdownToSlack(content)}}, }, } - resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) if err != nil { return err } @@ -237,7 +246,7 @@ func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content stri "text": fmt.Sprintf("# %s\n\n%s", title, content), }, } - resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) if err != nil { return err } @@ -254,7 +263,7 @@ func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content st "content": fmt.Sprintf("# %s\n\n%s", title, content), }, } - resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) if err != nil { return err } @@ -293,13 +302,10 @@ func sendIMTelegram(ch *models.NotificationChannelV2, title string, content stri } // perform request - resBody, err := performIMRequest[ResBody](webhookUrl, data) + _, err := performIMRequest(webhookUrl, data) if err != nil { return err } - if !resBody.Ok { - return errors.New(resBody.Description) - } return nil } @@ -312,13 +318,10 @@ func sendIMDiscord(ch *models.NotificationChannelV2, title string, content strin }, }, } - resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + _, err := performIMRequest(ch.WebhookUrl, data) if err != nil { return err } - if resBody.ErrCode != 0 { - return errors.New(resBody.ErrMsg) - } return nil } @@ -335,7 +338,7 @@ func sendIMMSTeams(ch *models.NotificationChannelV2, title string, content strin "body": []req.Param{ { "type": "TextBlock", - "text": title, + "text": fmt.Sprintf("**%s**", title), }, { "type": "TextBlock", @@ -345,12 +348,9 @@ func sendIMMSTeams(ch *models.NotificationChannelV2, title string, content strin }, }}, } - resBody, err := performIMRequest[ResBody](ch.WebhookUrl, data) + _, err := performIMRequest(ch.WebhookUrl, data) if err != nil { return err } - if resBody.ErrCode != 0 { - return errors.New(resBody.ErrMsg) - } return nil } From 96b89d05b8b807150628f6f24cce786f32d2d635 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 25 Jul 2024 22:58:39 +0800 Subject: [PATCH 076/106] refactor: Update NotificationChannelV2 model to include Telegram notification settings --- core/models/common/index_service.go | 145 ------------------ core/models/common/index_service_v2.go | 25 +++ core/models/delegate/model_user_role_test.go | 97 ------------ .../models/v2/notification_request_v2.go | 12 +- core/notification/constants.go | 5 + core/notification/im.go | 32 ++-- core/notification/service_v2.go | 29 +++- 7 files changed, 79 insertions(+), 266 deletions(-) delete mode 100644 core/models/common/index_service.go delete mode 100644 core/models/delegate/model_user_role_test.go diff --git a/core/models/common/index_service.go b/core/models/common/index_service.go deleted file mode 100644 index ff75b92e8..000000000 --- a/core/models/common/index_service.go +++ /dev/null @@ -1,145 +0,0 @@ -package common - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func CreateIndexes() { - // artifacts - mongo.GetMongoCol(interfaces.ModelColNameArtifact).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"_col": 1}}, - {Keys: bson.M{"_del": 1}}, - {Keys: bson.M{"_tid": 1}}, - }) - - // tags - mongo.GetMongoCol(interfaces.ModelColNameTag).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"col": 1}}, - {Keys: bson.M{"name": 1}}, - }) - - // nodes - mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, // key - {Keys: bson.M{"name": 1}}, // name - {Keys: bson.M{"is_master": 1}}, // is_master - {Keys: bson.M{"status": 1}}, // status - {Keys: bson.M{"enabled": 1}}, // enabled - {Keys: bson.M{"active": 1}}, // active - }) - - // projects - mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // spiders - mongo.GetMongoCol(interfaces.ModelColNameSpider).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - {Keys: bson.M{"type": 1}}, - {Keys: bson.M{"col_id": 1}}, - {Keys: bson.M{"project_id": 1}}, - }) - - // tasks - mongo.GetMongoCol(interfaces.ModelColNameTask).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"spider_id": 1}}, - {Keys: bson.M{"status": 1}}, - {Keys: bson.M{"node_id": 1}}, - {Keys: bson.M{"schedule_id": 1}}, - {Keys: bson.M{"type": 1}}, - {Keys: bson.M{"mode": 1}}, - {Keys: bson.M{"priority": 1}}, - {Keys: bson.M{"parent_id": 1}}, - {Keys: bson.M{"has_sub": 1}}, - {Keys: bson.M{"create_ts": -1}}, - }) - - // task stats - mongo.GetMongoCol(interfaces.ModelColNameTaskStat).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"create_ts": 1}}, - }) - - // schedules - mongo.GetMongoCol(interfaces.ModelColNameSchedule).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - {Keys: bson.M{"spider_id": 1}}, - {Keys: bson.M{"enabled": 1}}, - }) - - // users - mongo.GetMongoCol(interfaces.ModelColNameUser).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"username": 1}}, - {Keys: bson.M{"role": 1}}, - {Keys: bson.M{"email": 1}}, - }) - - // settings - mongo.GetMongoCol(interfaces.ModelColNameSetting).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, - }) - - // tokens - mongo.GetMongoCol(interfaces.ModelColNameToken).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // variables - mongo.GetMongoCol(interfaces.ModelColNameVariable).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, - }) - - // data sources - mongo.GetMongoCol(interfaces.ModelColNameDataSource).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // data collections - mongo.GetMongoCol(interfaces.ModelColNameDataCollection).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // extra values - mongo.GetMongoCol(interfaces.ModelColNameExtraValues).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"oid": 1}}, - {Keys: bson.M{"m": 1}}, - {Keys: bson.M{"t": 1}}, - {Keys: bson.M{"m": 1, "t": 1}}, - {Keys: bson.M{"oid": 1, "m": 1, "t": 1}}, - }) - - // roles - mongo.GetMongoCol(interfaces.ModelColNameRole).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // user role relations - mongo.GetMongoCol(interfaces.ModelColNameUserRole).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, - {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // permissions - mongo.GetMongoCol(interfaces.ModelColNamePermission).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // role permission relations - mongo.GetMongoCol(interfaces.ModelColNameRolePermission).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, - {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // cache - mongo.GetMongoCol(constants.CacheColName).MustCreateIndexes([]mongo2.IndexModel{ - { - Keys: bson.M{constants.CacheColTime: 1}, - Options: options.Index().SetExpireAfterSeconds(3600 * 24), - }, - }) -} diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index 9db29c88c..2805ad3ed 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -177,4 +177,29 @@ func CreateIndexesV2() { }, }, }) + + // notification requests + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NotificationRequestV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"created_ts", -1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 7), + }, + { + Keys: bson.D{ + {"channel_id", 1}, + }, + }, + { + Keys: bson.D{ + {"setting_id", 1}, + }, + }, + { + Keys: bson.D{ + {"status", 1}, + }, + }, + }) } diff --git a/core/models/delegate/model_user_role_test.go b/core/models/delegate/model_user_role_test.go deleted file mode 100644 index d55b90c6d..000000000 --- a/core/models/delegate/model_user_role_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package delegate_test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson/primitive" - "testing" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") - common.CreateIndexes() -} - -func TestUserRole_Add(t *testing.T) { - SetupTest(t) - - p := &models2.UserRole{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - require.NotNil(t, p.Id) - - a, err := delegate.NewModelDelegate(p).GetArtifact() - require.Nil(t, err) - require.Equal(t, p.Id, a.GetId()) - require.NotNil(t, a.GetSys().GetCreateTs()) - require.NotNil(t, a.GetSys().GetUpdateTs()) -} - -func TestUserRole_Save(t *testing.T) { - SetupTest(t) - - p := &models2.UserRole{ - UserId: primitive.NewObjectID(), - RoleId: primitive.NewObjectID(), - } - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - uid := primitive.NewObjectID() - rid := primitive.NewObjectID() - p.UserId = uid - p.RoleId = rid - err = delegate.NewModelDelegate(p).Save() - require.Nil(t, err) - - err = mongo.GetMongoCol(interfaces.ModelColNameUserRole).FindId(p.Id).One(&p) - require.Nil(t, err) - require.Equal(t, uid, p.UserId) - require.Equal(t, rid, p.RoleId) -} - -func TestUserRole_Delete(t *testing.T) { - SetupTest(t) - - p := &models2.UserRole{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - err = delegate.NewModelDelegate(p).Delete() - require.Nil(t, err) - - var a models2.Artifact - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - err = col.FindId(p.Id).One(&a) - require.Nil(t, err) - require.NotNil(t, a.Obj) - require.True(t, a.Del) -} - -func TestUserRole_AddDuplicates(t *testing.T) { - SetupTest(t) - - uid := primitive.NewObjectID() - rid := primitive.NewObjectID() - p := &models2.UserRole{ - UserId: uid, - RoleId: rid, - } - p2 := &models2.UserRole{ - UserId: uid, - RoleId: rid, - } - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - err = delegate.NewModelDelegate(p2).Add() - require.NotNil(t, err) -} diff --git a/core/models/models/v2/notification_request_v2.go b/core/models/models/v2/notification_request_v2.go index 1130cc928..cc2836d76 100644 --- a/core/models/models/v2/notification_request_v2.go +++ b/core/models/models/v2/notification_request_v2.go @@ -5,8 +5,12 @@ import "go.mongodb.org/mongo-driver/bson/primitive" type NotificationRequestV2 struct { any `collection:"notification_requests"` BaseModelV2[NotificationRequestV2] `bson:",inline"` - Status string `json:"status" bson:"status"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` - ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + Title string `json:"title" bson:"title"` + Content string `json:"content" bson:"content"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` + Setting *NotificationSettingV2 `json:"setting,omitempty" bson:"-"` + Channel *NotificationChannelV2 `json:"channel,omitempty" bson:"-"` } diff --git a/core/notification/constants.go b/core/notification/constants.go index 6c483d4be..d8de0738d 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -24,3 +24,8 @@ const ( ChannelIMProviderDingtalk = "dingtalk" // https://open.dingtalk.com/document/orgapp/custom-robot-access ChannelIMProviderLark = "lark" // https://www.larksuite.com/hc/en-US/articles/099698615114-use-webhook-triggers ) + +const ( + StatusSuccess = "success" + StatusError = "error" +) diff --git a/core/notification/im.go b/core/notification/im.go index 3af47cd41..52a1c4207 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -17,16 +17,15 @@ type ResBody struct { } func SendIMNotification(ch *models.NotificationChannelV2, title, content string) error { - // TODO: compatibility with different IM providers switch ch.Provider { case ChannelIMProviderLark: return sendIMLark(ch, title, content) - case ChannelIMProviderSlack: - return sendIMSlack(ch, title, content) case ChannelIMProviderDingtalk: return sendIMDingTalk(ch, title, content) case ChannelIMProviderWechatWork: return sendIMWechatWork(ch, title, content) + case ChannelIMProviderSlack: + return sendIMSlack(ch, title, content) case ChannelIMProviderTelegram: return sendIMTelegram(ch, title, content) case ChannelIMProviderDiscord: @@ -221,11 +220,12 @@ func sendIMLark(ch *models.NotificationChannelV2, title, content string) error { return nil } -func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error { +func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content string) error { data := req.Param{ - "blocks": []req.Param{ - {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, - {"type": "section", "text": req.Param{"type": "mrkdwn", "text": convertMarkdownToSlack(content)}}, + "msgtype": "markdown", + "markdown": req.Param{ + "title": title, + "text": fmt.Sprintf("# %s\n\n%s", title, content), }, } resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) @@ -238,12 +238,11 @@ func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error return nil } -func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content string) error { data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ - "title": title, - "text": fmt.Sprintf("# %s\n\n%s", title, content), + "content": fmt.Sprintf("# %s\n\n%s", title, content), }, } resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) @@ -256,20 +255,17 @@ func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content stri return nil } -func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content string) error { +func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error { data := req.Param{ - "msgtype": "markdown", - "markdown": req.Param{ - "content": fmt.Sprintf("# %s\n\n%s", title, content), + "blocks": []req.Param{ + {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, + {"type": "section", "text": req.Param{"type": "mrkdwn", "text": convertMarkdownToSlack(content)}}, }, } - resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) + _, err := performIMRequest(ch.WebhookUrl, data) if err != nil { return err } - if resBody.ErrCode != 0 { - return errors.New(resBody.ErrMsg) - } return nil } diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 407e61e62..82d8076a9 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -38,7 +38,7 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { case TypeMail: svc.SendMail(s, ch, title, content) case TypeIM: - svc.SendIM(ch, title, content) + svc.SendIM(s, ch, title, content) } }(chId) } @@ -56,13 +56,15 @@ func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.Notif if err != nil { log.Errorf("[NotificationServiceV2] send mail error: %v", err) } + go svc.saveRequest(s, ch, title, content, err) } -func (svc *ServiceV2) SendIM(ch *models.NotificationChannelV2, title, content string) { +func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { err := SendIMNotification(ch, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } + go svc.saveRequest(s, ch, title, content, err) } func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, args ...any) (content string) { @@ -361,6 +363,29 @@ func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { } } +func (svc *ServiceV2) saveRequest(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string, err error) { + status := StatusSuccess + errMsg := "" + if err != nil { + status = StatusError + errMsg = err.Error() + } + r := models.NotificationRequestV2{ + Status: status, + Error: errMsg, + SettingId: s.Id, + ChannelId: ch.Id, + Title: title, + Content: content, + } + r.SetCreatedAt(time.Now()) + r.SetUpdatedAt(time.Now()) + _, err = service.NewModelServiceV2[models.NotificationRequestV2]().InsertOne(r) + if err != nil { + log.Errorf("[NotificationServiceV2] save request error: %v", err) + } +} + func newNotificationServiceV2() *ServiceV2 { return &ServiceV2{} } From 9650f78d03cd160107e4c2e949f1ba07dc6d61e3 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 26 Jul 2024 12:07:20 +0800 Subject: [PATCH 077/106] refactor: Update IM notification function to include larger title size in MSTeams notifications --- core/notification/im.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/notification/im.go b/core/notification/im.go index 52a1c4207..181b8b89a 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -335,6 +335,7 @@ func sendIMMSTeams(ch *models.NotificationChannelV2, title string, content strin { "type": "TextBlock", "text": fmt.Sprintf("**%s**", title), + "size": "Large", }, { "type": "TextBlock", From 0e2170f6441bc7669b0c1869bc006dc21ceed76f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 26 Jul 2024 15:33:58 +0800 Subject: [PATCH 078/106] refactor: Update models to include sender email, name, and mail recipients for notification requests --- .../models/v2/notification_request_v2.go | 5 ++ .../models/v2/notification_setting_v2.go | 11 ++-- core/notification/constants.go | 1 + core/notification/service_v2.go | 61 ++++++++++++++----- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/core/models/models/v2/notification_request_v2.go b/core/models/models/v2/notification_request_v2.go index cc2836d76..be53a8d56 100644 --- a/core/models/models/v2/notification_request_v2.go +++ b/core/models/models/v2/notification_request_v2.go @@ -9,6 +9,11 @@ type NotificationRequestV2 struct { Error string `json:"error,omitempty" bson:"error,omitempty"` Title string `json:"title" bson:"title"` Content string `json:"content" bson:"content"` + SenderEmail string `json:"sender_email" bson:"sender_email"` + SenderName string `json:"sender_name" bson:"sender_name"` + MailTo string `json:"mail_to" bson:"mail_to"` + MailCc string `json:"mail_cc" bson:"mail_cc"` + MailBcc string `json:"mail_bcc" bson:"mail_bcc"` SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` Setting *NotificationSettingV2 `json:"setting,omitempty" bson:"-"` diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index 6f36c3ec3..517ae086e 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -21,12 +21,11 @@ type NotificationSettingV2 struct { TriggerTarget string `json:"trigger_target" bson:"trigger_target"` Trigger string `json:"trigger" bson:"trigger"` - HasMail bool `json:"has_mail" bson:"has_mail"` - SenderEmail string `json:"sender_email" bson:"sender_email"` - SenderName string `json:"sender_name" bson:"sender_name"` - MailTo string `json:"mail_to" bson:"mail_to"` - MailCc string `json:"mail_cc" bson:"mail_cc"` - MailBcc string `json:"mail_bcc" bson:"mail_bcc"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo string `json:"mail_to" bson:"mail_to,omitempty"` + MailCc string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` diff --git a/core/notification/constants.go b/core/notification/constants.go index d8de0738d..817f476f0 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -26,6 +26,7 @@ const ( ) const ( + StatusSending = "sending" StatusSuccess = "success" StatusError = "error" ) diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 82d8076a9..9e19978c0 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -51,20 +51,31 @@ func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.Notif mailCc := s.MailCc mailBcc := s.MailBcc + // request + r, _ := svc.createRequest(s, ch, title, content) + // send mail err := SendMail(s, ch, mailTo, mailCc, mailBcc, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mail error: %v", err) } - go svc.saveRequest(s, ch, title, content, err) + + // save request + go svc.saveRequest(r, err) } func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { + // request + r, _ := svc.createRequest(s, ch, title, content) + + // send mobile notification err := SendIMNotification(ch, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } - go svc.saveRequest(s, ch, title, content, err) + + // save request + go svc.saveRequest(r, err) } func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, args ...any) (content string) { @@ -363,24 +374,42 @@ func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { } } -func (svc *ServiceV2) saveRequest(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string, err error) { - status := StatusSuccess - errMsg := "" - if err != nil { - status = StatusError - errMsg = err.Error() - } +func (svc *ServiceV2) createRequest(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) (res *models.NotificationRequestV2, err error) { r := models.NotificationRequestV2{ - Status: status, - Error: errMsg, - SettingId: s.Id, - ChannelId: ch.Id, - Title: title, - Content: content, + Status: StatusSending, + SettingId: s.Id, + ChannelId: ch.Id, + Title: title, + Content: content, + SenderEmail: s.SenderEmail, + SenderName: s.SenderName, + MailTo: s.MailTo, + MailCc: s.MailCc, + MailBcc: s.MailBcc, } r.SetCreatedAt(time.Now()) r.SetUpdatedAt(time.Now()) - _, err = service.NewModelServiceV2[models.NotificationRequestV2]().InsertOne(r) + r.Id, err = service.NewModelServiceV2[models.NotificationRequestV2]().InsertOne(r) + if err != nil { + log.Errorf("[NotificationServiceV2] save request error: %v", err) + return nil, err + } + return &r, nil +} + +func (svc *ServiceV2) saveRequest(r *models.NotificationRequestV2, err error) { + if r == nil { + return + } + + if err != nil { + r.Status = StatusError + r.Error = err.Error() + } else { + r.Status = StatusSuccess + } + r.SetUpdatedAt(time.Now()) + err = service.NewModelServiceV2[models.NotificationRequestV2]().ReplaceById(r.Id, *r) if err != nil { log.Errorf("[NotificationServiceV2] save request error: %v", err) } From 8d5a0b0c0e716c7ff47fbda4e2708ecc3c52cc5e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Fri, 26 Jul 2024 18:15:15 +0800 Subject: [PATCH 079/106] refactor: Add Outlook authentication support for sending emails --- .../models/v2/notification_channel_v2.go | 3 + core/notification/mail.go | 21 ++++++- core/notification/mail_outlook.go | 55 +++++++++++++++++++ core/notification/oauth2_auth.go | 21 +++++++ 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 core/notification/mail_outlook.go create mode 100644 core/notification/oauth2_auth.go diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index 955076187..b93de702a 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -14,4 +14,7 @@ type NotificationChannelV2 struct { WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` + OutlookTenantId string `json:"outlook_tenant_id,omitempty" bson:"outlook_tenant_id,omitempty"` + OutlookClientId string `json:"outlook_client_id,omitempty" bson:"outlook_client_id,omitempty"` + OutlookClientSecret string `json:"outlook_client_secret,omitempty" bson:"outlook_client_secret,omitempty"` } diff --git a/core/notification/mail.go b/core/notification/mail.go index 549cc0e6b..88c4a716e 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -13,6 +13,20 @@ import ( ) func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc, title, content string) error { + // compatibility for different providers + var auth *XOAuth2Auth + if ch.Provider == ChannelMailProviderOutlook { + token, err := getOutlookToken(ch.OutlookTenantId, ch.OutlookClientId, ch.OutlookClientSecret) + if err != nil { + log.Errorf("failed to get outlook token: %v", err) + return err + } + auth = &XOAuth2Auth{ + Username: ch.SMTPUsername, + Token: token, + } + } + // config smtpConfig := smtpAuthentication{ Server: ch.SMTPServer, @@ -42,7 +56,7 @@ func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, } // send the email - if err := send(smtpConfig, options, content, text); err != nil { + if err := sendMail(smtpConfig, options, content, text, auth); err != nil { log.Errorf("failed to send email: %v", err) trace.PrintError(err) return err @@ -84,7 +98,7 @@ type sendOptions struct { } // send email -func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error { +func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string, auth *XOAuth2Auth) error { if smtpConfig.Server == "" { return errors.New("SMTP server config is empty") } @@ -132,6 +146,9 @@ func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, t m.AddAlternative("text/html", htmlBody) d := gomail.NewDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword) + if auth != nil { + d.Auth = auth + } return d.DialAndSend(m) } diff --git a/core/notification/mail_outlook.go b/core/notification/mail_outlook.go new file mode 100644 index 000000000..7e5b33da5 --- /dev/null +++ b/core/notification/mail_outlook.go @@ -0,0 +1,55 @@ +package notification + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" +) + +func getOutlookToken(tenantID, clientID, clientSecret string) (string, error) { + url := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/v2.0/token", tenantID) + data := map[string]string{ + "grant_type": "client_credentials", + "client_id": clientID, + "client_secret": clientSecret, + "scope": "https://outlook.office365.com/.default", + } + + formData := "" + for key, value := range data { + if formData != "" { + formData += "&" + } + formData += fmt.Sprintf("%s=%s", key, value) + } + + req, err := http.NewRequest("POST", url, bytes.NewBufferString(formData)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return "", err + } + + if token, ok := result["access_token"].(string); ok { + return token, nil + } + return "", fmt.Errorf("no access token found") +} diff --git a/core/notification/oauth2_auth.go b/core/notification/oauth2_auth.go new file mode 100644 index 000000000..4b80784ad --- /dev/null +++ b/core/notification/oauth2_auth.go @@ -0,0 +1,21 @@ +package notification + +import ( + "fmt" + "net/smtp" +) + +type XOAuth2Auth struct { + Username, Token string +} + +func (a *XOAuth2Auth) Start(_ *smtp.ServerInfo) (string, []byte, error) { + return "XOAUTH2", []byte("user=" + a.Username + "\x01auth=Bearer " + a.Token + "\x01\x01"), nil +} + +func (a *XOAuth2Auth) Next(fromServer []byte, more bool) ([]byte, error) { + if more { + return nil, fmt.Errorf("unexpected server challenge: %s", fromServer) + } + return nil, nil +} From e8bdfd228db15c969e1d47d46a772b5b51a21d4b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 27 Jul 2024 16:06:26 +0800 Subject: [PATCH 080/106] refactor: Update models to include sender email, name, and mail recipients for notification requests --- .../models/v2/notification_request_v2.go | 10 ++--- .../models/v2/notification_setting_v2.go | 10 ++--- core/notification/mail.go | 44 +++++-------------- 3 files changed, 20 insertions(+), 44 deletions(-) diff --git a/core/models/models/v2/notification_request_v2.go b/core/models/models/v2/notification_request_v2.go index be53a8d56..bbfe60239 100644 --- a/core/models/models/v2/notification_request_v2.go +++ b/core/models/models/v2/notification_request_v2.go @@ -9,11 +9,11 @@ type NotificationRequestV2 struct { Error string `json:"error,omitempty" bson:"error,omitempty"` Title string `json:"title" bson:"title"` Content string `json:"content" bson:"content"` - SenderEmail string `json:"sender_email" bson:"sender_email"` - SenderName string `json:"sender_name" bson:"sender_name"` - MailTo string `json:"mail_to" bson:"mail_to"` - MailCc string `json:"mail_cc" bson:"mail_cc"` - MailBcc string `json:"mail_bcc" bson:"mail_bcc"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` Setting *NotificationSettingV2 `json:"setting,omitempty" bson:"-"` diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index 517ae086e..eb8f4be88 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -21,11 +21,11 @@ type NotificationSettingV2 struct { TriggerTarget string `json:"trigger_target" bson:"trigger_target"` Trigger string `json:"trigger" bson:"trigger"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` - MailTo string `json:"mail_to" bson:"mail_to,omitempty"` - MailCc string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` - MailBcc string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` diff --git a/core/notification/mail.go b/core/notification/mail.go index 88c4a716e..c24de04f0 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -12,7 +12,7 @@ import ( "strings" ) -func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc, title, content string) error { +func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc []string, title, content string) error { // compatibility for different providers var auth *XOAuth2Auth if ch.Provider == ChannelMailProviderOutlook { @@ -92,9 +92,9 @@ type smtpAuthentication struct { // sendOptions are options for sending an email type sendOptions struct { Subject string - To string - Cc string - Bcc string + To []string + Cc []string + Bcc []string } // send email @@ -111,7 +111,7 @@ func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody strin return errors.New("SMTP user is empty") } - if options.To == "" { + if len(options.To) == 0 { return errors.New("no receiver emails configured") } @@ -120,26 +120,15 @@ func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody strin Address: smtpConfig.SenderEmail, } - var toList []string - if strings.Contains(options.To, ";") { - toList = strings.Split(options.To, ";") - // trim space - for i, to := range toList { - toList[i] = strings.TrimSpace(to) - } - } else { - toList = []string{options.To} - } - m := gomail.NewMessage() m.SetHeader("From", from.String()) - m.SetHeader("To", getRecipientList(options.To)...) + m.SetHeader("To", options.To...) m.SetHeader("Subject", options.Subject) - if options.Cc != "" { - m.SetHeader("Cc", getRecipientList(options.Cc)...) + if len(options.Cc) > 0 { + m.SetHeader("Cc", options.Cc...) } - if options.Bcc != "" { - m.SetHeader("Bcc", getRecipientList(options.Bcc)...) + if len(options.Bcc) > 0 { + m.SetHeader("Bcc", options.Bcc...) } m.SetBody("text/plain", txtBody) @@ -152,16 +141,3 @@ func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody strin return d.DialAndSend(m) } - -func getRecipientList(value string) (values []string) { - if strings.Contains(value, ";") { - values = strings.Split(value, ";") - // trim space - for i, v := range values { - values[i] = strings.TrimSpace(v) - } - } else { - values = []string{value} - } - return values -} From ea5ec117cc0bf1f265222733f931928746fe4fbf Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 27 Jul 2024 19:29:43 +0800 Subject: [PATCH 081/106] refactor: Remove unused code for SMTP authentication and add Gmail OAuth2 authentication support --- core/go.mod | 43 ++- core/go.sum | 202 ++++-------- core/grpc/test/main_test.go | 7 - .../test/model_base_service_server_test.go | 312 ------------------ core/grpc/test/model_delegate_server_test.go | 147 --------- core/grpc/test/node_server_test.go | 140 -------- .../models/v2/notification_channel_v2.go | 4 +- .../models/v2/notification_setting_v2.go | 11 +- core/notification/mail.go | 78 +++-- core/notification/mail_gmail.go | 61 ++++ core/notification/mail_outlook.go | 55 --- core/notification/oauth2_auth.go | 21 -- core/notification/oauth2_gmail.go | 47 +++ core/notification/service_v2.go | 7 +- go.work.sum | 278 +++++++++++++++- 15 files changed, 513 insertions(+), 900 deletions(-) delete mode 100644 core/grpc/test/main_test.go delete mode 100644 core/grpc/test/model_base_service_server_test.go delete mode 100644 core/grpc/test/model_delegate_server_test.go delete mode 100644 core/grpc/test/node_server_test.go create mode 100644 core/notification/mail_gmail.go delete mode 100644 core/notification/mail_outlook.go delete mode 100644 core/notification/oauth2_auth.go create mode 100644 core/notification/oauth2_gmail.go diff --git a/core/go.mod b/core/go.mod index 8e7030702..293710c2d 100644 --- a/core/go.mod +++ b/core/go.mod @@ -11,6 +11,7 @@ replace ( ) require ( + github.com/PuerkitoBio/goquery v1.9.2 github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -24,6 +25,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 @@ -42,7 +44,9 @@ require ( github.com/upper/db/v4 v4.6.0 go.mongodb.org/mongo-driver v1.15.1 go.uber.org/dig v1.10.0 - google.golang.org/grpc v1.64.0 + golang.org/x/oauth2 v0.21.0 + google.golang.org/api v0.189.0 + google.golang.org/grpc v1.65.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df ) @@ -52,9 +56,11 @@ require ( ) require ( + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect - github.com/PuerkitoBio/goquery v1.9.2 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -65,11 +71,13 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -79,8 +87,11 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -107,7 +118,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/onsi/gomega v1.30.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -135,23 +145,26 @@ require ( github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/goleak v1.1.11 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.20.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/core/go.sum b/core/go.sum index 5fd1eb825..92c0660e8 100644 --- a/core/go.sum +++ b/core/go.sum @@ -27,16 +27,19 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -55,41 +58,27 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= -github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= -github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= -github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= -github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= -github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= -github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= @@ -155,7 +144,6 @@ github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -185,10 +173,8 @@ github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -197,8 +183,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= -github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -217,7 +201,6 @@ github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXY github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -225,8 +208,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -242,8 +225,6 @@ github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaC github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -310,8 +291,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -332,19 +311,26 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= @@ -383,20 +369,10 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= -github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= -github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/imroc/req v0.3.2 h1:M/JkeU6RPmX+WYvT2vaaOL0K+q8ufL5LxwvJc4xeB4o= github.com/imroc/req v0.3.2/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -413,8 +389,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= @@ -431,32 +407,30 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= -github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -475,7 +449,6 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= @@ -508,8 +481,6 @@ github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc8 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= -github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -517,8 +488,6 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -527,11 +496,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -543,8 +509,6 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= -github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -561,11 +525,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= @@ -579,13 +539,11 @@ github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -605,8 +563,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= -github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -617,8 +573,6 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= @@ -626,8 +580,6 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= -github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= @@ -642,7 +594,6 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -677,8 +628,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -686,7 +635,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -702,7 +650,6 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -723,15 +670,6 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= -github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= -github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= -github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe h1:9YnI5plmy+ad6BM+JCLJb2ZV7/TNiE5l7SNKfumYKgc= -github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe/go.mod h1:JTFJA/t820uFDoyPpErFQ3rb3amdZoPtxcKervG0OE4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -744,23 +682,8 @@ github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -784,14 +707,26 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -821,7 +756,6 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -837,13 +771,12 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -888,7 +821,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -933,15 +865,14 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -984,7 +915,6 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1049,19 +979,16 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1069,8 +996,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1085,8 +1012,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1143,7 +1070,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1154,8 +1080,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1194,6 +1120,8 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1264,8 +1192,14 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f h1:RARaIm8pxYuxyNPbBQf5igT7XdOyCNtat1qAT2ZxjU4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1293,8 +1227,10 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1320,7 +1256,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= @@ -1330,9 +1265,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1373,8 +1305,6 @@ modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= -moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= -moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/core/grpc/test/main_test.go b/core/grpc/test/main_test.go deleted file mode 100644 index 232a8c6e8..000000000 --- a/core/grpc/test/main_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package test - -import "testing" - -func TestMain(m *testing.M) { - m.Run() -} diff --git a/core/grpc/test/model_base_service_server_test.go b/core/grpc/test/model_base_service_server_test.go deleted file mode 100644 index ad9b53724..000000000 --- a/core/grpc/test/model_base_service_server_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package test - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/node/test" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestModelBaseService_GetById(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // get by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().GetById(ctx, req) - require.Nil(t, err) - var p2 models.Project - err = json.Unmarshal(res.Data, &p2) - require.Nil(t, err) - - // validate - require.Equal(t, p.Id, p2.Id) - require.Equal(t, p.Name, p2.Name) - require.Equal(t, p.Description, p2.Description) -} - -func TestModelBaseService_Get(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // get - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().Get(ctx, req) - require.Nil(t, err) - var p2 models.Project - err = json.Unmarshal(res.Data, &p2) - require.Nil(t, err) - - // validate - require.Equal(t, p.Id, p2.Id) - require.Equal(t, p.Name, p2.Name) - require.Equal(t, p.Description, p2.Description) -} - -func TestModelBaseService_GetList(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // get list - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().GetList(ctx, req) - require.Nil(t, err) - var list []models.Project - err = json.Unmarshal(res.Data, &list) - require.Nil(t, err) - - // validate - require.Equal(t, n, len(list)) -} - -func TestModelBaseService_DeleteById(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().DeleteById(ctx, req) - require.Nil(t, err) - - // validate - p, err = test.T.ModelSvc.GetProjectById(p.Id) - require.Equal(t, mongo2.ErrNoDocuments, err) -} - -func TestModelBaseService_Delete(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().Delete(ctx, req) - require.Nil(t, err) - - // validate - p, err = test.T.ModelSvc.GetProjectById(p.Id) - require.Equal(t, mongo2.ErrNoDocuments, err) -} - -func TestModelBaseService_DeleteList(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().DeleteList(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, 0, len(list)) -} - -func TestModelBaseService_ForceDeleteList(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().ForceDeleteList(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, 0, len(list)) -} - -func TestModelBaseService_UpdateById(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // update by id - ctx, cancel := T.Client.Context() - defer cancel() - update := bson.M{ - "name": "test-new-project", - } - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id, Update: update}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().UpdateById(ctx, req) - require.Nil(t, err) - - // validate - p2, err := test.T.ModelSvc.GetProjectById(p.Id) - require.Nil(t, err) - require.Equal(t, "test-new-project", p2.Name) -} - -func TestModelBaseService_Update(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // update - ctx, cancel := T.Client.Context() - defer cancel() - update := bson.M{ - "name": "test-new-project", - } - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}, Update: update}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().Update(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, 0, len(list)) - list, err = test.T.ModelSvc.GetProjectList(bson.M{"name": "test-new-project"}, nil) - require.Nil(t, err) - require.Equal(t, n, len(list)) -} - -func TestModelBaseService_Insert(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // insert - var docs []interface{} - n := 10 - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Id: primitive.NewObjectID(), - Name: "test-project", - }) - } - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Docs: docs}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().Insert(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, n, len(list)) -} - -func TestModelBaseService_Count(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // count - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().Count(ctx, req) - require.Nil(t, err) - - // validate - var total int - err = json.Unmarshal(res.Data, &total) - require.Nil(t, err) - require.Equal(t, n, total) -} diff --git a/core/grpc/test/model_delegate_server_test.go b/core/grpc/test/model_delegate_server_test.go deleted file mode 100644 index c761b9b68..000000000 --- a/core/grpc/test/model_delegate_server_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/node/test" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestModelDelegate_Do(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // get artifact - a := modelDelegateGetArtifact(t) - require.Equal(t, project.GetId(), a.GetId()) - - // save - modelDelegateSave(t) - project, err = test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) - require.Nil(t, err) - require.Equal(t, "test-new-project", project.Name) - require.Equal(t, "test-new-description", project.Description) - - // delete - modelDelegateDelete(t) - _, err = test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) - require.Equal(t, mongo2.ErrNoDocuments, err) -} - -func TestModelDelegate_Do_All(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAddAll(t) - modelDelegateValidateAddAll(t) -} - -func modelDelegateAdd(t *testing.T) { - // modelDelegateAdd - project := &models.Project{ - Name: "test-project", - Description: "test-description", - } - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - err := projectD.Add() - require.Nil(t, err) -} - -func modelDelegateGetArtifact(t *testing.T) interfaces.ModelArtifact { - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - a, err := projectD.GetArtifact() - require.Nil(t, err) - return a -} - -func modelDelegateSave(t *testing.T) { - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - project.Name = "test-new-project" - project.Description = "test-new-description" - - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - err = projectD.Save() - require.Nil(t, err) -} - -func modelDelegateDelete(t *testing.T) { - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) - require.Nil(t, err) - - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - err = projectD.Delete() - require.Nil(t, err) -} - -func modelDelegateAddAll(t *testing.T) { - var err error - cfgOpt := client.WithDelegateConfigPath(T.Client.GetConfigPath()) - m := models.NewModelMap() - err = client.NewModelDelegate(&m.Tag, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Node, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Project, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Spider, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Task, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Job, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Schedule, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.User, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Setting, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Token, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Variable, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.User, cfgOpt).Add() - require.Nil(t, err) -} - -func modelDelegateValidateAddAll(t *testing.T) { - var err error - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) -} diff --git a/core/grpc/test/node_server_test.go b/core/grpc/test/node_server_test.go deleted file mode 100644 index de858bed9..000000000 --- a/core/grpc/test/node_server_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package test - -import ( - "context" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/node/test" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func TestGrpcServer_Register(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // register - register(t) - - // validate - workerNodeKey := T.WorkerNodeInfo.Key - workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, workerNodeKey, workerNode.Key) - require.Equal(t, constants.NodeStatusRegistered, workerNode.Status) -} - -func TestGrpcServer_Register_Existing(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add to db - node := &models.Node{ - Key: T.WorkerNodeInfo.Key, - IsMaster: false, - Status: constants.NodeStatusUnregistered, - } - nodeD := delegate.NewModelDelegate(node) - err = nodeD.Add() - require.Nil(t, err) - - // register - register(t) - - // validate - workerNodeKey := T.WorkerNodeInfo.Key - workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, workerNodeKey, workerNode.Key) - require.Equal(t, constants.NodeStatusRegistered, workerNode.Status) -} - -func TestGrpcServer_SendHeartbeat(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // register - register(t) - - // send heartbeat - sendHeartbeat(t) - - // validate - workerNodeKey := T.WorkerNodeInfo.Key - workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, workerNodeKey, workerNode.Key) - require.Equal(t, constants.NodeStatusOnline, workerNode.Status) -} - -func TestGrpcServer_Subscribe(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // register - register(t) - - // handle client message - go handleClientMessage(t) - - time.Sleep(1 * time.Second) - - // server PING client - sub, err := T.Server.GetSubscribe("node:" + T.WorkerNodeInfo.Key) - require.Nil(t, err) - require.NotNil(t, sub) - err = sub.GetStream().Send(&grpc.StreamMessage{ - Code: grpc.StreamMessageCode_PING, - NodeKey: T.MasterNodeInfo.Key, - }) - require.Nil(t, err) - - // wait - time.Sleep(1 * time.Second) - - // validate - workerNode, err := test.T.ModelSvc.GetNodeByKey(T.WorkerNodeInfo.Key, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, workerNode.Status) -} - -func register(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - res, err := T.Client.GetNodeClient().Register(ctx, T.Client.NewRequest(T.WorkerNodeInfo)) - require.Nil(t, err) - require.Equal(t, grpc.ResponseCode_OK, res.Code) -} - -func sendHeartbeat(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - res, err := T.Client.GetNodeClient().SendHeartbeat(ctx, T.Client.NewRequest(T.WorkerNodeInfo)) - require.Nil(t, err) - require.Equal(t, grpc.ResponseCode_OK, res.Code) -} - -func handleClientMessage(t *testing.T) { - msgCh := T.Client.GetMessageChannel() - for { - msg := <-msgCh - switch msg.Code { - case grpc.StreamMessageCode_PING: - require.Equal(t, T.MasterNodeInfo.Key, msg.NodeKey) - res, err := T.Client.GetNodeClient().SendHeartbeat(context.Background(), T.Client.NewRequest(T.WorkerNodeInfo)) - require.Nil(t, err) - require.NotNil(t, res) - } - } -} diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go index b93de702a..2e9ebed3a 100644 --- a/core/models/models/v2/notification_channel_v2.go +++ b/core/models/models/v2/notification_channel_v2.go @@ -14,7 +14,5 @@ type NotificationChannelV2 struct { WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` - OutlookTenantId string `json:"outlook_tenant_id,omitempty" bson:"outlook_tenant_id,omitempty"` - OutlookClientId string `json:"outlook_client_id,omitempty" bson:"outlook_client_id,omitempty"` - OutlookClientSecret string `json:"outlook_client_secret,omitempty" bson:"outlook_client_secret,omitempty"` + GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` } diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index eb8f4be88..b47821fd4 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -21,11 +21,12 @@ type NotificationSettingV2 struct { TriggerTarget string `json:"trigger_target" bson:"trigger_target"` Trigger string `json:"trigger" bson:"trigger"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` - MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` - MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` - MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` diff --git a/core/notification/mail.go b/core/notification/mail.go index c24de04f0..49cd5bd02 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -13,18 +13,10 @@ import ( ) func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc []string, title, content string) error { - // compatibility for different providers - var auth *XOAuth2Auth - if ch.Provider == ChannelMailProviderOutlook { - token, err := getOutlookToken(ch.OutlookTenantId, ch.OutlookClientId, ch.OutlookClientSecret) - if err != nil { - log.Errorf("failed to get outlook token: %v", err) - return err - } - auth = &XOAuth2Auth{ - Username: ch.SMTPUsername, - Token: token, - } + // sender email + senderEmail := ch.SMTPUsername + if s.UseCustomSenderEmail { + senderEmail = s.SenderEmail } // config @@ -32,7 +24,7 @@ func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, Server: ch.SMTPServer, Port: ch.SMTPPort, SenderIdentity: s.SenderName, - SenderEmail: s.SenderEmail, + SenderEmail: senderEmail, SMTPUser: ch.SMTPUsername, SMTPPassword: ch.SMTPPassword, } @@ -55,14 +47,12 @@ func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content = GetTheme() + content } - // send the email - if err := sendMail(smtpConfig, options, content, text, auth); err != nil { - log.Errorf("failed to send email: %v", err) - trace.PrintError(err) - return err + switch ch.Provider { + case ChannelMailProviderGmail: + return sendMailGmail(ch, smtpConfig, options, content, text) + default: + return sendMail(smtpConfig, options, content, text) } - - return nil } func isHtml(content string) bool { @@ -97,30 +87,19 @@ type sendOptions struct { Bcc []string } -// send email -func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string, auth *XOAuth2Auth) error { - if smtpConfig.Server == "" { - return errors.New("SMTP server config is empty") - } - - if smtpConfig.Port == 0 { - return errors.New("SMTP port config is empty") - } - - if smtpConfig.SMTPUser == "" { - return errors.New("SMTP user is empty") - } - +func getMailMessage(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) (m *gomail.Message, err error) { if len(options.To) == 0 { - return errors.New("no receiver emails configured") + return nil, errors.New("no receiver emails configured") } + // from from := mail.Address{ Name: smtpConfig.SenderIdentity, Address: smtpConfig.SenderEmail, } - m := gomail.NewMessage() + // message + m = gomail.NewMessage() m.SetHeader("From", from.String()) m.SetHeader("To", options.To...) m.SetHeader("Subject", options.Subject) @@ -130,14 +109,33 @@ func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody strin if len(options.Bcc) > 0 { m.SetHeader("Bcc", options.Bcc...) } - m.SetBody("text/plain", txtBody) m.AddAlternative("text/html", htmlBody) - d := gomail.NewDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword) - if auth != nil { - d.Auth = auth + return m, nil +} + +// send email +func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error { + if smtpConfig.Server == "" { + return errors.New("SMTP server config is empty") + } + + if smtpConfig.Port == 0 { + return errors.New("SMTP port config is empty") + } + + if smtpConfig.SMTPUser == "" { + return errors.New("SMTP user is empty") } + m, err := getMailMessage(smtpConfig, options, htmlBody, txtBody) + if err != nil { + return err + } + + // dialer + d := gomail.NewDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword) + return d.DialAndSend(m) } diff --git a/core/notification/mail_gmail.go b/core/notification/mail_gmail.go new file mode 100644 index 000000000..aebf4e5ff --- /dev/null +++ b/core/notification/mail_gmail.go @@ -0,0 +1,61 @@ +package notification + +import ( + "context" + "encoding/base64" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/trace" + "golang.org/x/oauth2/google" + "google.golang.org/api/gmail/v1" + "strings" +) + +func sendMailGmail(ch *models.NotificationChannelV2, smtpConfig smtpAuthentication, options sendOptions, htmlBody, txtBody string) error { + // 读取服务账户 JSON 密钥 + b := []byte(ch.GoogleOAuth2Json) + + // 使用服务账户 JSON 密钥文件创建 JWT 配置 + config, err := google.JWTConfigFromJSON(b, gmail.GmailSendScope) + if err != nil { + log.Errorf("Unable to parse service account key file to config: %v", err) + return trace.TraceError(err) + } + + // 使用服务账户的电子邮件地址来模拟用户 + config.Subject = ch.SMTPUsername + + // 创建 Gmail 服务 + client := config.Client(context.Background()) + srv, err := gmail.New(client) + if err != nil { + log.Errorf("Unable to create Gmail client: %v", err) + return trace.TraceError(err) + } + + // 创建 MIME 邮件 + m, err := getMailMessage(smtpConfig, options, htmlBody, txtBody) + if err != nil { + return err + } + + var buf strings.Builder + if _, err := m.WriteTo(&buf); err != nil { + log.Errorf("Unable to write message: %v", err) + return trace.TraceError(err) + } + + // 将邮件内容进行 base64 编码 + gmsg := &gmail.Message{ + Raw: base64.URLEncoding.EncodeToString([]byte(buf.String())), + } + + // 发送邮件 + _, err = srv.Users.Messages.Send("me", gmsg).Do() + if err != nil { + log.Errorf("Unable to send email: %v", err) + return trace.TraceError(err) + } + + return nil +} diff --git a/core/notification/mail_outlook.go b/core/notification/mail_outlook.go deleted file mode 100644 index 7e5b33da5..000000000 --- a/core/notification/mail_outlook.go +++ /dev/null @@ -1,55 +0,0 @@ -package notification - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" -) - -func getOutlookToken(tenantID, clientID, clientSecret string) (string, error) { - url := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/v2.0/token", tenantID) - data := map[string]string{ - "grant_type": "client_credentials", - "client_id": clientID, - "client_secret": clientSecret, - "scope": "https://outlook.office365.com/.default", - } - - formData := "" - for key, value := range data { - if formData != "" { - formData += "&" - } - formData += fmt.Sprintf("%s=%s", key, value) - } - - req, err := http.NewRequest("POST", url, bytes.NewBufferString(formData)) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - var result map[string]interface{} - if err := json.Unmarshal(body, &result); err != nil { - return "", err - } - - if token, ok := result["access_token"].(string); ok { - return token, nil - } - return "", fmt.Errorf("no access token found") -} diff --git a/core/notification/oauth2_auth.go b/core/notification/oauth2_auth.go deleted file mode 100644 index 4b80784ad..000000000 --- a/core/notification/oauth2_auth.go +++ /dev/null @@ -1,21 +0,0 @@ -package notification - -import ( - "fmt" - "net/smtp" -) - -type XOAuth2Auth struct { - Username, Token string -} - -func (a *XOAuth2Auth) Start(_ *smtp.ServerInfo) (string, []byte, error) { - return "XOAUTH2", []byte("user=" + a.Username + "\x01auth=Bearer " + a.Token + "\x01\x01"), nil -} - -func (a *XOAuth2Auth) Next(fromServer []byte, more bool) ([]byte, error) { - if more { - return nil, fmt.Errorf("unexpected server challenge: %s", fromServer) - } - return nil, nil -} diff --git a/core/notification/oauth2_gmail.go b/core/notification/oauth2_gmail.go new file mode 100644 index 000000000..b6c1bbede --- /dev/null +++ b/core/notification/oauth2_gmail.go @@ -0,0 +1,47 @@ +package notification + +import ( + "context" + "github.com/apex/log" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "net/smtp" + "time" +) + +// 获取服务账户的OAuth2配置 +func getGmailOAuth2Token(oauth2Json string) (token *oauth2.Token, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // 读取服务账户 JSON 密钥 + b := []byte(oauth2Json) + + // 使用服务账户 JSON 密钥文件创建 JWT 配置 + config, err := google.JWTConfigFromJSON(b, "https://mail.google.com/") + if err != nil { + log.Errorf("Unable to parse service account key file to config: %v", err) + return nil, err + } + + // 使用服务账户的电子邮件和访问令牌 + token, err = config.TokenSource(ctx).Token() + if err != nil { + log.Errorf("Unable to generate token: %v", err) + return nil, err + } + return token, nil +} + +// GmailOAuth2Auth 自定义OAuth2认证 +type GmailOAuth2Auth struct { + username, accessToken string +} + +func (a *GmailOAuth2Auth) Start(_ *smtp.ServerInfo) (string, []byte, error) { + return "XOAUTH2", []byte("user=" + a.username + "\x01auth=Bearer " + a.accessToken + "\x01\x01"), nil +} + +func (a *GmailOAuth2Auth) Next(_ []byte, _ bool) ([]byte, error) { + return nil, nil +} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 9e19978c0..69360ecd1 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -46,7 +46,6 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { } func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { - // TODO: parse to/cc/bcc mailTo := s.MailTo mailCc := s.MailCc mailBcc := s.MailBcc @@ -375,13 +374,17 @@ func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { } func (svc *ServiceV2) createRequest(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) (res *models.NotificationRequestV2, err error) { + senderEmail := ch.SMTPUsername + if s.UseCustomSenderEmail { + senderEmail = s.SenderEmail + } r := models.NotificationRequestV2{ Status: StatusSending, SettingId: s.Id, ChannelId: ch.Id, Title: title, Content: content, - SenderEmail: s.SenderEmail, + SenderEmail: senderEmail, SenderName: s.SenderName, MailTo: s.MailTo, MailCc: s.MailCc, diff --git a/go.work.sum b/go.work.sum index 0c5e3202d..967f3c1b1 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= +cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= @@ -9,76 +11,111 @@ cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbD cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= cloud.google.com/go/accessapproval v1.7.5 h1:uzmAMSgYcnlHa9X9YSQZ4Q1wlfl4NNkZyQgho1Z6p04= cloud.google.com/go/accessapproval v1.7.5/go.mod h1:g88i1ok5dvQ9XJsxpUInWWvUBrIZhyPDPbk4T01OoJ0= cloud.google.com/go/accessapproval v1.7.7 h1:vO95gvBi7qUgfA9SflexQs9hB4U4tnri/GwADIrLQy8= cloud.google.com/go/accessapproval v1.7.7/go.mod h1:10ZDPYiTm8tgxuMPid8s2DL93BfCt6xBh/Vg0Xd8pU0= +cloud.google.com/go/accessapproval v1.7.9 h1:mp1X2FsNRdTYTVw4b6eF4OQ+7l6EpLnZlcatXiFWJTg= +cloud.google.com/go/accessapproval v1.7.9/go.mod h1:teNI+P/xzZ3dppGXEYFvSmuOvmTjLE9toPq21WHssYc= cloud.google.com/go/accesscontextmanager v1.8.5 h1:2GLNaNu9KRJhJBFTIVRoPwk6xE5mUDgD47abBq4Zp/I= cloud.google.com/go/accesscontextmanager v1.8.5/go.mod h1:TInEhcZ7V9jptGNqN3EzZ5XMhT6ijWxTGjzyETwmL0Q= cloud.google.com/go/accesscontextmanager v1.8.7 h1:GgdNoDwZR5RIO3j8XwXqa6Gc6q5mP3KYMdFC7FEVyG4= cloud.google.com/go/accesscontextmanager v1.8.7/go.mod h1:jSvChL1NBQ+uLY9zUBdPy9VIlozPoHptdBnRYeWuQoM= +cloud.google.com/go/accesscontextmanager v1.8.9 h1:oVjc3eFQP92zezKsof5ly6ENhuNSsgadRdFKhUn7L9g= +cloud.google.com/go/accesscontextmanager v1.8.9/go.mod h1:IXvQesVgOC7aXgK9OpYFn5eWnzz8fazegIiJ5WnCOVw= cloud.google.com/go/aiplatform v1.60.0 h1:0cSrii1ZeLr16MbBoocyy5KVnrSdiQ3KN/vtrTe7RqE= cloud.google.com/go/aiplatform v1.60.0/go.mod h1:eTlGuHOahHprZw3Hio5VKmtThIOak5/qy6pzdsqcQnM= cloud.google.com/go/aiplatform v1.67.0 h1:YWeqD4BjYwrmY4fa+isGcw0P81lJ3dKVxbWxdBchoiU= cloud.google.com/go/aiplatform v1.67.0/go.mod h1:s/sJ6btBEr6bKnrNWdK9ZgHCvwbZNdP90b3DDtxxw+Y= +cloud.google.com/go/aiplatform v1.68.0 h1:EPPqgHDJpBZKRvv+OsB3cr0jYz3EL2pZ+802rBPcG8U= +cloud.google.com/go/aiplatform v1.68.0/go.mod h1:105MFA3svHjC3Oazl7yjXAmIR89LKhRAeNdnDKJczME= cloud.google.com/go/analytics v0.23.0 h1:Q+y94XH84jM8SK8O7qiY/PJRexb6n7dRbQ6PiUa4YGM= cloud.google.com/go/analytics v0.23.0/go.mod h1:YPd7Bvik3WS95KBok2gPXDqQPHy08TsCQG6CdUCb+u0= cloud.google.com/go/analytics v0.23.2 h1:O0fj88npvQFxg8LfXo7fArcSrC/wtAstGuWQ7dCHWjg= cloud.google.com/go/analytics v0.23.2/go.mod h1:vtE3olAXZ6edJYk1UOndEs6EfaEc9T2B28Y4G5/a7Fo= +cloud.google.com/go/analytics v0.23.4 h1:5c425wSQBb+YAGr7ukgRFRAKa8SwlqTSapbb+CTJAEA= +cloud.google.com/go/analytics v0.23.4/go.mod h1:1iTnQMOr6zRdkecW+gkxJpwV0Q/djEIII3YlXmyf7UY= cloud.google.com/go/apigateway v1.6.5 h1:sPXnpk+6TneKIrjCjcpX5YGsAKy3PTdpIchoj8/74OE= cloud.google.com/go/apigateway v1.6.5/go.mod h1:6wCwvYRckRQogyDDltpANi3zsCDl6kWi0b4Je+w2UiI= cloud.google.com/go/apigateway v1.6.7 h1:DO5Vn3zmY1aDyfoqni8e8+x+lwrfLCoAAbEui9NB0y8= cloud.google.com/go/apigateway v1.6.7/go.mod h1:7wAMb/33Rzln+PrGK16GbGOfA1zAO5Pq6wp19jtIt7c= +cloud.google.com/go/apigateway v1.6.9 h1:vxZBKroYYCplsNrjggtniokb83Rk9mDitaiBN9nppdQ= +cloud.google.com/go/apigateway v1.6.9/go.mod h1:YE9XDTFwq859O6TpZNtatBMDWnMRZOiTVF+Ru3oCBeY= cloud.google.com/go/apigeeconnect v1.6.5 h1:CrfIKv9Go3fh/QfQgisU3MeP90Ww7l/sVGmr3TpECo8= cloud.google.com/go/apigeeconnect v1.6.5/go.mod h1:MEKm3AiT7s11PqTfKE3KZluZA9O91FNysvd3E6SJ6Ow= cloud.google.com/go/apigeeconnect v1.6.7 h1:z08Xuv7ZtaB2d4jsJi9/WhbnnI5s19wlLDZpssn3Fus= cloud.google.com/go/apigeeconnect v1.6.7/go.mod h1:hZxCKvAvDdKX8+eT0g5eEAbRSS9Gkzi+MPWbgAMAy5U= +cloud.google.com/go/apigeeconnect v1.6.9 h1:WO8XlUGugxvdKBj5hQnv8l7+SsVXgJVA97iNXyFgUb8= +cloud.google.com/go/apigeeconnect v1.6.9/go.mod h1:tl53uGgVG1A00qK1dF6wGIji0CQIMrLdNccJ6+R221U= cloud.google.com/go/apigeeregistry v0.8.3 h1:C+QU2K+DzDjk4g074ouwHQGkoff1h5OMQp6sblCVreQ= cloud.google.com/go/apigeeregistry v0.8.3/go.mod h1:aInOWnqF4yMQx8kTjDqHNXjZGh/mxeNlAf52YqtASUs= cloud.google.com/go/apigeeregistry v0.8.5 h1:o1C/+IvzwYeV1doum61XmJQ/Bwpk/4+2DT1JyVu2x64= cloud.google.com/go/apigeeregistry v0.8.5/go.mod h1:ZMg60hq2K35tlqZ1VVywb9yjFzk9AJ7zqxrysOxLi3o= +cloud.google.com/go/apigeeregistry v0.8.7 h1:K05SFNKzwvApZqVUcwg/6oFzn/b9WUrDN8pIdfD51qU= +cloud.google.com/go/apigeeregistry v0.8.7/go.mod h1:Jge1HQaIkNU8JYSDY7l5SveeSKvGPvtLjzNjLU2+0N8= cloud.google.com/go/appengine v1.8.5 h1:l2SviT44zWQiOv8bPoMBzW0vOcMO22iO0s+nVtVhdts= cloud.google.com/go/appengine v1.8.5/go.mod h1:uHBgNoGLTS5di7BvU25NFDuKa82v0qQLjyMJLuPQrVo= cloud.google.com/go/appengine v1.8.7 h1:qYrjEHEFY7+CL4QlHIHuwTgrTnZbSKzdPFqgjZDsQNo= cloud.google.com/go/appengine v1.8.7/go.mod h1:1Fwg2+QTgkmN6Y+ALGwV8INLbdkI7+vIvhcKPZCML0g= +cloud.google.com/go/appengine v1.8.9 h1:rI/aezyrwereUE0i/umbA6rZIgpJpBImFcy3JJEcQd0= +cloud.google.com/go/appengine v1.8.9/go.mod h1:sw8T321TAto/u6tMinv3AV63olGH/hw7RhG4ZgNhqFs= cloud.google.com/go/area120 v0.8.5 h1:vTs08KPLN/iMzTbxpu5ciL06KcsrVPMjz4IwcQyZ4uY= cloud.google.com/go/area120 v0.8.5/go.mod h1:BcoFCbDLZjsfe4EkCnEq1LKvHSK0Ew/zk5UFu6GMyA0= cloud.google.com/go/area120 v0.8.7 h1:sUrR96yokdL6tTTXK0X13V1TLMta8/1u328bRG5lWZc= cloud.google.com/go/area120 v0.8.7/go.mod h1:L/xTq4NLP9mmxiGdcsVz7y1JLc9DI8pfaXRXbnjkR6w= +cloud.google.com/go/area120 v0.8.9 h1:38uHviqcdB2S83yPfOXzDxf0KTG/W2DsXMuY/uf2T8c= +cloud.google.com/go/area120 v0.8.9/go.mod h1:epLvbmajRp919r1LGdvS1zgcHJt/1MTQJJ9+r0/NBQc= cloud.google.com/go/artifactregistry v1.14.7 h1:W9sVlyb1VRcUf83w7aM3yMsnp4HS4PoyGqYQNG0O5lI= cloud.google.com/go/artifactregistry v1.14.7/go.mod h1:0AUKhzWQzfmeTvT4SjfI4zjot72EMfrkvL9g9aRjnnM= cloud.google.com/go/artifactregistry v1.14.9 h1:SSvoD0ofOydm5gA1++15pW9VPgQbk0OmNlcb7JczoO4= cloud.google.com/go/artifactregistry v1.14.9/go.mod h1:n2OsUqbYoUI2KxpzQZumm6TtBgtRf++QulEohdnlsvI= +cloud.google.com/go/artifactregistry v1.14.11 h1:NZzHn5lPKyi2kgtM9Atu6IBvqslL7Fu1+5EkOYZd+yk= +cloud.google.com/go/artifactregistry v1.14.11/go.mod h1:ahyKXer42EOIddYzk2zYfvZnByGPdAYhXqBbRBsGizE= cloud.google.com/go/asset v1.17.2 h1:xgFnBP3luSbUcC9RWJvb3Zkt+y/wW6PKwPHr3ssnIP8= cloud.google.com/go/asset v1.17.2/go.mod h1:SVbzde67ehddSoKf5uebOD1sYw8Ab/jD/9EIeWg99q4= cloud.google.com/go/asset v1.19.1 h1:mCqyoaDjDzaW1RqmmQtCJuawb9nca5bEu7HvVcpZDwg= cloud.google.com/go/asset v1.19.1/go.mod h1:kGOS8DiCXv6wU/JWmHWCgaErtSZ6uN5noCy0YwVaGfs= +cloud.google.com/go/asset v1.19.3 h1:vl8wy3jpRa3ATctym5tiICp70iymSyOVbpKb3tKA668= +cloud.google.com/go/asset v1.19.3/go.mod h1:1j8NNcHsbSE/KeHMZrizPIS6c8nm0WjEAPoFXzXNCj4= cloud.google.com/go/assuredworkloads v1.11.5 h1:gCrN3IyvqY3cP0wh2h43d99CgH3G+WYs9CeuFVKChR8= cloud.google.com/go/assuredworkloads v1.11.5/go.mod h1:FKJ3g3ZvkL2D7qtqIGnDufFkHxwIpNM9vtmhvt+6wqk= cloud.google.com/go/assuredworkloads v1.11.7 h1:xieyFA+JKyTDkO/Z9UyVEpkHW8pDYykU51O4G0pvXEg= cloud.google.com/go/assuredworkloads v1.11.7/go.mod h1:CqXcRH9N0KCDtHhFisv7kk+cl//lyV+pYXGi1h8rCEU= +cloud.google.com/go/assuredworkloads v1.11.9 h1:xMjLtM24zy8yWGZlNtYxXo9fBj7ArWTsNkXKlRBZlqw= +cloud.google.com/go/assuredworkloads v1.11.9/go.mod h1:uZ6+WHiT4iGn1iM1wk5njKnKJWiM3v/aYhDoCoHxs1w= cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg= cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/automl v1.13.5 h1:ijiJy9sYWh75WrqImXsfWc1e3HR3iO+ef9fvW03Ig/4= cloud.google.com/go/automl v1.13.5/go.mod h1:MDw3vLem3yh+SvmSgeYUmUKqyls6NzSumDm9OJ3xJ1Y= cloud.google.com/go/automl v1.13.7 h1:w9AyogtMLXbcy5kzXPvk/Q3MGQkgJH7ZDB8fAUUxTt8= cloud.google.com/go/automl v1.13.7/go.mod h1:E+s0VOsYXUdXpq0y4gNZpi0A/s6y9+lAarmV5Eqlg40= +cloud.google.com/go/automl v1.13.9 h1:GzYpU33Zo2tQ+8amLasjeBPawpKfBYnLGHVMQcyiFv4= +cloud.google.com/go/automl v1.13.9/go.mod h1:KECCWW2AFsRuEVxUJEIXxcm3yPLf1rxS+qsBamyacMc= cloud.google.com/go/baremetalsolution v1.2.4 h1:LFydisRmS7hQk9P/YhekwuZGqb45TW4QavcrMToWo5A= cloud.google.com/go/baremetalsolution v1.2.4/go.mod h1:BHCmxgpevw9IEryE99HbYEfxXkAEA3hkMJbYYsHtIuY= cloud.google.com/go/baremetalsolution v1.2.6 h1:W4oSMS6vRCo9DLr1RPyDP8oeLverbvhJRzaZSsipft8= cloud.google.com/go/baremetalsolution v1.2.6/go.mod h1:KkS2BtYXC7YGbr42067nzFr+ABFMs6cxEcA1F+cedIw= +cloud.google.com/go/baremetalsolution v1.2.8 h1:mM8zaxertfV5gaNGloJdJY87z7l8WcNkhw96VB1IGTQ= +cloud.google.com/go/baremetalsolution v1.2.8/go.mod h1:Ai8ENs7ADMYWQ45DtfygUc6WblhShfi3kNPvuGv8/ok= cloud.google.com/go/batch v1.8.0 h1:2HK4JerwVaIcCh/lJiHwh6+uswPthiMMWhiSWLELayk= cloud.google.com/go/batch v1.8.0/go.mod h1:k8V7f6VE2Suc0zUM4WtoibNrA6D3dqBpB+++e3vSGYc= cloud.google.com/go/batch v1.8.7 h1:zaQwOAd7TlE84pwPHavNMsnv5zRyRV8ym2DJ4iQ2cV0= cloud.google.com/go/batch v1.8.7/go.mod h1:O5/u2z8Wc7E90Bh4yQVLQIr800/0PM5Qzvjac3Jxt4k= +cloud.google.com/go/batch v1.9.0 h1:WlOqpQMOtWvOLIs7vCxBwYZGaB76i3olsBCVUvszY3M= +cloud.google.com/go/batch v1.9.0/go.mod h1:VhRaG/bX2EmeaPSHvtptP5OAhgYuTrvtTAulKM68oiI= cloud.google.com/go/beyondcorp v1.0.4 h1:qs0J0O9Ol2h1yA0AU+r7l3hOCPzs2MjE1d6d/kaHIKo= cloud.google.com/go/beyondcorp v1.0.4/go.mod h1:Gx8/Rk2MxrvWfn4WIhHIG1NV7IBfg14pTKv1+EArVcc= cloud.google.com/go/beyondcorp v1.0.6 h1:KBcujO3QRvBIwzZLtvQEPB9SXdovHnMBx0V/uhucH9o= cloud.google.com/go/beyondcorp v1.0.6/go.mod h1:wRkenqrVRtnGFfnyvIg0zBFUdN2jIfeojFF9JJDwVIA= +cloud.google.com/go/beyondcorp v1.0.8 h1:Dw9VO8fzZ2GWsfgx4wST03hjaZmRNv09lt0GZuzyVxM= +cloud.google.com/go/beyondcorp v1.0.8/go.mod h1:2WaEvUnw+1ZIUNu227h71X/Q8ypcWWowii9TQ4xlfo0= cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.59.1 h1:CpT+/njKuKT3CEmswm6IbhNu9u35zt5dO4yPDLW+nG4= cloud.google.com/go/bigquery v1.59.1/go.mod h1:VP1UJYgevyTwsV7desjzNzDND5p6hZB+Z8gZJN1GQUc= @@ -88,132 +125,195 @@ cloud.google.com/go/billing v1.18.2 h1:oWUEQvuC4JvtnqLZ35zgzdbuHt4Itbftvzbe6aEyF cloud.google.com/go/billing v1.18.2/go.mod h1:PPIwVsOOQ7xzbADCwNe8nvK776QpfrOAUkvKjCUcpSE= cloud.google.com/go/billing v1.18.5 h1:GbOg1uGvoV8FXxMStFoNcq5z9AEUwCpKt/6GNcuDSZM= cloud.google.com/go/billing v1.18.5/go.mod h1:lHw7fxS6p7hLWEPzdIolMtOd0ahLwlokW06BzbleKP8= +cloud.google.com/go/billing v1.18.7 h1:1Y7DdC2i8JQctWpd1ycra5iK+2LzwgFi+TxTqF3Yyp8= +cloud.google.com/go/billing v1.18.7/go.mod h1:RreCBJPmaN/lzCz/2Xl1hA+OzWGqrzDsax4Qjjp0CbA= cloud.google.com/go/binaryauthorization v1.8.1 h1:1jcyh2uIUwSZkJ/JmL8kd5SUkL/Krbv8zmYLEbAz6kY= cloud.google.com/go/binaryauthorization v1.8.1/go.mod h1:1HVRyBerREA/nhI7yLang4Zn7vfNVA3okoAR9qYQJAQ= cloud.google.com/go/binaryauthorization v1.8.3 h1:RHnEM4HXbWShlGhPA0Jzj2YYETCHxmisNMU0OE2fXQM= cloud.google.com/go/binaryauthorization v1.8.3/go.mod h1:Cul4SsGlbzEsWPOz2sH8m+g2Xergb6ikspUyQ7iOThE= +cloud.google.com/go/binaryauthorization v1.8.5 h1:ly5gQoJGHbuOM7E+pND38pTiQ0pZ4zTEOfJlfyfIIew= +cloud.google.com/go/binaryauthorization v1.8.5/go.mod h1:2npTMgNJPsmUg0jfmDDORuqBkTPEW6ZSTHXzfxTvN1M= cloud.google.com/go/certificatemanager v1.7.5 h1:UMBr/twXvH3jcT5J5/YjRxf2tvwTYIfrpemTebe0txc= cloud.google.com/go/certificatemanager v1.7.5/go.mod h1:uX+v7kWqy0Y3NG/ZhNvffh0kuqkKZIXdvlZRO7z0VtM= cloud.google.com/go/certificatemanager v1.8.1 h1:XURrQhj5COWAEvICivbGID/Hu67AvMYHAhMRIyc3Ux8= cloud.google.com/go/certificatemanager v1.8.1/go.mod h1:hDQzr50Vx2gDB+dOfmDSsQzJy/UPrYRdzBdJ5gAVFIc= +cloud.google.com/go/certificatemanager v1.8.3 h1:feyxS5Q8eWQNXQcVAcdooQEKGT/1B/qCcYvamOen7fc= +cloud.google.com/go/certificatemanager v1.8.3/go.mod h1:QS0jxTu5wgEbzaYgGs/GBYKvVgAgc9jnYaaTFH8jRtE= cloud.google.com/go/channel v1.17.5 h1:/omiBnyFjm4S1ETHoOmJbL7LH7Ljcei4rYG6Sj3hc80= cloud.google.com/go/channel v1.17.5/go.mod h1:FlpaOSINDAXgEext0KMaBq/vwpLMkkPAw9b2mApQeHc= cloud.google.com/go/channel v1.17.7 h1:PrplNaAS6Dn187e+OcGzyEKETX8iL3tCaDqcPPW7Zoo= cloud.google.com/go/channel v1.17.7/go.mod h1:b+FkgBrhMKM3GOqKUvqHFY/vwgp+rwsAuaMd54wCdN4= +cloud.google.com/go/channel v1.17.9 h1:rqF5CjW6KnOmlVZ75PNkuXYh5nh8dIsIWQjHLLwPy3Y= +cloud.google.com/go/channel v1.17.9/go.mod h1:h9emIJm+06sK1FxqC3etsWdG87tg92T24wimlJs6lhY= cloud.google.com/go/cloudbuild v1.15.1 h1:ZB6oOmJo+MTov9n629fiCrO9YZPOg25FZvQ7gIHu5ng= cloud.google.com/go/cloudbuild v1.15.1/go.mod h1:gIofXZSu+XD2Uy+qkOrGKEx45zd7s28u/k8f99qKals= cloud.google.com/go/cloudbuild v1.16.1 h1:zkCG1dBezxRM3dtgQ9h1Y+IJ7V+lARWgp0l9k/SZsfU= cloud.google.com/go/cloudbuild v1.16.1/go.mod h1:c2KUANTtCBD8AsRavpPout6Vx8W+fsn5zTsWxCpWgq4= +cloud.google.com/go/cloudbuild v1.16.3 h1:BIT0cFWQDT4XTVMyyZsjXvltVqBwvJ/RAKIRBqkgXf0= +cloud.google.com/go/cloudbuild v1.16.3/go.mod h1:KJYZAwTUaDKDdEHwLj/EmnpmwLkMuq+fGnBEHA1LlE4= cloud.google.com/go/clouddms v1.7.4 h1:Sr0Zo5EAcPQiCBgHWICg3VGkcdS/LLP1d9SR7qQBM/s= cloud.google.com/go/clouddms v1.7.4/go.mod h1:RdrVqoFG9RWI5AvZ81SxJ/xvxPdtcRhFotwdE79DieY= cloud.google.com/go/clouddms v1.7.6 h1:Q47KKoA0zsNcC9U5aCmop5TPPItVq4cx7Wwqgra+5PU= cloud.google.com/go/clouddms v1.7.6/go.mod h1:8HWZ2tznZ0mNAtTpfnRNT0QOThqn9MBUqTj0Lx8npIs= +cloud.google.com/go/clouddms v1.7.8 h1:WEz8ECgv4ZinRc84xcW1wTsFfLNb60yrSsIdsEJFRDk= +cloud.google.com/go/clouddms v1.7.8/go.mod h1:KQpBMxH99ZTPK4LgXkYUntzRQ5hcNkjpGRbNSRzW9Nk= cloud.google.com/go/cloudtasks v1.12.6 h1:EUt1hIZ9bLv8Iz9yWaCrqgMnIU+Tdh0yXM1MMVGhjfE= cloud.google.com/go/cloudtasks v1.12.6/go.mod h1:b7c7fe4+TJsFZfDyzO51F7cjq7HLUlRi/KZQLQjDsaY= cloud.google.com/go/cloudtasks v1.12.8 h1:Y0HUuiCAVk9BojLItOycBl91tY25NXH8oFsyi1IC/U4= cloud.google.com/go/cloudtasks v1.12.8/go.mod h1:aX8qWCtmVf4H4SDYUbeZth9C0n9dBj4dwiTYi4Or/P4= +cloud.google.com/go/cloudtasks v1.12.10 h1:2mdGqvYFm9HwPh//ckbcX8mZJgyG+F1TWk+82+eLuwM= +cloud.google.com/go/cloudtasks v1.12.10/go.mod h1:OHJzRAdE+7H00cdsINhb21ugVLDgk3Uh4r0holCB5XQ= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute v1.27.0 h1:EGawh2RUnfHT5g8f/FX3Ds6KZuIBC77hZoDrBvEZw94= cloud.google.com/go/compute v1.27.0/go.mod h1:LG5HwRmWFKM2C5XxHRiNzkLLXW48WwvyVC0mfWsYPOM= +cloud.google.com/go/compute v1.27.2 h1:5cE5hdrwJV/92ravlwIFRGnyH9CpLGhh4N0ZDVTU+BA= +cloud.google.com/go/compute v1.27.2/go.mod h1:YQuHkNEwP3bIz4LBYQqf4DIMfFtTDtnEgnwG0mJQQ9I= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.13.0 h1:6Vs/YnDG5STGjlWMEjN/xtmft7MrOTOnOZYUZtGTx0w= cloud.google.com/go/contactcenterinsights v1.13.0/go.mod h1:ieq5d5EtHsu8vhe2y3amtZ+BE+AQwX5qAy7cpo0POsI= cloud.google.com/go/contactcenterinsights v1.13.2 h1:46ertIh+cGkTg/lN7fN+TOx09SoM65dpdUp96vXBcMY= cloud.google.com/go/contactcenterinsights v1.13.2/go.mod h1:AfkSB8t7mt2sIY6WpfO61nD9J9fcidIchtxm9FqJVXk= +cloud.google.com/go/contactcenterinsights v1.13.4 h1:BvtC33BbX+3p+v+VB0AZ6djRrcXP+qPqfWsIR+kz5v8= +cloud.google.com/go/contactcenterinsights v1.13.4/go.mod h1:6OWSyQxeaQRxhkyMhtE+RFOOlsMcKOTukv8nnjxbNCQ= cloud.google.com/go/container v1.31.0 h1:MAaNH7VRNPWEhvqOypq2j+7ONJKrKzon4v9nS3nLZe0= cloud.google.com/go/container v1.31.0/go.mod h1:7yABn5s3Iv3lmw7oMmyGbeV6tQj86njcTijkkGuvdZA= cloud.google.com/go/container v1.35.1 h1:Vbu/3PZNrgV1Z5DGcRubQdUccX/uMUDNc+NgHNIfbEk= cloud.google.com/go/container v1.35.1/go.mod h1:udm8fgLm3TtpnjFN4QLLjZezAIIp/VnMo316yIRVRQU= +cloud.google.com/go/container v1.37.2 h1:g5zm1SUBZ+q7IvtI5hM/6xcpf2C/bFfN2EXzS07Iz9k= +cloud.google.com/go/container v1.37.2/go.mod h1:2ly7zpBmWtYjjuoB3fHyq8Gqrxaj2NIwzwVRpUcKYXk= cloud.google.com/go/containeranalysis v0.11.4 h1:doJ0M1ljS4hS0D2UbHywlHGwB7sQLNrt9vFk9Zyi7vY= cloud.google.com/go/containeranalysis v0.11.4/go.mod h1:cVZT7rXYBS9NG1rhQbWL9pWbXCKHWJPYraE8/FTSYPE= cloud.google.com/go/containeranalysis v0.11.6 h1:mSrneOVadcpnDZHJebg+ts/10azGTUKOCSQET7KdT7g= cloud.google.com/go/containeranalysis v0.11.6/go.mod h1:YRf7nxcTcN63/Kz9f86efzvrV33g/UV8JDdudRbYEUI= +cloud.google.com/go/containeranalysis v0.11.8 h1:1rkYgK2szbRH311mRw/3lkeEOqrjN+2gOD7AZhMUxZw= +cloud.google.com/go/containeranalysis v0.11.8/go.mod h1:2ru4oxs6dCcaG3ZsmKAy4yMmG68ukOuS/IRCMEHYpLo= cloud.google.com/go/datacatalog v1.19.3 h1:A0vKYCQdxQuV4Pi0LL9p39Vwvg4jH5yYveMv50gU5Tw= cloud.google.com/go/datacatalog v1.19.3/go.mod h1:ra8V3UAsciBpJKQ+z9Whkxzxv7jmQg1hfODr3N3YPJ4= cloud.google.com/go/datacatalog v1.20.1 h1:czcba5mxwRM5V//jSadyig0y+8aOHmN7gUl9GbHu59E= cloud.google.com/go/datacatalog v1.20.1/go.mod h1:Jzc2CoHudhuZhpv78UBAjMEg3w7I9jHA11SbRshWUjk= +cloud.google.com/go/datacatalog v1.20.3 h1:lzMtWaUlaz9Bd9anvq2KBZwcFujzhVuxhIz1MsqRJv8= +cloud.google.com/go/datacatalog v1.20.3/go.mod h1:AKC6vAy5urnMg5eJK3oUjy8oa5zMbiY33h125l8lmlo= cloud.google.com/go/dataflow v0.9.5 h1:RYHtcPhmE664+F0Je46p+NvFbG8z//KCXp+uEqB4jZU= cloud.google.com/go/dataflow v0.9.5/go.mod h1:udl6oi8pfUHnL0z6UN9Lf9chGqzDMVqcYTcZ1aPnCZQ= cloud.google.com/go/dataflow v0.9.7 h1:wKEakCbRevlwsWqTn34pWJUFmdbx0HKwpRH6HhU7NIs= cloud.google.com/go/dataflow v0.9.7/go.mod h1:3BjkOxANrm1G3+/EBnEsTEEgJu1f79mFqoOOZfz3v+E= +cloud.google.com/go/dataflow v0.9.9 h1:7qTWGXfpM2z3assRznIXJLw+XJNlucHFcvFAYhclQ+o= +cloud.google.com/go/dataflow v0.9.9/go.mod h1:Wk/92E1BvhV7qs/dWb+3dN26uGgyp/H1Jr5ZJxeD3dw= cloud.google.com/go/dataform v0.9.2 h1:5e4eqGrd0iDTCg4Q+VlAao5j2naKAA7xRurNtwmUknU= cloud.google.com/go/dataform v0.9.2/go.mod h1:S8cQUwPNWXo7m/g3DhWHsLBoufRNn9EgFrMgne2j7cI= cloud.google.com/go/dataform v0.9.4 h1:MiK1Us7YP9+sdNViUE4X2B2vLScrKcjOPw5b6uamZvE= cloud.google.com/go/dataform v0.9.4/go.mod h1:jjo4XY+56UrNE0wsEQsfAw4caUs4DLJVSyFBDelRDtQ= +cloud.google.com/go/dataform v0.9.6 h1:8BMoPO9CD3qmPqnunVi73JcvwQrkjLILPXZKFExsjZc= +cloud.google.com/go/dataform v0.9.6/go.mod h1:JKDPMfcYMu9oUMubIvvAGWTBX0sw4o/JIjCcczzbHmk= cloud.google.com/go/datafusion v1.7.5 h1:HQ/BUOP8OIGJxuztpYvNvlb+/U+/Bfs9SO8tQbh61fk= cloud.google.com/go/datafusion v1.7.5/go.mod h1:bYH53Oa5UiqahfbNK9YuYKteeD4RbQSNMx7JF7peGHc= cloud.google.com/go/datafusion v1.7.7 h1:ViFnMnUK7LNcWvisZgihxXit76JxSHFeijYI5U/gjOE= cloud.google.com/go/datafusion v1.7.7/go.mod h1:qGTtQcUs8l51lFA9ywuxmZJhS4ozxsBSus6ItqCUWMU= +cloud.google.com/go/datafusion v1.7.9 h1:ZwicZskyu64L8Y6+zvZQjIav5A1xYwM0nqpk88HKLmY= +cloud.google.com/go/datafusion v1.7.9/go.mod h1:ciYV8FL0JmrwgoJ7CH64oUHiI0oOf2VLE45LWKT51Ls= cloud.google.com/go/datalabeling v0.8.5 h1:GpIFRdm0qIZNsxqURFJwHt0ZBJZ0nF/mUVEigR7PH/8= cloud.google.com/go/datalabeling v0.8.5/go.mod h1:IABB2lxQnkdUbMnQaOl2prCOfms20mcPxDBm36lps+s= cloud.google.com/go/datalabeling v0.8.7 h1:M6irSHns6VxMro+IbvDxDJLD6tkfjlW+mo2MPaM23KA= cloud.google.com/go/datalabeling v0.8.7/go.mod h1:/PPncW5gxrU15UzJEGQoOT3IobeudHGvoExrtZ8ZBwo= +cloud.google.com/go/datalabeling v0.8.9 h1:4ndOrLlhYErzhJGciRJx+s33+6P4cS23GnROnfaJ6hE= +cloud.google.com/go/datalabeling v0.8.9/go.mod h1:61QutR66VZFgN8boHhl4/FTfxenNzihykv18BgxwSrg= cloud.google.com/go/dataplex v1.14.2 h1:fxIfdU8fxzR3clhOoNI7XFppvAmndxDu1AMH+qX9WKQ= cloud.google.com/go/dataplex v1.14.2/go.mod h1:0oGOSFlEKef1cQeAHXy4GZPB/Ife0fz/PxBf+ZymA2U= cloud.google.com/go/dataplex v1.16.0 h1:e8SV0yKuSjgHEZaQcZwjKXe0ta1jZrvLxX/2i/IAG+8= cloud.google.com/go/dataplex v1.16.0/go.mod h1:OlBoytuQ56+7aUCC03D34CtoF/4TJ5SiIrLsBdDu87Q= +cloud.google.com/go/dataplex v1.18.0 h1:kXCHm9TqTr5BhZnsSD32iCRmf1S+Hho+UDqXr3Gdw7s= +cloud.google.com/go/dataplex v1.18.0/go.mod h1:THLDVG07lcY1NgqVvjTV1mvec+rFHwpDwvSd+196MMc= cloud.google.com/go/dataproc v1.12.0 h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3YgGU= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataproc/v2 v2.4.0 h1:/u81Fd+BvCLp+xjctI1DiWVJn6cn9/s3Akc8xPH02yk= cloud.google.com/go/dataproc/v2 v2.4.0/go.mod h1:3B1Ht2aRB8VZIteGxQS/iNSJGzt9+CA0WGnDVMEm7Z4= cloud.google.com/go/dataproc/v2 v2.4.2 h1:RNMG5ffWKdbWOkwvjC4GqxLaxEaWFpm2hQCF2WFW/vo= cloud.google.com/go/dataproc/v2 v2.4.2/go.mod h1:smGSj1LZP3wtnsM9eyRuDYftNAroAl6gvKp/Wk64XDE= +cloud.google.com/go/dataproc/v2 v2.5.1 h1:AYq1wJCKHrSG4KtxMQPkn1b0/uaHULHbXXTukCgou90= +cloud.google.com/go/dataproc/v2 v2.5.1/go.mod h1:5s2CuQyTPX7e19ZRMLicfPFNgXrvsVct3xz94UvWFeQ= cloud.google.com/go/dataqna v0.8.5 h1:9ybXs3nr9BzxSGC04SsvtuXaHY0qmJSLIpIAbZo9GqQ= cloud.google.com/go/dataqna v0.8.5/go.mod h1:vgihg1mz6n7pb5q2YJF7KlXve6tCglInd6XO0JGOlWM= cloud.google.com/go/dataqna v0.8.7 h1:qM60MGNTGsSJuzAziVJjtRA7pGby2dA8OuqdVRe/lYo= cloud.google.com/go/dataqna v0.8.7/go.mod h1:hvxGaSvINAVH5EJJsONIwT1y+B7OQogjHPjizOFoWOo= +cloud.google.com/go/dataqna v0.8.9 h1:7kiDfd4c/pSW8jmeeOac/H+PYgwLrIt4L88s4JiFRZU= +cloud.google.com/go/dataqna v0.8.9/go.mod h1:wrw1SL/zLRlVgf0d8P0ZBJ2hhGaLbwoNRsW6m1mn64g= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.15.0 h1:0P9WcsQeTWjuD1H14JIY7XQscIPQ4Laje8ti96IC5vg= cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= cloud.google.com/go/datastore v1.17.0 h1:UEmzuUdyDE58HV2jcb0BoqwCAwsJS2mtHapCsMmhVh0= cloud.google.com/go/datastore v1.17.0/go.mod h1:RiRZU0G6VVlIVlv1HRo3vSAPFHULV0ddBNsXO+Sony4= +cloud.google.com/go/datastore v1.17.1 h1:6Me8ugrAOAxssGhSo8im0YSuy4YvYk4mbGvCadAH5aE= +cloud.google.com/go/datastore v1.17.1/go.mod h1:mtzZ2HcVtz90OVrEXXGDc2pO4NM1kiBQy8YV4qGe0ZM= cloud.google.com/go/datastream v1.10.4 h1:o1QDKMo/hk0FN7vhoUQURREuA0rgKmnYapB+1M+7Qz4= cloud.google.com/go/datastream v1.10.4/go.mod h1:7kRxPdxZxhPg3MFeCSulmAJnil8NJGGvSNdn4p1sRZo= cloud.google.com/go/datastream v1.10.6 h1:FfNUy9j3aRQ99L4a5Rdm82RMuiw0BIe3lpPn2ykom8k= cloud.google.com/go/datastream v1.10.6/go.mod h1:lPeXWNbQ1rfRPjBFBLUdi+5r7XrniabdIiEaCaAU55o= +cloud.google.com/go/datastream v1.10.8 h1:INvIDTnuti68Lmdizp2GUyRFjN9k3X7IowX0Ixy9Vto= +cloud.google.com/go/datastream v1.10.8/go.mod h1:6nkPjnk5Qr602Wq+YQ+/RWUOX5h4voMTz5abgEOYPCM= cloud.google.com/go/deploy v1.17.1 h1:m27Ojwj03gvpJqCbodLYiVmE9x4/LrHGGMjzc0LBfM4= cloud.google.com/go/deploy v1.17.1/go.mod h1:SXQyfsXrk0fBmgBHRzBjQbZhMfKZ3hMQBw5ym7MN/50= cloud.google.com/go/deploy v1.19.0 h1:fzbObuGgoViO0ArFuOQIJ2yr5bH5YzbORVvMDBrDC5I= cloud.google.com/go/deploy v1.19.0/go.mod h1:BW9vAujmxi4b/+S7ViEuYR65GiEsqL6Mhf5S/9TeDRU= +cloud.google.com/go/deploy v1.19.2 h1:C8T/Hna2lDE8qbwP75G+iJclnrIj7oblBoQoc1cfDWc= +cloud.google.com/go/deploy v1.19.2/go.mod h1:i6zfU9FZkqFgWIvO2/gsodGU9qF4tF9mBgoMdfnf6as= cloud.google.com/go/dialogflow v1.49.0 h1:KqG0oxGE71qo0lRVyAoeBozefCvsMfcDzDjoLYSY0F4= cloud.google.com/go/dialogflow v1.49.0/go.mod h1:dhVrXKETtdPlpPhE7+2/k4Z8FRNUp6kMV3EW3oz/fe0= cloud.google.com/go/dialogflow v1.53.0 h1:C9wQ0odRYQsar0XqwCQb0c13BkRBsoSjOaejOg5ntgQ= cloud.google.com/go/dialogflow v1.53.0/go.mod h1:LqAvxq7bXiiGC3/DWIz9XXCxth2z2qpSnBAAmlNOj6U= +cloud.google.com/go/dialogflow v1.54.2 h1:uS7IDkXIUR5EduLfyPmgTpZ27RcUIHby7JKsk4fBPdo= +cloud.google.com/go/dialogflow v1.54.2/go.mod h1:avkFNYog+U127jKpGzW1FOllBwZy3OfCz1K1eE9RGh8= cloud.google.com/go/dlp v1.11.2 h1:lTipOuJaSjlYnnotPMbEhKURLC6GzCMDDzVbJAEbmYM= cloud.google.com/go/dlp v1.11.2/go.mod h1:9Czi+8Y/FegpWzgSfkRlyz+jwW6Te9Rv26P3UfU/h/w= cloud.google.com/go/dlp v1.14.0 h1:/GQVl5gOPR2dUemrR2YJxZG5D9MCE3AYgmDxjzP54jI= cloud.google.com/go/dlp v1.14.0/go.mod h1:4fvEu3EbLsHrgH3QFdFlTNIiCP5mHwdYhS/8KChDIC4= +cloud.google.com/go/dlp v1.14.2 h1:oR15Jcd/grn//eftZ/B0DJ99lTaeN8vOf8TK5xhKEvc= +cloud.google.com/go/dlp v1.14.2/go.mod h1:+uwRt+6wZ3PL0wsmZ1cUAj0Mt9kyeV3WcIKPW03wJVU= cloud.google.com/go/documentai v1.25.0 h1:lI62GMEEPO6vXJI9hj+G9WjOvnR0hEjvjokrnex4cxA= cloud.google.com/go/documentai v1.25.0/go.mod h1:ftLnzw5VcXkLItp6pw1mFic91tMRyfv6hHEY5br4KzY= cloud.google.com/go/documentai v1.30.0 h1:6KI6P04WExzrfbciW5RTEQScBEY98Fc4VtS04ufT3Js= cloud.google.com/go/documentai v1.30.0/go.mod h1:3Qt8PMt3S8W6w3VeoYFraaMS2GJRrXFnvkyn+GpB1n0= +cloud.google.com/go/documentai v1.30.3 h1:D75r7hqnc9Zz6aRV8fzs/1V94R5YIv+FDJivUT4r+n4= +cloud.google.com/go/documentai v1.30.3/go.mod h1:aMxiOouLr36hyahLhI3OwAcsy7plOTiXR/RmK+MHbSg= cloud.google.com/go/domains v0.9.5 h1:Mml/R6s3vQQvFPpi/9oX3O5dRirgjyJ8cksK8N19Y7g= cloud.google.com/go/domains v0.9.5/go.mod h1:dBzlxgepazdFhvG7u23XMhmMKBjrkoUNaw0A8AQB55Y= cloud.google.com/go/domains v0.9.7 h1:IixFIMRzUJWZUAOe8s/K2X4Bvtp0A3xjHLljfNC4aSo= cloud.google.com/go/domains v0.9.7/go.mod h1:u/yVf3BgfPJW3QDZl51qTJcDXo9PLqnEIxfGmGgbHEc= +cloud.google.com/go/domains v0.9.9 h1:kIqgwkIph6Mw+m1nWafdEBrGqPPZ1J98hqO11gkL4BM= +cloud.google.com/go/domains v0.9.9/go.mod h1:/ewEPIaNmTrElY7u9BZPcLPnoP1NJJXGvISDDapwVNU= cloud.google.com/go/edgecontainer v1.1.5 h1:tBY32km78ScpK2aOP84JoW/+wtpx5WluyPUSEE3270U= cloud.google.com/go/edgecontainer v1.1.5/go.mod h1:rgcjrba3DEDEQAidT4yuzaKWTbkTI5zAMu3yy6ZWS0M= cloud.google.com/go/edgecontainer v1.2.1 h1:xa6MIQhGylE24QdWaxhfIfAJE3Pupcr+i77WEx3NJrg= cloud.google.com/go/edgecontainer v1.2.1/go.mod h1:OE2D0lbkmGDVYLCvpj8Y0M4a4K076QB7E2JupqOR/qU= +cloud.google.com/go/edgecontainer v1.2.3 h1:F5UsQ/A4GjkV9dTBi3KMFGXPa/6OdTk5/Dce2bdYonM= +cloud.google.com/go/edgecontainer v1.2.3/go.mod h1:gMKe2JfE0OT0WuCJArzIndAmMWDPCIYGSWYIpJ6M7oM= cloud.google.com/go/errorreporting v0.3.0 h1:kj1XEWMu8P0qlLhm3FwcaFsUvXChV/OraZwA70trRR0= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/errorreporting v0.3.1 h1:E/gLk+rL7u5JZB9oq72iL1bnhVlLrnfslrgcptjJEUE= +cloud.google.com/go/errorreporting v0.3.1/go.mod h1:6xVQXU1UuntfAf+bVkFk6nld41+CPyF2NSPCyXE3Ztk= cloud.google.com/go/essentialcontacts v1.6.6 h1:13eHn5qBnsawxI7mIrv4jRIEmQ1xg0Ztqw5ZGqtUNfA= cloud.google.com/go/essentialcontacts v1.6.6/go.mod h1:XbqHJGaiH0v2UvtuucfOzFXN+rpL/aU5BCZLn4DYl1Q= cloud.google.com/go/essentialcontacts v1.6.8 h1:p5Y7ZNVPiV9pEAHzvWiPcSiQRMQqcuHxOP0ZOP0vVww= cloud.google.com/go/essentialcontacts v1.6.8/go.mod h1:EHONVDSum2xxG2p+myyVda/FwwvGbY58ZYC4XqI/lDQ= +cloud.google.com/go/essentialcontacts v1.6.10 h1:zI+3LgjRcv7StB7O35sWqCg79OKDx5sRR4GAq36fi+s= +cloud.google.com/go/essentialcontacts v1.6.10/go.mod h1:wQlXvEb/0hB0C0d4H6/90P8CiZcYewkvJ3VoUVFPi4E= cloud.google.com/go/eventarc v1.13.4 h1:ORkd6/UV5FIdA8KZQDLNZYKS7BBOrj0p01DXPmT4tE4= cloud.google.com/go/eventarc v1.13.4/go.mod h1:zV5sFVoAa9orc/52Q+OuYUG9xL2IIZTbbuTHC6JSY8s= cloud.google.com/go/eventarc v1.13.6 h1:we+qx5uCZ88aQzQS3MJXRvAh/ik+EmqVyjcW1oYFW44= cloud.google.com/go/eventarc v1.13.6/go.mod h1:QReOaYnDNdjwAQQWNC7nfr63WnaKFUw7MSdQ9PXJYj0= +cloud.google.com/go/eventarc v1.13.8 h1:2sbz7e95cv6zm2mNrMJlAQ6J93qQsGCQzw4lYa5GWJQ= +cloud.google.com/go/eventarc v1.13.8/go.mod h1:Xq3SsMoOAn7RmacXgJO7kq818iRLFF0bVhH780qlmTs= cloud.google.com/go/filestore v1.8.1 h1:X5G4y/vrUo1B8Nsz93qSWTMAcM8LXbGUldq33OdcdCw= cloud.google.com/go/filestore v1.8.1/go.mod h1:MbN9KcaM47DRTIuLfQhJEsjaocVebNtNQhSLhKCF5GM= cloud.google.com/go/filestore v1.8.3 h1:CpRnsUpMU5gxUKyfh7TD0SM+E+7E4ORaDea2JctKfpY= cloud.google.com/go/filestore v1.8.3/go.mod h1:QTpkYpKBF6jlPRmJwhLqXfJQjVrQisplyb4e2CwfJWc= +cloud.google.com/go/filestore v1.8.5 h1:yAHY3pGq6/IX4sLQqPpfaqfnSk1LmCdVkWNwzIP4X7c= +cloud.google.com/go/filestore v1.8.5/go.mod h1:o8KvHyl5V30kIdrPX6hE+RknscXCUFXWSxYsEWeFfRU= cloud.google.com/go/firestore v1.6.1 h1:8rBq3zRjnHx8UtBvaOWqBB1xq9jH6/wltfQLlTMh2Fw= cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= cloud.google.com/go/firestore v1.14.0 h1:8aLcKnMPoldYU3YHgu4t2exrKhLQkqaXAGqT0ljrFVw= @@ -224,32 +324,46 @@ cloud.google.com/go/functions v1.16.0 h1:IWVylmK5F6hJ3R5zaRW7jI5PrWhCvtBVU4axQLm cloud.google.com/go/functions v1.16.0/go.mod h1:nbNpfAG7SG7Duw/o1iZ6ohvL7mc6MapWQVpqtM29n8k= cloud.google.com/go/functions v1.16.2 h1:83bd2lCgtu2nLbX2jrqsrQhIs7VuVA1N6Op5syeRVIg= cloud.google.com/go/functions v1.16.2/go.mod h1:+gMvV5E3nMb9EPqX6XwRb646jTyVz8q4yk3DD6xxHpg= +cloud.google.com/go/functions v1.16.4 h1:+mNEYegIO1ToQXsWEhEI6cI1lm+VAeu0pAmc+atYOaY= +cloud.google.com/go/functions v1.16.4/go.mod h1:uDp5MbH0kCtXe3uBluq3Zi7bEDuHqcn60mAHxUsNezI= cloud.google.com/go/gaming v1.9.0 h1:7vEhFnZmd931Mo7sZ6pJy7uQPDxF7m7v8xtBheG08tc= cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= cloud.google.com/go/gkebackup v1.3.5 h1:iuE8KNtTsPOc79qeWoNS8zOWoXPD9SAdOmwgxtlCmh8= cloud.google.com/go/gkebackup v1.3.5/go.mod h1:KJ77KkNN7Wm1LdMopOelV6OodM01pMuK2/5Zt1t4Tvc= cloud.google.com/go/gkebackup v1.5.0 h1:wysUXEkggPwENZY3BXroOyWoyVfPypzaqNHgOZD9Kck= cloud.google.com/go/gkebackup v1.5.0/go.mod h1:eLaf/+n8jEmIvOvDriGjo99SN7wRvVadoqzbZu0WzEw= +cloud.google.com/go/gkebackup v1.5.2 h1:sdGeTG6O+JPI7rRiVNy7wO4r4CELChfNe7C8BWPOJRM= +cloud.google.com/go/gkebackup v1.5.2/go.mod h1:ZuWJKacdXtjiO8ry9RrdT57gvcsU7c7/FTqqwjdNUjk= cloud.google.com/go/gkeconnect v0.8.5 h1:17d+ZSSXKqG/RwZCq3oFMIWLPI8Zw3b8+a9/BEVlwH0= cloud.google.com/go/gkeconnect v0.8.5/go.mod h1:LC/rS7+CuJ5fgIbXv8tCD/mdfnlAadTaUufgOkmijuk= cloud.google.com/go/gkeconnect v0.8.7 h1:BfXsTXYs5xlicAlgbtlo8Cw+YdzU3PrlBg7dATJUwrk= cloud.google.com/go/gkeconnect v0.8.7/go.mod h1:iUH1jgQpTyNFMK5LgXEq2o0beIJ2p7KKUUFerkf/eGc= +cloud.google.com/go/gkeconnect v0.8.9 h1:cXA4NWFlB174ub2kIaGLGrKxgTFjDWPzEs766i6Frww= +cloud.google.com/go/gkeconnect v0.8.9/go.mod h1:gl758q5FLXewQZIsxQ7vHyYmLcGBuubvQO6J3yFDh08= cloud.google.com/go/gkehub v0.14.5 h1:RboLNFzf9wEMSo7DrKVBlf+YhK/A/jrLN454L5Tz99Q= cloud.google.com/go/gkehub v0.14.5/go.mod h1:6bzqxM+a+vEH/h8W8ec4OJl4r36laxTs3A/fMNHJ0wA= cloud.google.com/go/gkehub v0.14.7 h1:bHwcvgh8AmcYm6p6/ZrWW3a7J7sKBDtqtsyVXKssnPs= cloud.google.com/go/gkehub v0.14.7/go.mod h1:NLORJVTQeCdxyAjDgUwUp0A6BLEaNLq84mCiulsM4OE= +cloud.google.com/go/gkehub v0.14.9 h1:fWHBKtPwH7Wp5JjNxlPLanYYmXj6XuHjIRk6oa4yqkY= +cloud.google.com/go/gkehub v0.14.9/go.mod h1:W2rDU2n2xgMpf3/BqpT6ffUX/I8yez87rrW/iGRz6Kk= cloud.google.com/go/gkemulticloud v1.1.1 h1:rsSZAGLhyjyE/bE2ToT5fqo1qSW7S+Ubsc9jFOcbhSI= cloud.google.com/go/gkemulticloud v1.1.1/go.mod h1:C+a4vcHlWeEIf45IB5FFR5XGjTeYhF83+AYIpTy4i2Q= cloud.google.com/go/gkemulticloud v1.2.0 h1:zaWBakKPT6mPHVn5iefuRqttjpbNsb8LlMw9KgfyfyU= cloud.google.com/go/gkemulticloud v1.2.0/go.mod h1:iN5wBxTLPR6VTBWpkUsOP2zuPOLqZ/KbgG1bZir1Cng= +cloud.google.com/go/gkemulticloud v1.2.2 h1:Msgg//raevqYlNZ+N8HFfO707wYVCyUnPKQPkt1g288= +cloud.google.com/go/gkemulticloud v1.2.2/go.mod h1:VMsMYDKpUVYNrhese31TVJMVXPLEtFT/AnIarqlcwVo= cloud.google.com/go/grafeas v0.3.4 h1:D4x32R/cHX3MTofKwirz015uEdVk4uAxvZkZCZkOrF4= cloud.google.com/go/grafeas v0.3.4/go.mod h1:A5m316hcG+AulafjAbPKXBO/+I5itU4LOdKO2R/uDIc= cloud.google.com/go/grafeas v0.3.5 h1:Z87HxC4vnjR1kWWtzP6BuQXa6xBmndRK/kaz4iu6oMA= cloud.google.com/go/grafeas v0.3.5/go.mod h1:y54iTBcI+lgUdI+kAPKb8jtPqeTkA2dsYzWSrQtpc5s= +cloud.google.com/go/grafeas v0.3.6 h1:7bcA10EBgTsxeAVypJhz2Dv3fhrdlO7Ml8l7ZZA2IkE= +cloud.google.com/go/grafeas v0.3.6/go.mod h1:to6ECAPgRO2xeqD8ISXHc70nObJuaKZThreQOjeOH3o= cloud.google.com/go/gsuiteaddons v1.6.5 h1:CZEbaBwmbYdhFw21Fwbo+C35HMe36fTE0FBSR4KSfWg= cloud.google.com/go/gsuiteaddons v1.6.5/go.mod h1:Lo4P2IvO8uZ9W+RaC6s1JVxo42vgy+TX5a6hfBZ0ubs= cloud.google.com/go/gsuiteaddons v1.6.7 h1:06Jg3JeLslEfBYX1sDqOPLnF7a3wmhNcDUXF/fVOb50= cloud.google.com/go/gsuiteaddons v1.6.7/go.mod h1:u+sGBvr07OKNnOnQiB/Co1q4U2cjo50ERQwvnlcpNis= +cloud.google.com/go/gsuiteaddons v1.6.9 h1:uezUQ2jCcW4jkvB0tbJkMCNVdIa/qGgqnxEqOF8IvwY= +cloud.google.com/go/gsuiteaddons v1.6.9/go.mod h1:qITZZoLzQhMQ6Re+izKEvz4C+M1AP13S+XuEpS26824= cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= @@ -257,30 +371,44 @@ cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWO cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/iam v1.1.10 h1:ZSAr64oEhQSClwBL670MsJAW5/RLiC6kfw3Bqmd5ZDI= +cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= cloud.google.com/go/iap v1.9.4 h1:94zirc2r4t6KzhAMW0R6Dme005eTP6yf7g6vN4IhRrA= cloud.google.com/go/iap v1.9.4/go.mod h1:vO4mSq0xNf/Pu6E5paORLASBwEmphXEjgCFg7aeNu1w= cloud.google.com/go/iap v1.9.6 h1:rcuRS9XfOgr1v6TAoihVeSXntOnpVhFlVHtPfgOkLAo= cloud.google.com/go/iap v1.9.6/go.mod h1:YiK+tbhDszhaVifvzt2zTEF2ch9duHtp6xzxj9a0sQk= +cloud.google.com/go/iap v1.9.8 h1:oqS5GMxyEDFndqwURKMIaRJ0GXygLJf/2bzue0WkrOU= +cloud.google.com/go/iap v1.9.8/go.mod h1:jQzSbtpYRbBoMdOINr/OqUxBY9rhyqLx04utTCmJ6oo= cloud.google.com/go/ids v1.4.5 h1:xd4U7pgl3GHV+MABnv1BF4/Vy/zBF7CYC8XngkOLzag= cloud.google.com/go/ids v1.4.5/go.mod h1:p0ZnyzjMWxww6d2DvMGnFwCsSxDJM666Iir1bK1UuBo= cloud.google.com/go/ids v1.4.7 h1:wtd+r415yrfZ8LsB6yH6WrOZ26tYt7w6wy3i5a4HQZ8= cloud.google.com/go/ids v1.4.7/go.mod h1:yUkDC71u73lJoTaoONy0dsA0T7foekvg6ZRg9IJL0AA= +cloud.google.com/go/ids v1.4.9 h1:JIYwGad3q7kADDAIMw0E/3OR3vtDqjSliRBlWAm+WNk= +cloud.google.com/go/ids v1.4.9/go.mod h1:1pL+mhlvtUNphwBSK91yO8NoTVQYwOpqim1anIVBwbM= cloud.google.com/go/iot v1.7.5 h1:munTeBlbqI33iuTYgXy7S8lW2TCgi5l1hA4roSIY+EE= cloud.google.com/go/iot v1.7.5/go.mod h1:nq3/sqTz3HGaWJi1xNiX7F41ThOzpud67vwk0YsSsqs= cloud.google.com/go/iot v1.7.7 h1:M9SKIj9eoxoXCzytkLZVAuf5wmoui1OeDqEjC97wRbY= cloud.google.com/go/iot v1.7.7/go.mod h1:tr0bCOSPXtsg64TwwZ/1x+ReTWKlQRVXbM+DnrE54yM= +cloud.google.com/go/iot v1.7.9 h1:dsroR14QUU7i2/GC4AcEv1MvKS0VZCYWWTCxxyq2iYo= +cloud.google.com/go/iot v1.7.9/go.mod h1:1fi6x4CexbygNgRPn+tcxCjOZFTl+4G6Adbo6sLPR7c= cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= cloud.google.com/go/kms v1.17.1 h1:5k0wXqkxL+YcXd4viQzTqCgzzVKKxzgrK+rCZJytEQs= cloud.google.com/go/kms v1.17.1/go.mod h1:DCMnCF/apA6fZk5Cj4XsD979OyHAqFasPuA5Sd0kGlQ= +cloud.google.com/go/kms v1.18.2 h1:EGgD0B9k9tOOkbPhYW1PHo2W0teamAUYMOUIcDRMfPk= +cloud.google.com/go/kms v1.18.2/go.mod h1:YFz1LYrnGsXARuRePL729oINmN5J/5e7nYijgvfiIeY= cloud.google.com/go/language v1.12.3 h1:iaJZg6K4j/2PvZZVcjeO/btcWWIllVRBhuTFjGO4LXs= cloud.google.com/go/language v1.12.3/go.mod h1:evFX9wECX6mksEva8RbRnr/4wi/vKGYnAJrTRXU8+f8= cloud.google.com/go/language v1.12.5 h1:kOYJEcuZgyUX/i/4DFrfXPcrddm1XCQD2lDI5hIFmZQ= cloud.google.com/go/language v1.12.5/go.mod h1:w/6a7+Rhg6Bc2Uzw6thRdKKNjnOzfKTJuxzD0JZZ0nM= +cloud.google.com/go/language v1.12.7 h1:b8Ilb9pBrXj6aMMD0s8EEp28MSiBMo3FWPHAPNImIy4= +cloud.google.com/go/language v1.12.7/go.mod h1:4s/11zABvI/gv+li/+ICe+cErIaN9hYmilf9wrc5Py0= cloud.google.com/go/lifesciences v0.9.5 h1:gXvN70m2p+4zgJFzaz6gMKaxTuF9WJ0USYoMLWAOm8g= cloud.google.com/go/lifesciences v0.9.5/go.mod h1:OdBm0n7C0Osh5yZB7j9BXyrMnTRGBJIZonUMxo5CzPw= cloud.google.com/go/lifesciences v0.9.7 h1:qqEmApr5YFOQjkrU8Jy6o6QpkESqfGbfrE6bnUZZbV8= cloud.google.com/go/lifesciences v0.9.7/go.mod h1:FQ713PhjAOHqUVnuwsCe1KPi9oAdaTfh58h1xPiW13g= +cloud.google.com/go/lifesciences v0.9.9 h1:b9AaxLtWOu9IShII4fdLVDOS03CVCsqWX5zXufyRrDU= +cloud.google.com/go/lifesciences v0.9.9/go.mod h1:4c8eLVKz7/FPw6lvoHx2/JQX1rVM8+LlYmBp8h5H3MQ= cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= cloud.google.com/go/logging v1.10.0 h1:f+ZXMqyrSJ5vZ5pE/zr0xC8y/M9BLNzQeLBwfeZ+wY4= @@ -289,86 +417,127 @@ cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPk cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXmjuEaE= cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= +cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= cloud.google.com/go/managedidentities v1.6.5 h1:+bpih1piZVLxla/XBqeSUzJBp8gv9plGHIMAI7DLpDM= cloud.google.com/go/managedidentities v1.6.5/go.mod h1:fkFI2PwwyRQbjLxlm5bQ8SjtObFMW3ChBGNqaMcgZjI= cloud.google.com/go/managedidentities v1.6.7 h1:uWA9WQyfA0JdkeAFymWUsa3qE9tC33LUElla790Ou1A= cloud.google.com/go/managedidentities v1.6.7/go.mod h1:UzslJgHnc6luoyx2JV19cTCi2Fni/7UtlcLeSYRzTV8= +cloud.google.com/go/managedidentities v1.6.9 h1:ktrpu0TWbtLm2wHUUOxXCftD2e8qZvtQZlFLjKyQXUA= +cloud.google.com/go/managedidentities v1.6.9/go.mod h1:R7+78iH2j/SCTInutWINxGxEY0PH5rpbWt6uRq0Tn+Y= cloud.google.com/go/maps v1.6.4 h1:EVCZAiDvog9So46460BGbCasPhi613exoaQbpilMVlk= cloud.google.com/go/maps v1.6.4/go.mod h1:rhjqRy8NWmDJ53saCfsXQ0LKwBHfi6OSh5wkq6BaMhI= cloud.google.com/go/maps v1.11.0 h1:+//LeUr5ARVau1wVsxLkLnFtYviq9YFS+fB/mhfAOGQ= cloud.google.com/go/maps v1.11.0/go.mod h1:XcSsd8lg4ZhLPCtJ2YHcu/xLVePBzZOlI7GmR2cRCws= +cloud.google.com/go/maps v1.11.3 h1:Un4DDZMLfvQT0kAne82lScQib5QJoBg2NDRVJkBokMg= +cloud.google.com/go/maps v1.11.3/go.mod h1:4iKNrUzFISQ4RoiWCqIFEAAVtgKb2oQ09AVx8GheOUg= cloud.google.com/go/mediatranslation v0.8.5 h1:c76KdIXljQHSCb/Cy47S8H4s05A4zbK3pAFGzwcczZo= cloud.google.com/go/mediatranslation v0.8.5/go.mod h1:y7kTHYIPCIfgyLbKncgqouXJtLsU+26hZhHEEy80fSs= cloud.google.com/go/mediatranslation v0.8.7 h1:izgww3TlyvWyDWdFKnrASpbh12IkAuw8o2ION8sAjX0= cloud.google.com/go/mediatranslation v0.8.7/go.mod h1:6eJbPj1QJwiCP8R4K413qMx6ZHZJUi9QFpApqY88xWU= +cloud.google.com/go/mediatranslation v0.8.9 h1:ptRvYRCZPwEk1oHIlSUg7a74czyS7VUP8869PXeaIT8= +cloud.google.com/go/mediatranslation v0.8.9/go.mod h1:3MjXTUsEzrMC9My6e9o7TOmgIUGlyrkVAxjzcmxBUdU= cloud.google.com/go/memcache v1.10.5 h1:yeDv5qxRedFosvpMSEswrqUsJM5OdWvssPHFliNFTc4= cloud.google.com/go/memcache v1.10.5/go.mod h1:/FcblbNd0FdMsx4natdj+2GWzTq+cjZvMa1I+9QsuMA= cloud.google.com/go/memcache v1.10.7 h1:hE7f3ze3+eWh/EbYXEz7oXkm0LXcr7UCoLklwi7gsLU= cloud.google.com/go/memcache v1.10.7/go.mod h1:SrU6+QBhvXJV0TA59+B3oCHtLkPx37eqdKmRUlmSE1k= +cloud.google.com/go/memcache v1.10.9 h1:Wks0hJCdprJkYn0kYTW5oto3NodsGqn98Urvj3fJgX4= +cloud.google.com/go/memcache v1.10.9/go.mod h1:06evGxt9E1Mf/tYsXJNdXuRj5qzspVd0Tt18kXYDD5c= cloud.google.com/go/metastore v1.13.4 h1:dR7vqWXlK6IYR8Wbu9mdFfwlVjodIBhd1JRrpZftTEg= cloud.google.com/go/metastore v1.13.4/go.mod h1:FMv9bvPInEfX9Ac1cVcRXp8EBBQnBcqH6gz3KvJ9BAE= cloud.google.com/go/metastore v1.13.6 h1:otHcJkci5f/sNRedrSM7eM81QRnu0yZ3HvkvWGphABA= cloud.google.com/go/metastore v1.13.6/go.mod h1:OBCVMCP7X9vA4KKD+5J4Q3d+tiyKxalQZnksQMq5MKY= +cloud.google.com/go/metastore v1.13.8 h1:aGLOZ6tPsGveXVST2c6tf2mjFm5bEcBij8qh4qInz+I= +cloud.google.com/go/metastore v1.13.8/go.mod h1:2uLJBAXn5EDYJx9r7mZtxZifCKpakZUCvNfzI7ejUiE= cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg= cloud.google.com/go/monitoring v1.19.0 h1:NCXf8hfQi+Kmr56QJezXRZ6GPb80ZI7El1XztyUuLQI= cloud.google.com/go/monitoring v1.19.0/go.mod h1:25IeMR5cQ5BoZ8j1eogHE5VPJLlReQ7zFp5OiLgiGZw= +cloud.google.com/go/monitoring v1.20.1 h1:XmM6uk4+mI2ZhWdI2n/2GNhJdpeQN+1VdG2UWEDhX48= +cloud.google.com/go/monitoring v1.20.1/go.mod h1:FYSe/brgfuaXiEzOQFhTjsEsJv+WePyK71X7Y8qo6uQ= cloud.google.com/go/networkconnectivity v1.14.4 h1:GBfXFhLyPspnaBE3nI/BRjdhW8vcbpT9QjE/4kDCDdc= cloud.google.com/go/networkconnectivity v1.14.4/go.mod h1:PU12q++/IMnDJAB+3r+tJtuCXCfwfN+C6Niyj6ji1Po= cloud.google.com/go/networkconnectivity v1.14.6 h1:jYpQ86mZ7OYZc7WadvCIlIaPXmXhr5nD7wgE/ekMVpM= cloud.google.com/go/networkconnectivity v1.14.6/go.mod h1:/azB7+oCSmyBs74Z26EogZ2N3UcXxdCHkCPcz8G32bU= +cloud.google.com/go/networkconnectivity v1.14.8 h1:PSOYigOrl3pTFfRBPQk5uRlxSxn0G1HY7FNZPGz5Quw= +cloud.google.com/go/networkconnectivity v1.14.8/go.mod h1:QQ/XTMk7U5fzv1cVNUCQJEjpkVEE+nYOK7mg3hVTuiI= cloud.google.com/go/networkmanagement v1.9.4 h1:aLV5GcosBNmd6M8+a0ekB0XlLRexv4fvnJJrYnqeBcg= cloud.google.com/go/networkmanagement v1.9.4/go.mod h1:daWJAl0KTFytFL7ar33I6R/oNBH8eEOX/rBNHrC/8TA= cloud.google.com/go/networkmanagement v1.13.2 h1:Ex1/aYkA0areleSmOGXHvEFBGohteIYJr2SGPrjOUe0= cloud.google.com/go/networkmanagement v1.13.2/go.mod h1:24VrV/5HFIOXMEtVQEUoB4m/w8UWvUPAYjfnYZcBc4c= +cloud.google.com/go/networkmanagement v1.13.4 h1:CUX6YYtC6DpV0BzsaovqWExieVPDxmUxvQVlEjf0mwQ= +cloud.google.com/go/networkmanagement v1.13.4/go.mod h1:dGTeJfDPQv0yGDt6gncj4XAPwxktjpCn5ZxQajStW8g= cloud.google.com/go/networksecurity v0.9.5 h1:+caSxBTj0E8OYVh/5wElFdjEMO1S/rZtE1152Cepchc= cloud.google.com/go/networksecurity v0.9.5/go.mod h1:KNkjH/RsylSGyyZ8wXpue8xpCEK+bTtvof8SBfIhMG8= cloud.google.com/go/networksecurity v0.9.7 h1:aepEkfiwOvUL9eu3ginVZhTaXDRHncQKi9lTT1BycH0= cloud.google.com/go/networksecurity v0.9.7/go.mod h1:aB6UiPnh/l32+TRvgTeOxVRVAHAFFqvK+ll3idU5BoY= +cloud.google.com/go/networksecurity v0.9.9 h1:DDqzpqx1u1vDiYW2bBr0r3A5kIw3D5f4RtQkWiRd7Jg= +cloud.google.com/go/networksecurity v0.9.9/go.mod h1:aLS+6sLeZkMhLx9ntTMJG4qWHdvDPctqMOb6ggz9m5s= cloud.google.com/go/notebooks v1.11.3 h1:FH48boYmrWVQ6k0Mx/WrnNafXncT5iSYxA8CNyWTgy0= cloud.google.com/go/notebooks v1.11.3/go.mod h1:0wQyI2dQC3AZyQqWnRsp+yA+kY4gC7ZIVP4Qg3AQcgo= cloud.google.com/go/notebooks v1.11.5 h1:sFU1ETg1HfIN/Tev8gD0dleAITLv7cHp0JClwFmJ6bo= cloud.google.com/go/notebooks v1.11.5/go.mod h1:pz6P8l2TvhWqAW3sysIsS0g2IUJKOzEklsjWJfi8sd4= +cloud.google.com/go/notebooks v1.11.7 h1:/SeTEbFaU3cwzvc0ycM3nJ+8DvSTS8oeOWKi0bzEItM= +cloud.google.com/go/notebooks v1.11.7/go.mod h1:lTjloYceMboZanBFC/JSZYet/K+JuO0mLAXVVhb/6bQ= cloud.google.com/go/optimization v1.6.3 h1:63NZaWyN+5rZEKHPX4ACpw3BjgyeuY8+rCehiCMaGPY= cloud.google.com/go/optimization v1.6.3/go.mod h1:8ve3svp3W6NFcAEFr4SfJxrldzhUl4VMUJmhrqVKtYA= cloud.google.com/go/optimization v1.6.5 h1:FPfowA/LEckKTQT0A4NJMI2bSou999c2ZyFX1zGiYxY= cloud.google.com/go/optimization v1.6.5/go.mod h1:eiJjNge1NqqLYyY75AtIGeQWKO0cvzD1ct/moCFaP2Q= +cloud.google.com/go/optimization v1.6.7 h1:HFaCNq1upokZP4cPelqszhUShkmIypWma5IGe4fh4CA= +cloud.google.com/go/optimization v1.6.7/go.mod h1:FREForRqqjTsJbElYyWSgb54WXUzTMTRyjVT+Tl80v8= cloud.google.com/go/orchestration v1.8.5 h1:YHgWMlrPttIVGItgGfuvO2KM7x+y9ivN/Yk92pMm1a4= cloud.google.com/go/orchestration v1.8.5/go.mod h1:C1J7HesE96Ba8/hZ71ISTV2UAat0bwN+pi85ky38Yq8= cloud.google.com/go/orchestration v1.9.2 h1:C2WL4ZnclXsh4XickGhKYKlPjqVZj35y1sbRjdsZ3g4= cloud.google.com/go/orchestration v1.9.2/go.mod h1:8bGNigqCQb/O1kK7PeStSNlyi58rQvZqDiuXT9KAcbg= +cloud.google.com/go/orchestration v1.9.4 h1:xwqKYWlnDMLETKpZmPg+edCehC7w4G11d/8JSqutC5I= +cloud.google.com/go/orchestration v1.9.4/go.mod h1:jk5hczI8Tciq+WCkN32GpjWJs67GSmAA0XHFUlELJLw= cloud.google.com/go/orgpolicy v1.12.1 h1:2JbXigqBJVp8Dx5dONUttFqewu4fP0p3pgOdIZAhpYU= cloud.google.com/go/orgpolicy v1.12.1/go.mod h1:aibX78RDl5pcK3jA8ysDQCFkVxLj3aOQqrbBaUL2V5I= cloud.google.com/go/orgpolicy v1.12.3 h1:fGftW2bPi8vTjQm57xlwtLBZQcrgC+c3HMFBzJ+KWPc= cloud.google.com/go/orgpolicy v1.12.3/go.mod h1:6BOgIgFjWfJzTsVcib/4QNHOAeOjCdaBj69aJVs//MA= +cloud.google.com/go/orgpolicy v1.12.5 h1:NEbK9U6HuhjXOUI1+fJVdIEh0FHiJtGVq4kYQQ5B8t8= +cloud.google.com/go/orgpolicy v1.12.5/go.mod h1:f778/jOHKp6cP6NbbQgjy4SDfQf6BoVGiSWdxky3ONQ= cloud.google.com/go/osconfig v1.12.5 h1:Mo5jGAxOMKH/PmDY7fgY19yFcVbvwREb5D5zMPQjFfo= cloud.google.com/go/osconfig v1.12.5/go.mod h1:D9QFdxzfjgw3h/+ZaAb5NypM8bhOMqBzgmbhzWViiW8= cloud.google.com/go/osconfig v1.12.7 h1:HXsXGFaFaLTklwKgSob/GSE+c3verYDQDgreFaosxyc= cloud.google.com/go/osconfig v1.12.7/go.mod h1:ID7Lbqr0fiihKMwAOoPomWRqsZYKWxfiuafNZ9j1Y1M= +cloud.google.com/go/osconfig v1.13.0 h1:k+nAmaTcJ08BSR1yGadRZyLwRSvk5XgaZJinS1sEz4Q= +cloud.google.com/go/osconfig v1.13.0/go.mod h1:tlACnQi1rtSLnHRYzfw9SH9zXs0M7S1jqiW2EOCn2Y0= cloud.google.com/go/oslogin v1.13.1 h1:1K4nOT5VEZNt7XkhaTXupBYos5HjzvJMfhvyD2wWdFs= cloud.google.com/go/oslogin v1.13.1/go.mod h1:vS8Sr/jR7QvPWpCjNqy6LYZr5Zs1e8ZGW/KPn9gmhws= cloud.google.com/go/oslogin v1.13.3 h1:7AgOWH1oMPrB1AVU0/f47ADdOt+XfdBY7QRb8tcMUp8= cloud.google.com/go/oslogin v1.13.3/go.mod h1:WW7Rs1OJQ1iSUckZDilvNBSNPE8on740zF+4ZDR4o8U= +cloud.google.com/go/oslogin v1.13.5 h1:IptgM0b9yNJzEbC5rEetbRAcxsuRXDMuSX/65qASvE8= +cloud.google.com/go/oslogin v1.13.5/go.mod h1:V+QzBAbZBZJq9CmTyzKrh3rpMiWIr1OBn6RL4mMVWXI= cloud.google.com/go/phishingprotection v0.8.5 h1:DH3WFLzEoJdW/6xgsmoDqOwT1xddFi7gKu0QGZQhpGU= cloud.google.com/go/phishingprotection v0.8.5/go.mod h1:g1smd68F7mF1hgQPuYn3z8HDbNre8L6Z0b7XMYFmX7I= cloud.google.com/go/phishingprotection v0.8.7 h1:CbCjfR/pgDHyRMu94o9nuGwaONEcarWnUfSGGw+I2ZI= cloud.google.com/go/phishingprotection v0.8.7/go.mod h1:FtYaOyGc/HQQU7wY4sfwYZBFDKAL+YtVBjUj8E3A3/I= +cloud.google.com/go/phishingprotection v0.8.9 h1:Gg3XeqWW0g97MKvexeMytrxu31UHDjUd0bbzHa40D8o= +cloud.google.com/go/phishingprotection v0.8.9/go.mod h1:xNojFKIdq+hNGNpOZOEGVGA4Mdhm2yByMli2Ni/RV0w= cloud.google.com/go/policytroubleshooter v1.10.3 h1:c0WOzC6hz964QWNBkyKfna8A2jOIx1zzZa43Gx/P09o= cloud.google.com/go/policytroubleshooter v1.10.3/go.mod h1:+ZqG3agHT7WPb4EBIRqUv4OyIwRTZvsVDHZ8GlZaoxk= cloud.google.com/go/policytroubleshooter v1.10.5 h1:LGt85MZUKlq9oqsbBL9+M6jAyeuR1TtCx6k5HfAQxTY= cloud.google.com/go/policytroubleshooter v1.10.5/go.mod h1:bpOf94YxjWUqsVKokzPBibMSAx937Jp2UNGVoMAtGYI= +cloud.google.com/go/policytroubleshooter v1.10.7 h1:A3KZBrc2Qzq5jQI8M8hW4GscOBZzIvoOhwRiE41pqcY= +cloud.google.com/go/policytroubleshooter v1.10.7/go.mod h1:/JxxZOSCT8nASvH/SP4Bj81EnDFwZhFThG7mgVWIoPY= cloud.google.com/go/privatecatalog v0.9.5 h1:UZ0assTnATXSggoxUIh61RjTQ4P9zCMk/kEMbn0nMYA= cloud.google.com/go/privatecatalog v0.9.5/go.mod h1:fVWeBOVe7uj2n3kWRGlUQqR/pOd450J9yZoOECcQqJk= cloud.google.com/go/privatecatalog v0.9.7 h1:wGZKKJhYyuf4gcAEywQqQ6F19yxhBJGnzgyxOTbJjBw= cloud.google.com/go/privatecatalog v0.9.7/go.mod h1:NWLa8MCL6NkRSt8jhL8Goy2A/oHkvkeAxiA0gv0rIXI= +cloud.google.com/go/privatecatalog v0.9.9 h1:fV9+FuZuN6pup4h3qh/0HGpssJrkI5EyZVLQEEvzrA4= +cloud.google.com/go/privatecatalog v0.9.9/go.mod h1:attFfOEf8ECrCuCdT3WYY8wyMKRZt4iB1bEWYFzPn50= cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= cloud.google.com/go/pubsub v1.36.1 h1:dfEPuGCHGbWUhaMCTHUFjfroILEkx55iUmKBZTP5f+Y= cloud.google.com/go/pubsub v1.36.1/go.mod h1:iYjCa9EzWOoBiTdd4ps7QoMtMln5NwaZQpK1hbRfBDE= cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc= cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA= +cloud.google.com/go/pubsub v1.40.0 h1:0LdP+zj5XaPAGtWr2V6r88VXJlmtaB/+fde1q3TU8M0= +cloud.google.com/go/pubsub v1.40.0/go.mod h1:BVJI4sI2FyXp36KFKvFwcfDRDfR8MiLT8mMhmIhdAeA= cloud.google.com/go/pubsublite v1.8.1 h1:pX+idpWMIH30/K7c0epN6V703xpIcMXWRjKJsz0tYGY= cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= cloud.google.com/go/pubsublite v1.8.2 h1:jLQozsEVr+c6tOU13vDugtnaBSUy/PD5zK6mhm+uF1Y= @@ -377,66 +546,98 @@ cloud.google.com/go/recaptchaenterprise/v2 v2.9.2 h1:U3Wfq12X9cVMuTpsWDSURnXF0Z9 cloud.google.com/go/recaptchaenterprise/v2 v2.9.2/go.mod h1:trwwGkfhCmp05Ll5MSJPXY7yvnO0p4v3orGANAFHAuU= cloud.google.com/go/recaptchaenterprise/v2 v2.13.0 h1:+QG02kE63W13vXI+rwAxFF3EhGX6K7gXwFz9OKwKcHw= cloud.google.com/go/recaptchaenterprise/v2 v2.13.0/go.mod h1:jNYyn2ScR4DTg+VNhjhv/vJQdaU8qz+NpmpIzEE7HFQ= +cloud.google.com/go/recaptchaenterprise/v2 v2.14.0 h1:revhoyewcQrpKccogfKNO2ul3aQbD11BU+ZsRpOWlgw= +cloud.google.com/go/recaptchaenterprise/v2 v2.14.0/go.mod h1:pwC/eCyXq37YV3NSaiJsfOmuoTDkzURnVKAWGSkjDUY= cloud.google.com/go/recommendationengine v0.8.5 h1:ineqLswaCSBY0csYv5/wuXJMBlxATK6Xc5jJkpiTEdM= cloud.google.com/go/recommendationengine v0.8.5/go.mod h1:A38rIXHGFvoPvmy6pZLozr0g59NRNREz4cx7F58HAsQ= cloud.google.com/go/recommendationengine v0.8.7 h1:N6n/TEr0FQzeP4ZtvF5daMszOhdZI94uMiPiAi9kFMo= cloud.google.com/go/recommendationengine v0.8.7/go.mod h1:YsUIbweUcpm46OzpVEsV5/z+kjuV6GzMxl7OAKIGgKE= +cloud.google.com/go/recommendationengine v0.8.9 h1:jewIoRtf1F4WgtIDdPEKDqpvPU+utN02sFw3iYbmvwM= +cloud.google.com/go/recommendationengine v0.8.9/go.mod h1:QgE5f6s20QhCXf4UR9KMI/Q6Spykd2zEYXX2oBz6Cbs= cloud.google.com/go/recommender v1.12.1 h1:LVLYS3r3u0MSCxQSDUtLSkporEGi9OAE6hGvayrZNPs= cloud.google.com/go/recommender v1.12.1/go.mod h1:gf95SInWNND5aPas3yjwl0I572dtudMhMIG4ni8nr+0= cloud.google.com/go/recommender v1.12.3 h1:v9x75vXP5wMXw3QG3xmgjVHLlqYufuLn/ht3oNWCA3w= cloud.google.com/go/recommender v1.12.3/go.mod h1:OgN0MjV7/6FZUUPgF2QPQtYErtZdZc4u+5onvurcGEI= +cloud.google.com/go/recommender v1.12.5 h1:91NMrObmes2zA+gI0+QhCFH1oTPHlMGFTAJy5MTD2eg= +cloud.google.com/go/recommender v1.12.5/go.mod h1:ggh5JNuG5ajpRqqcEkgni/DjpS7x12ktO+Edu8bmCJM= cloud.google.com/go/redis v1.14.2 h1:QF0maEdVv0Fj/2roU8sX3NpiDBzP9ICYTO+5F32gQNo= cloud.google.com/go/redis v1.14.2/go.mod h1:g0Lu7RRRz46ENdFKQ2EcQZBAJ2PtJHJLuiiRuEXwyQw= cloud.google.com/go/redis v1.16.0 h1:1veL/h/x5bgzG2CLK2cdG3plWdgO0p1qoHgwFBqG7+c= cloud.google.com/go/redis v1.16.0/go.mod h1:NLzG3Ur8ykVIZk+i5ienRnycsvWzQ0uCLcil6Htc544= +cloud.google.com/go/redis v1.16.2 h1:QbarPMu22tuUOqi3ynNKk2mQWl7xitMTxAaAUaBUFsE= +cloud.google.com/go/redis v1.16.2/go.mod h1:bn/4nXSZkoH4QTXRjqWR2AZ0WA1b13ct354nul2SSiU= cloud.google.com/go/resourcemanager v1.9.5 h1:AZWr1vWVDKGwfLsVhcN+vcwOz3xqqYxtmMa0aABCMms= cloud.google.com/go/resourcemanager v1.9.5/go.mod h1:hep6KjelHA+ToEjOfO3garMKi/CLYwTqeAw7YiEI9x8= cloud.google.com/go/resourcemanager v1.9.7 h1:SdvD0PaPX60+yeKoSe16mawFpM0EPuiPPihTIVlhRsY= cloud.google.com/go/resourcemanager v1.9.7/go.mod h1:cQH6lJwESufxEu6KepsoNAsjrUtYYNXRwxm4QFE5g8A= +cloud.google.com/go/resourcemanager v1.9.9 h1:9JgRo4uBdCLJpWb6c+1+q7QPyWzH0LSCKUcF/IliKNk= +cloud.google.com/go/resourcemanager v1.9.9/go.mod h1:vCBRKurJv+XVvRZ0XFhI/eBrBM7uBOPFjMEwSDMIflY= cloud.google.com/go/resourcesettings v1.6.5 h1:BTr5MVykJwClASci/7Og4Qfx70aQ4n3epsNLj94ZYgw= cloud.google.com/go/resourcesettings v1.6.5/go.mod h1:WBOIWZraXZOGAgoR4ukNj0o0HiSMO62H9RpFi9WjP9I= cloud.google.com/go/resourcesettings v1.7.0 h1:yEuByg5XBHhTG9wPEU7GiEtC9Orp1wSEyiiX4IPqoSY= cloud.google.com/go/resourcesettings v1.7.0/go.mod h1:pFzZYOQMyf1hco9pbNWGEms6N/2E7nwh0oVU1Tz+4qA= +cloud.google.com/go/resourcesettings v1.7.2 h1:Q3udMNHhYLrzVNrCYEpZ6f70Rf6nHpiPFay1ILwcQ80= +cloud.google.com/go/resourcesettings v1.7.2/go.mod h1:mNdB5Wl9/oVr9Da3OrEstSyXCT949ignvO6ZrmYdmGU= cloud.google.com/go/retail v1.16.0 h1:Fn1GuAua1c6crCGqfJ1qMxG1Xh10Tg/x5EUODEHMqkw= cloud.google.com/go/retail v1.16.0/go.mod h1:LW7tllVveZo4ReWt68VnldZFWJRzsh9np+01J9dYWzE= cloud.google.com/go/retail v1.16.2 h1:msP5a8BOxVym2DvoubeWAxAeV6VhYkKnYHc2XOkd/+U= cloud.google.com/go/retail v1.16.2/go.mod h1:T7UcBh4/eoxRBpP3vwZCoa+PYA9/qWRTmOCsV8DRdZ0= +cloud.google.com/go/retail v1.17.2 h1:RovE7VK3TEFDECBXwVWItL21+QQ4WY6otLCZHqExMBQ= +cloud.google.com/go/retail v1.17.2/go.mod h1:Ad6D8tkDZatI1X7szhhYWiatZmH6nSUfZ3WeCECyA0E= cloud.google.com/go/run v1.3.4 h1:m9WDA7DzTpczhZggwYlZcBWgCRb+kgSIisWn1sbw2rQ= cloud.google.com/go/run v1.3.4/go.mod h1:FGieuZvQ3tj1e9GnzXqrMABSuir38AJg5xhiYq+SF3o= cloud.google.com/go/run v1.3.7 h1:E4Z5e681Qh7UJrJRMCgYhp+3tkcoXiaKGh3UZmUPaAQ= cloud.google.com/go/run v1.3.7/go.mod h1:iEUflDx4Js+wK0NzF5o7hE9Dj7QqJKnRj0/b6rhVq20= +cloud.google.com/go/run v1.3.9 h1:De6XlIBjzEFXPzDQ/hJgvieh4H/105mhkkwxL5DmH0o= +cloud.google.com/go/run v1.3.9/go.mod h1:Ep/xsiUt5ZOwNptGl1FBlHb+asAgqB+9RDJKBa/c1mI= cloud.google.com/go/scheduler v1.10.6 h1:5U8iXLoQ03qOB+ZXlAecU7fiE33+u3QiM9nh4cd0eTE= cloud.google.com/go/scheduler v1.10.6/go.mod h1:pe2pNCtJ+R01E06XCDOJs1XvAMbv28ZsQEbqknxGOuE= cloud.google.com/go/scheduler v1.10.8 h1:Jn/unfNUgRiNJRc1nrApzimKiVj91UYlLT8mMfpUu48= cloud.google.com/go/scheduler v1.10.8/go.mod h1:0YXHjROF1f5qTMvGTm4o7GH1PGAcmu/H/7J7cHOiHl0= +cloud.google.com/go/scheduler v1.10.10 h1:KYdENFZip7O2Jk/zuNzEPIv+ZQokkWnNZ5AnrIuooYo= +cloud.google.com/go/scheduler v1.10.10/go.mod h1:nOLkchaee8EY0g73hpv613pfnrZwn/dU2URYjJbRLR0= cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= cloud.google.com/go/secretmanager v1.13.1 h1:TTGo2Vz7ZxYn2QbmuFP7Zo4lDm5VsbzBjDReo3SA5h4= cloud.google.com/go/secretmanager v1.13.1/go.mod h1:y9Ioh7EHp1aqEKGYXk3BOC+vkhlHm9ujL7bURT4oI/4= +cloud.google.com/go/secretmanager v1.13.3 h1:VqUVYY3U6uFXOhPdZgAoZH9m8E6p7eK02TsDRj2SBf4= +cloud.google.com/go/secretmanager v1.13.3/go.mod h1:e45+CxK0w6GaL4hS+KabgQskl4RdSS30b+HRf0TH0kk= cloud.google.com/go/security v1.15.5 h1:wTKJQ10j8EYgvE8Y+KhovxDRVDk2iv/OsxZ6GrLP3kE= cloud.google.com/go/security v1.15.5/go.mod h1:KS6X2eG3ynWjqcIX976fuToN5juVkF6Ra6c7MPnldtc= cloud.google.com/go/security v1.17.0 h1:u4RCnEQPvlrrnFRFinU0T3WsjtrsQErkWBfqTM5oUQI= cloud.google.com/go/security v1.17.0/go.mod h1:eSuFs0SlBv1gWg7gHIoF0hYOvcSwJCek/GFXtgO6aA0= +cloud.google.com/go/security v1.17.2 h1:pEkUeR1PFNwoFAIXPMa4PBCYb75UT8LmNfjQy1fm/Co= +cloud.google.com/go/security v1.17.2/go.mod h1:6eqX/AgDw56KwguEBfFNiNQ+Vzi+V6+GopklexYuJ0U= cloud.google.com/go/securitycenter v1.24.4 h1:/5jjkZ+uGe8hZ7pvd7pO30VW/a+pT2MrrdgOqjyucKQ= cloud.google.com/go/securitycenter v1.24.4/go.mod h1:PSccin+o1EMYKcFQzz9HMMnZ2r9+7jbc+LvPjXhpwcU= cloud.google.com/go/securitycenter v1.30.0 h1:Y8C0I/mzLbaxAl5cw3EaLox0Rvpy+VUwEuCGWIQDMU8= cloud.google.com/go/securitycenter v1.30.0/go.mod h1:/tmosjS/dfTnzJxOzZhTXdX3MXWsCmPWfcYOgkJmaJk= +cloud.google.com/go/securitycenter v1.32.0 h1:UJvalA9NoLhU0DWLa10qMSvMucEe+iQOqxC4/KGqMys= +cloud.google.com/go/securitycenter v1.32.0/go.mod h1:s1dN6hM6HZyzUyJrqBoGvhxR/GecT5u48sidMIgDxTo= cloud.google.com/go/servicedirectory v1.11.4 h1:da7HFI1229kyzIyuVEzHXip0cw0d+E0s8mjQby0WN+k= cloud.google.com/go/servicedirectory v1.11.4/go.mod h1:Bz2T9t+/Ehg6x+Y7Ycq5xiShYLD96NfEsWNHyitj1qM= cloud.google.com/go/servicedirectory v1.11.7 h1:c3OAhTcZ8LbIiKps5T3p6i0QcPI8/aWYwOfoZobICKo= cloud.google.com/go/servicedirectory v1.11.7/go.mod h1:fiO/tM0jBpVhpCAe7Yp5HmEsmxSUcOoc4vPrO02v68I= +cloud.google.com/go/servicedirectory v1.11.9 h1:KivmF5S9i6av+7tgkHgcosC51jEtmC9UvgayezP2Uqo= +cloud.google.com/go/servicedirectory v1.11.9/go.mod h1:qiDNuIS2qxuuroSmPNuXWxoFMvsEudKXP62Wos24BsU= cloud.google.com/go/shell v1.7.5 h1:3Fq2hzO0ZSyaqBboJrFkwwf/qMufDtqwwA6ep8EZxEI= cloud.google.com/go/shell v1.7.5/go.mod h1:hL2++7F47/IfpfTO53KYf1EC+F56k3ThfNEXd4zcuiE= cloud.google.com/go/shell v1.7.7 h1:HxCzcUxSsCh6FJWkmbOUrGI1sKe4E1Yy4vaykn4RhJ4= cloud.google.com/go/shell v1.7.7/go.mod h1:7OYaMm3TFMSZBh8+QYw6Qef+fdklp7CjjpxYAoJpZbQ= +cloud.google.com/go/shell v1.7.9 h1:CPn8dHSJgZsIaMtGw5iMoF/6Ab7l5A2g34CIjVxlU3c= +cloud.google.com/go/shell v1.7.9/go.mod h1:h3wVC6qaQ1nIlSWMasl1e/uwmepVbZpjSk/Bn7ZafSc= cloud.google.com/go/spanner v1.56.0 h1:o/Cv7/zZ1WgRXVCd5g3Nc23ZI39p/1pWFqFwvg6Wcu8= cloud.google.com/go/spanner v1.56.0/go.mod h1:DndqtUKQAt3VLuV2Le+9Y3WTnq5cNKrnLb/Piqcj+h0= cloud.google.com/go/spanner v1.63.0 h1:P6+BY70Wtol4MtryBgnXZVTZfsdySEvWfz0EpyLwHi4= cloud.google.com/go/spanner v1.63.0/go.mod h1:iqDx7urZpgD7RekZ+CFvBRH6kVTW1ZSEb2HMDKOp5Cc= +cloud.google.com/go/spanner v1.64.0 h1:ltyPbHA/nRAtAhU/o742dXBCI1eNHPeaRY09Ja8B+hM= +cloud.google.com/go/spanner v1.64.0/go.mod h1:TOFx3pb2UwPsDGlE1gTehW+y6YlU4IFk+VdDHSGQS/M= cloud.google.com/go/speech v1.21.1 h1:nuFc+Kj5B8de75nN4FdPyUbI2SiBoHZG6BLurXL56Q0= cloud.google.com/go/speech v1.21.1/go.mod h1:E5GHZXYQlkqWQwY5xRSLHw2ci5NMQNG52FfMU1aZrIA= cloud.google.com/go/speech v1.23.1 h1:TcWEAOLQH1Lb2fhHS6/GjvAh+ue0dt4xUDHXHG6vF04= cloud.google.com/go/speech v1.23.1/go.mod h1:UNgzNxhNBuo/OxpF1rMhA/U2rdai7ILL6PBXFs70wq0= +cloud.google.com/go/speech v1.23.3 h1:zuiX3ExV9jv1rrTFFyYZF5DvYys0/JByeErC50Hyw+g= +cloud.google.com/go/speech v1.23.3/go.mod h1:u7tK/jxhzRZwZ5Nujhau7iLI3+VfJKYhpoZTjU7hRsE= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= @@ -447,66 +648,98 @@ cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkp cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= +cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= cloud.google.com/go/storagetransfer v1.10.4 h1:dy4fL3wO0VABvzM05ycMUPFHxTPbJz9Em8ikAJVqSbI= cloud.google.com/go/storagetransfer v1.10.4/go.mod h1:vef30rZKu5HSEf/x1tK3WfWrL0XVoUQN/EPDRGPzjZs= cloud.google.com/go/storagetransfer v1.10.6 h1:CXmoNEvz7y2NtHFZuH3Z8ASN43rxRINWa2Q/IlBzM2k= cloud.google.com/go/storagetransfer v1.10.6/go.mod h1:3sAgY1bx1TpIzfSzdvNGHrGYldeCTyGI/Rzk6Lc6A7w= +cloud.google.com/go/storagetransfer v1.10.8 h1:hFCYNbls3DoAA49BZ8bWfmdUPfwLa708h1F6gPy76OE= +cloud.google.com/go/storagetransfer v1.10.8/go.mod h1:fEGWYffkV9OYOKms8nxyJWIZA7iEWPl2Mybk6bpQnEk= cloud.google.com/go/talent v1.6.6 h1:JssV0CE3FNujuSWn7SkosOzg7qrMxVnt6txOfGcMSa4= cloud.google.com/go/talent v1.6.6/go.mod h1:y/WQDKrhVz12WagoarpAIyKKMeKGKHWPoReZ0g8tseQ= cloud.google.com/go/talent v1.6.8 h1:RoyEtftfJrbwJcu63zuWE4IjC76xMyVsJBhmleIp3bE= cloud.google.com/go/talent v1.6.8/go.mod h1:kqPAJvhxmhoUTuqxjjk2KqA8zUEeTDmH+qKztVubGlQ= +cloud.google.com/go/talent v1.6.10 h1:Zc1FO2NTLjCNztqnyll7DwKobFYomyCijRlqbJj+7mc= +cloud.google.com/go/talent v1.6.10/go.mod h1:q2/qIb2Eb2svmeBfkCGIia/NGmkcScdyYSyNNOgFRLI= cloud.google.com/go/texttospeech v1.7.5 h1:dxY2Q5mHCbrGa3oPR2O3PCicdnvKa1JmwGQK36EFLOw= cloud.google.com/go/texttospeech v1.7.5/go.mod h1:tzpCuNWPwrNJnEa4Pu5taALuZL4QRRLcb+K9pbhXT6M= cloud.google.com/go/texttospeech v1.7.7 h1:qR6Mu+EM2OfaZR1/Rl8BDBTVfi2X5OtwKKvJRSQyG+o= cloud.google.com/go/texttospeech v1.7.7/go.mod h1:XO4Wr2VzWHjzQpMe3gS58Oj68nmtXMyuuH+4t0wy9eA= +cloud.google.com/go/texttospeech v1.7.9 h1:wn9UNRlEw+vCDFd2NBVPrNGFwB+n/cV20i81MBlbwas= +cloud.google.com/go/texttospeech v1.7.9/go.mod h1:nuo7l7CVWUMvaTgswbn/hhn2Tv73/WbenqGyc236xpo= cloud.google.com/go/tpu v1.6.5 h1:C8YyYda8WtNdBoCgFwwBzZd+S6+EScHOxM/z1h0NNp8= cloud.google.com/go/tpu v1.6.5/go.mod h1:P9DFOEBIBhuEcZhXi+wPoVy/cji+0ICFi4TtTkMHSSs= cloud.google.com/go/tpu v1.6.7 h1:ngQokxUB1z2gvHn3vAf04m7SFnNYMiQIIpny81fCGAs= cloud.google.com/go/tpu v1.6.7/go.mod h1:o8qxg7/Jgt7TCgZc3jNkd4kTsDwuYD3c4JTMqXZ36hU= +cloud.google.com/go/tpu v1.6.9 h1:e6TbpIGmKdFFjW/OH8uQl0U0+t0K4TVN5mO2C+zBBtQ= +cloud.google.com/go/tpu v1.6.9/go.mod h1:6C7Ed7Le5Y1vWGR+8lQWsh/gmqK6l53lgji0YXBU40o= cloud.google.com/go/trace v1.10.5 h1:0pr4lIKJ5XZFYD9GtxXEWr0KkVeigc3wlGpZco0X1oA= cloud.google.com/go/trace v1.10.5/go.mod h1:9hjCV1nGBCtXbAE4YK7OqJ8pmPYSxPA0I67JwRd5s3M= cloud.google.com/go/trace v1.10.7 h1:gK8z2BIJQ3KIYGddw9RJLne5Fx0FEXkrEQzPaeEYVvk= cloud.google.com/go/trace v1.10.7/go.mod h1:qk3eiKmZX0ar2dzIJN/3QhY2PIFh1eqcIdaN5uEjQPM= +cloud.google.com/go/trace v1.10.9 h1:Cy6D1Zdz8up4mIPUWModTuIGDr3fh7AZaCnR+uyxpgA= +cloud.google.com/go/trace v1.10.9/go.mod h1:vtWRnvEh+d8h2xljwxVwsdxxpoWZkxcNYnJF3FuJUV8= cloud.google.com/go/translate v1.10.1 h1:upovZ0wRMdzZvXnu+RPam41B0mRJ+coRXFP2cYFJ7ew= cloud.google.com/go/translate v1.10.1/go.mod h1:adGZcQNom/3ogU65N9UXHOnnSvjPwA/jKQUMnsYXOyk= cloud.google.com/go/translate v1.10.3 h1:g+B29z4gtRGsiKDoTF+bNeH25bLRokAaElygX2FcZkE= cloud.google.com/go/translate v1.10.3/go.mod h1:GW0vC1qvPtd3pgtypCv4k4U8B7EdgK9/QEF2aJEUovs= +cloud.google.com/go/translate v1.10.5 h1:HGFw8dhEp6xYCDWG5fRNwZHfY6MiyCh97RHBBkzsuNM= +cloud.google.com/go/translate v1.10.5/go.mod h1:n9fFca4U/EKr2GzJKrnQXemlYhfo1mT1nSt7Rt4l/VA= cloud.google.com/go/video v1.20.4 h1:TXwotxkShP1OqgKsbd+b8N5hrIHavSyLGvYnLGCZ7xc= cloud.google.com/go/video v1.20.4/go.mod h1:LyUVjyW+Bwj7dh3UJnUGZfyqjEto9DnrvTe1f/+QrW0= cloud.google.com/go/video v1.21.0 h1:ue/1C8TF8H2TMzKMBdNnFxT7QaeWMtqfDr9TSQGgUhA= cloud.google.com/go/video v1.21.0/go.mod h1:Kqh97xHXZ/bIClgDHf5zkKvU3cvYnLyRefmC8yCBqKI= +cloud.google.com/go/video v1.21.2 h1:f/Ez6k2aeN+1+XoAaFCTTqOD+oq8c38fHDi8vd9D3tg= +cloud.google.com/go/video v1.21.2/go.mod h1:UNXGQj3Hdyb70uaF9JeeM8Y8BAmAzLEMSWmyBKY2iVM= cloud.google.com/go/videointelligence v1.11.5 h1:mYaWH8uhUCXLJCN3gdXswKzRa2+lK0zN6/KsIubm6pE= cloud.google.com/go/videointelligence v1.11.5/go.mod h1:/PkeQjpRponmOerPeJxNPuxvi12HlW7Em0lJO14FC3I= cloud.google.com/go/videointelligence v1.11.7 h1:SKBkFTuOclESLjQL1LwraqVFm2fL5oL9tbzKITU+FOY= cloud.google.com/go/videointelligence v1.11.7/go.mod h1:iMCXbfjurmBVgKuyLedTzv90kcnppOJ6ttb0+rLDID0= +cloud.google.com/go/videointelligence v1.11.9 h1:fGlVXtrk3mIh2DFIggTQ4xoA2VruiTkXZHCl6IDY0Bk= +cloud.google.com/go/videointelligence v1.11.9/go.mod h1:Mv0dgb6U12BfBRPj39nM/7gcAFS1+VVGpTiyMJ/ShPo= cloud.google.com/go/vision/v2 v2.8.0 h1:W52z1b6LdGI66MVhE70g/NFty9zCYYcjdKuycqmlhtg= cloud.google.com/go/vision/v2 v2.8.0/go.mod h1:ocqDiA2j97pvgogdyhoxiQp2ZkDCyr0HWpicywGGRhU= cloud.google.com/go/vision/v2 v2.8.2 h1:j9RxG8DcyJO/D7/ps2pOey8VZys+TMqF79bWAhuM7QU= cloud.google.com/go/vision/v2 v2.8.2/go.mod h1:BHZA1LC7dcHjSr9U9OVhxMtLKd5l2jKPzLRALEJvuaw= +cloud.google.com/go/vision/v2 v2.8.4 h1:kBZ62LquS8V8u+N8wWTLgn2tHqaC4poQuGjRaaR+WGE= +cloud.google.com/go/vision/v2 v2.8.4/go.mod h1:qlmeVbmCfPNuD1Kwa7/evqCJYoJ7WhiZ2XeVSYwiOaA= cloud.google.com/go/vmmigration v1.7.5 h1:5v9RT2vWyuw3pK2ox0HQpkoftO7Q7/8591dTxxQc79g= cloud.google.com/go/vmmigration v1.7.5/go.mod h1:pkvO6huVnVWzkFioxSghZxIGcsstDvYiVCxQ9ZH3eYI= cloud.google.com/go/vmmigration v1.7.7 h1:bf2qKqEN7iqT62IptQ/FDadoDLJI9sthyrW3PVaH8bY= cloud.google.com/go/vmmigration v1.7.7/go.mod h1:qYIK5caZY3IDMXQK+A09dy81QU8qBW0/JDTc39OaKRw= +cloud.google.com/go/vmmigration v1.7.9 h1:+X5Frseyehz8ZvnVSRZYXAwEEQXjS4oKK4EV/0KbS9s= +cloud.google.com/go/vmmigration v1.7.9/go.mod h1:x5LQyAESUXsI7/QAQY6BV8xEjIrlkGI+S+oau/Sb0Gs= cloud.google.com/go/vmwareengine v1.1.1 h1:EGdDi9QbqThfZq3ILcDK5g+m9jTevc34AY5tACx5v7k= cloud.google.com/go/vmwareengine v1.1.1/go.mod h1:nMpdsIVkUrSaX8UvmnBhzVzG7PPvNYc5BszcvIVudYs= cloud.google.com/go/vmwareengine v1.1.3 h1:x4KwHB4JlBEzMaITVhrbbpHrU+2I5LrlvHGEEluT0vc= cloud.google.com/go/vmwareengine v1.1.3/go.mod h1:UoyF6LTdrIJRvDN8uUB8d0yimP5A5Ehkr1SRzL1APZw= +cloud.google.com/go/vmwareengine v1.1.5 h1:tzqTbh5CAqZDVJrEgbRGDFgPyCx5bjIPH5Cm0xqVamA= +cloud.google.com/go/vmwareengine v1.1.5/go.mod h1:Js6QbSeC1OgpyygalCrMj90wa93O3kFgcs/u1YzCKsU= cloud.google.com/go/vpcaccess v1.7.5 h1:XyL6hTLtEM/eE4F1GEge8xUN9ZCkiVWn44K/YA7z1rQ= cloud.google.com/go/vpcaccess v1.7.5/go.mod h1:slc5ZRvvjP78c2dnL7m4l4R9GwL3wDLcpIWz6P/ziig= cloud.google.com/go/vpcaccess v1.7.7 h1:F5woMLufKnshmDvPVxCzoC+Di12RYXQ1W8kNmpBT8z0= cloud.google.com/go/vpcaccess v1.7.7/go.mod h1:EzfSlgkoAnFWEMznZW0dVNvdjFjEW97vFlKk4VNBhwY= +cloud.google.com/go/vpcaccess v1.7.9 h1:LbQaXRQMTPCPmJKoVIW/2vvj80FCiGG+lAyOzNpKs6M= +cloud.google.com/go/vpcaccess v1.7.9/go.mod h1:Y0BlcnG9yTkoM6IL6auBeKvVEXL4LmNIxzscekrn/uk= cloud.google.com/go/webrisk v1.9.5 h1:251MvGuC8wisNN7+jqu9DDDZAi38KiMXxOpA/EWy4dE= cloud.google.com/go/webrisk v1.9.5/go.mod h1:aako0Fzep1Q714cPEM5E+mtYX8/jsfegAuS8aivxy3U= cloud.google.com/go/webrisk v1.9.7 h1:EWTSVagWWeQjVAsebiF/wJMwC5bq6Zz3LqOmD9Uid4s= cloud.google.com/go/webrisk v1.9.7/go.mod h1:7FkQtqcKLeNwXCdhthdXHIQNcFWPF/OubrlyRcLHNuQ= +cloud.google.com/go/webrisk v1.9.9 h1:WmSWTAIpQEKscbnbVUeWWdq+p11Q8P1Gn6ADI8yAQCI= +cloud.google.com/go/webrisk v1.9.9/go.mod h1:Wre67XdNQbt0LCBrvwVNBS5ORb8ssixq/u04CCZoO+k= cloud.google.com/go/websecurityscanner v1.6.5 h1:YqWZrZYabG88TZt7364XWRJGhxmxhony2ZUyZEYMF2k= cloud.google.com/go/websecurityscanner v1.6.5/go.mod h1:QR+DWaxAz2pWooylsBF854/Ijvuoa3FCyS1zBa1rAVQ= cloud.google.com/go/websecurityscanner v1.6.7 h1:R5OW5SNRqD0DSEmyWLUMNYAXWYnz/NLSXBawVFrc9a0= cloud.google.com/go/websecurityscanner v1.6.7/go.mod h1:EpiW84G5KXxsjtFKK7fSMQNt8JcuLA8tQp7j0cyV458= +cloud.google.com/go/websecurityscanner v1.6.9 h1:4tbX6llT8kBqUJbpB4Wjj9sqWNYwCUGt3WP6uVVv00w= +cloud.google.com/go/websecurityscanner v1.6.9/go.mod h1:xrMxPiHB5iFxvc2tqbfUr6inPox6q6y7Wg0LTyZOKTw= cloud.google.com/go/workflows v1.12.4 h1:uHNmUiatTbPQ4H1pabwfzpfEYD4BBnqDHqMm2IesOh4= cloud.google.com/go/workflows v1.12.4/go.mod h1:yQ7HUqOkdJK4duVtMeBCAOPiN1ZF1E9pAMX51vpwB/w= cloud.google.com/go/workflows v1.12.6 h1:2bE69mh68law1UZWPjgmvOQsjsGSppRudABAXwNAy58= cloud.google.com/go/workflows v1.12.6/go.mod h1:oDbEHKa4otYg4abwdw2Z094jB0TLLiFGAPA78EDAKag= +cloud.google.com/go/workflows v1.12.8 h1:n5SOGamA/HtlpWAIXxKXpuGq1ta3wDpyOftDgjIcNHU= +cloud.google.com/go/workflows v1.12.8/go.mod h1:b7akG38W6lHmyPc+WYJxIYl1rEv79bBMYVwEZmp3aJQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= @@ -544,6 +777,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= @@ -556,6 +791,8 @@ github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -581,8 +818,6 @@ github.com/fasthttp/websocket v1.4.3-rc.6 h1:omHqsl8j+KXpmzRjF8bmzOSYJ8GnS0E3efi github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CACNqNvCqQCoty/Lc= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= @@ -593,6 +828,7 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -602,6 +838,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= +github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -627,14 +865,10 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= @@ -694,20 +928,16 @@ github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4 github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= -github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgx/v4 v4.18.0/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= -github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -735,6 +965,7 @@ github.com/lyft/protoc-gen-star v0.5.3 h1:zSGLzsUew8RT+ZKPHc3jnf8XLaVyHzTcAFBzHt github.com/lyft/protoc-gen-star/v2 v2.0.3 h1:/3+/2sWyXeMLzKd1bX+ixWKgEMsULrIivpDsuaF441o= github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -842,8 +1073,6 @@ go.etcd.io/etcd/client/v2 v2.305.12/go.mod h1:aQ/yhsxMu+Oht1FOupSr60oBvcS9cKXHrz go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0/go.mod h1:tIKj3DbO8N9Y2xo52og3irLsPI4GW02DSMtrVgNMgxg= @@ -852,8 +1081,6 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0/go.mod h1:rdENBZMT2OE6Ne/KLwpiXudnAsbdrdBaqBvTN8M8BgA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= @@ -862,8 +1089,6 @@ go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xC go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= @@ -890,6 +1115,7 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -911,6 +1137,7 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= @@ -926,6 +1153,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -935,7 +1163,9 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -973,6 +1203,7 @@ google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOn google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -989,6 +1220,9 @@ google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4 google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= @@ -1004,14 +1238,18 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go. google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78 h1:YqFWYZXim8bG9v68xU8WjTZmYKb5M5dMeSOWIp6jogI= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:vh/N7795ftP0AkN1w8XKqN4w1OdUKXW5Eummda+ofv8= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240314234333-6e1732d8331c h1:4z0DVWmDWWZ4OeQHLrb6lLBE3uCgSLs9DDA5Zb36DFg= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240314234333-6e1732d8331c/go.mod h1:IN9OQUXZ0xT+26MDwZL8fJcYw+y99b0eYPA2U15Jt8o= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240521202816-d264139d666e h1:Px+x8PNp8izq1ORW6jI007V/fRZ3bWrgcWHImtBduXc= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240521202816-d264139d666e/go.mod h1:0J6mmn3XAEjfNbPvpH63c0RXCjGNFcCzlEfWSN4In+k= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240722135656-d784300faade h1:fc+h2kSr2nW2DHxAdGYeX3bnkr4iFsKHUu9Fi6Rh4Y8= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20240722135656-d784300faade/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= @@ -1024,12 +1262,16 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= @@ -1052,8 +1294,10 @@ google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= modernc.org/b v1.0.2 h1:iPC2u39ebzq12GOC2yXT4mve0HrWcH85cz+midWjzeo= From 01332490d37c8c52021d6a5478e9fa97cbbcf1da Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 29 Jul 2024 12:39:56 +0800 Subject: [PATCH 082/106] refactor: Update user service to use sync.Once for singleton initialization --- core/user/service_v2.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/user/service_v2.go b/core/user/service_v2.go index c0ab759c2..696b7d957 100644 --- a/core/user/service_v2.go +++ b/core/user/service_v2.go @@ -1,6 +1,7 @@ package user import ( + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" @@ -14,6 +15,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "sync" "time" ) @@ -181,7 +183,7 @@ func (svc *ServiceV2) getSecretFunc() jwt.Keyfunc { } } -func NewUserServiceV2() (svc *ServiceV2, err error) { +func newUserServiceV2() (svc *ServiceV2, err error) { // service svc = &ServiceV2{ modelSvc: service.NewModelServiceV2[models.UserV2](), @@ -191,6 +193,7 @@ func NewUserServiceV2() (svc *ServiceV2, err error) { // initialize if err := svc.Init(); err != nil { + log.Errorf("failed to initialize user service: %v", err) return nil, trace.TraceError(err) } @@ -198,15 +201,14 @@ func NewUserServiceV2() (svc *ServiceV2, err error) { } var userSvcV2 *ServiceV2 +var userSvcV2Once sync.Once func GetUserServiceV2() (svc *ServiceV2, err error) { - if userSvcV2 != nil { - return userSvcV2, nil - } - svc, err = NewUserServiceV2() - if err != nil { - return nil, err - } - userSvcV2 = svc - return svc, nil + userSvcV2Once.Do(func() { + userSvcV2, err = newUserServiceV2() + if err != nil { + return + } + }) + return userSvcV2, nil } From 779d831a6d7e04e0efcf84d650e0131ded22ee62 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 30 Jul 2024 12:35:18 +0800 Subject: [PATCH 083/106] feat: Add NotificationAlertV2 model Add NotificationAlertV2 model to handle notification alerts in the system. This model includes fields for name, description, enabled status, metric target ID, metric name, operator, target value, and level. --- core/models/models/v2/notification_alert_v2.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 core/models/models/v2/notification_alert_v2.go diff --git a/core/models/models/v2/notification_alert_v2.go b/core/models/models/v2/notification_alert_v2.go new file mode 100644 index 000000000..ad35841b4 --- /dev/null +++ b/core/models/models/v2/notification_alert_v2.go @@ -0,0 +1,16 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationAlertV2 struct { + any `collection:"notification_alerts"` + BaseModelV2[NotificationAlertV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` + MetricName string `json:"metric_name" bson:"metric_name"` + Operator string `json:"operator" bson:"operator"` + TargetValue float32 `json:"target_value" bson:"target_value"` + Level string `json:"level" bson:"level"` +} From 5b67717ebe51f18785697f4e4d307b447537ce6c Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 30 Jul 2024 15:46:02 +0800 Subject: [PATCH 084/106] =?UTF-8?q?=E2=9C=A8=20feat(models):=20add=20HasMe?= =?UTF-8?q?tricTarget=20and=20LastingSeconds=20fields=20to=20NotificationA?= =?UTF-8?q?lertV2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/models/models/v2/notification_alert_v2.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/models/models/v2/notification_alert_v2.go b/core/models/models/v2/notification_alert_v2.go index ad35841b4..bd947b5fd 100644 --- a/core/models/models/v2/notification_alert_v2.go +++ b/core/models/models/v2/notification_alert_v2.go @@ -8,9 +8,11 @@ type NotificationAlertV2 struct { Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` Enabled bool `json:"enabled" bson:"enabled"` + HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` MetricName string `json:"metric_name" bson:"metric_name"` Operator string `json:"operator" bson:"operator"` + LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` TargetValue float32 `json:"target_value" bson:"target_value"` Level string `json:"level" bson:"level"` } From 86a7beb9e263bb761d456e1f3358238e07468c89 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 30 Jul 2024 17:33:42 +0800 Subject: [PATCH 085/106] =?UTF-8?q?=E2=9C=A8=20add:=20add=20AlertId=20fiel?= =?UTF-8?q?d=20to=20NotificationSettingV2=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/models/models/v2/notification_setting_v2.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index b47821fd4..45ba90d1f 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -30,4 +30,6 @@ type NotificationSettingV2 struct { ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` + + AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty"` } From 7fe770ae9d154e9d7e7f0912da8dfd3c8768504d Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2024 15:58:41 +0800 Subject: [PATCH 086/106] feat: Update notification trigger patterns and add alert trigger The code changes modify the notification constants and models to update the trigger patterns for tasks and nodes. Additionally, a new trigger for alerts is added to the constants. This change allows for more flexible matching of notification triggers. --- core/constants/notification.go | 5 +- core/grpc/server/task_server_v2.go | 6 +- .../models/models/v2/notification_alert_v2.go | 1 + .../models/v2/notification_setting_v2.go | 5 +- core/notification/entity.go | 12 ++- core/notification/service_v2.go | 94 ++++++++++++++++++- 6 files changed, 109 insertions(+), 14 deletions(-) diff --git a/core/constants/notification.go b/core/constants/notification.go index 34cced2e3..19beaf521 100644 --- a/core/constants/notification.go +++ b/core/constants/notification.go @@ -1,8 +1,8 @@ package constants const ( - NotificationTriggerTargetTask = "task" - NotificationTriggerTargetNode = "node" + NotificationTriggerPatternTask = "^task" + NotificationTriggerPatternNode = "^node" ) const ( @@ -12,6 +12,7 @@ const ( NotificationTriggerNodeStatusChange = "node_status_change" NotificationTriggerNodeOnline = "node_online" NotificationTriggerNodeOffline = "node_offline" + NotificationTriggerAlert = "alert" ) const ( diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index d90760fce..3ae0ddb08 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -168,8 +168,10 @@ func (svr TaskServerV2) SendNotification(_ context.Context, request *grpc.TaskSe // settings settings, err := service.NewModelServiceV2[models2.NotificationSettingV2]().GetMany(bson.M{ - "enabled": true, - "trigger_target": constants.NotificationTriggerTargetTask, + "enabled": true, + "trigger": bson.M{ + "$regex": constants.NotificationTriggerPatternTask, + }, }, nil) if err != nil { return nil, trace.TraceError(err) diff --git a/core/models/models/v2/notification_alert_v2.go b/core/models/models/v2/notification_alert_v2.go index bd947b5fd..d91e44361 100644 --- a/core/models/models/v2/notification_alert_v2.go +++ b/core/models/models/v2/notification_alert_v2.go @@ -15,4 +15,5 @@ type NotificationAlertV2 struct { LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` TargetValue float32 `json:"target_value" bson:"target_value"` Level string `json:"level" bson:"level"` + TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` } diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go index 45ba90d1f..7a932b3bc 100644 --- a/core/models/models/v2/notification_setting_v2.go +++ b/core/models/models/v2/notification_setting_v2.go @@ -17,9 +17,8 @@ type NotificationSettingV2 struct { TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - TriggerTarget string `json:"trigger_target" bson:"trigger_target"` - Trigger string `json:"trigger" bson:"trigger"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Trigger string `json:"trigger" bson:"trigger"` SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty"` diff --git a/core/notification/entity.go b/core/notification/entity.go index b7e410270..cf7933eef 100644 --- a/core/notification/entity.go +++ b/core/notification/entity.go @@ -3,9 +3,11 @@ package notification import "github.com/crawlab-team/crawlab/core/models/models/v2" type VariableData struct { - Task *models.TaskV2 `json:"task"` - TaskStat *models.TaskStatV2 `json:"task_stat"` - Spider *models.SpiderV2 `json:"spider"` - Node *models.NodeV2 `json:"node"` - Schedule *models.ScheduleV2 `json:"schedule"` + Task *models.TaskV2 `json:"task"` + TaskStat *models.TaskStatV2 `json:"task_stat"` + Spider *models.SpiderV2 `json:"spider"` + Node *models.NodeV2 `json:"node"` + Schedule *models.ScheduleV2 `json:"schedule"` + Alert *models.NotificationAlertV2 `json:"alert"` + Metric *models.MetricV2 `json:"metric"` } diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 69360ecd1..377e2e686 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -264,6 +264,43 @@ func (svc *ServiceV2) geContentWithVariables(template string, variables []entity case "updated_by": content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.UpdatedBy)) } + + case "alert": + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Id.Hex()) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Name) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Description) + case "enabled": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Alert.Enabled)) + case "metric_name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.MetricName) + case "operator": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Operator) + case "lasting_seconds": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Alert.LastingSeconds)) + case "target_value": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTargetValue(vd.Alert)) + case "level": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Level) + } + + case "metric": + if vd.Metric == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "type": + content = strings.ReplaceAll(content, v.GetKey(), vd.Metric.Type) + case "node_id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Metric.NodeId.Hex()) + default: + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedMetricValue(v.Name, vd.Metric)) + } + } } return content @@ -282,6 +319,10 @@ func (svc *ServiceV2) getVariableData(args ...any) (vd VariableData) { vd.Node = arg.(*models.NodeV2) case *models.ScheduleV2: vd.Schedule = arg.(*models.ScheduleV2) + case *models.NotificationAlertV2: + vd.Alert = arg.(*models.NotificationAlertV2) + case *models.MetricV2: + vd.Metric = arg.(*models.MetricV2) } } return vd @@ -336,6 +377,53 @@ func (svc *ServiceV2) getFormattedTime(t time.Time) (res string) { return t.Local().Format(time.DateTime) } +func (svc *ServiceV2) getFormattedTargetValue(a *models.NotificationAlertV2) (res string) { + if strings.HasSuffix(a.MetricName, "_percent") { + return fmt.Sprintf("%.2f%%", a.TargetValue) + } else if strings.HasSuffix(a.MetricName, "_memory") { + return fmt.Sprintf("%dMB", int(a.TargetValue/(1024*1024))) + } else if strings.HasSuffix(a.MetricName, "_disk") { + return fmt.Sprintf("%dGB", int(a.TargetValue/(1024*1024*1024))) + } else if strings.HasSuffix(a.MetricName, "_rate") { + return fmt.Sprintf("%.2fMB/s", a.TargetValue/(1024*1024)) + } else { + return fmt.Sprintf("%f", a.TargetValue) + } +} + +func (svc *ServiceV2) getFormattedMetricValue(metricName string, m *models.MetricV2) (res string) { + switch metricName { + case "cpu_usage_percent": + return fmt.Sprintf("%.2f%%", m.CpuUsagePercent) + case "total_memory": + return fmt.Sprintf("%dMB", m.TotalMemory/(1024*1024)) + case "available_memory": + return fmt.Sprintf("%dMB", m.AvailableMemory/(1024*1024)) + case "used_memory": + return fmt.Sprintf("%dMB", m.UsedMemory/(1024*1024)) + case "used_memory_percent": + return fmt.Sprintf("%.2f%%", m.UsedMemoryPercent) + case "total_disk": + return fmt.Sprintf("%dGB", m.TotalDisk/(1024*1024*1024)) + case "available_disk": + return fmt.Sprintf("%dGB", m.AvailableDisk/(1024*1024*1024)) + case "used_disk": + return fmt.Sprintf("%dGB", m.UsedDisk/(1024*1024*1024)) + case "used_disk_percent": + return fmt.Sprintf("%.2f%%", m.UsedDiskPercent) + case "disk_read_bytes_rate": + return fmt.Sprintf("%.2fMB/s", m.DiskReadBytesRate/(1024*1024)) + case "disk_write_bytes_rate": + return fmt.Sprintf("%.2fMB/s", m.DiskWriteBytesRate/(1024*1024)) + case "network_bytes_sent_rate": + return fmt.Sprintf("%.2fMB/s", m.NetworkBytesSentRate/(1024*1024)) + case "network_bytes_recv_rate": + return fmt.Sprintf("%.2fMB/s", m.NetworkBytesRecvRate/(1024*1024)) + default: + return "N/A" + } +} + func (svc *ServiceV2) convertMarkdownToHtml(content string) (html string) { return string(markdown.ToHTML([]byte(content), nil, nil)) } @@ -347,8 +435,10 @@ func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { // settings settings, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetMany(bson.M{ - "enabled": true, - "trigger_target": constants.NotificationTriggerTargetNode, + "enabled": true, + "trigger": bson.M{ + "$regex": constants.NotificationTriggerPatternNode, + }, }, nil) if err != nil { log.Errorf("get notification settings error: %v", err) From dd6921d32bdf4cf82bb7c3ecbff69153966b688a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 31 Jul 2024 16:02:34 +0800 Subject: [PATCH 087/106] feat: Update notification trigger patterns and add alert trigger --- core/go.mod | 12 +++++------- core/go.sum | 20 +------------------- go.work.sum | 8 +++++--- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/core/go.mod b/core/go.mod index 293710c2d..5291ef40a 100644 --- a/core/go.mod +++ b/core/go.mod @@ -15,11 +15,11 @@ require ( github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 github.com/cenkalti/backoff/v4 v4.3.0 - github.com/crawlab-team/crawlab/db v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/fs v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/grpc v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/trace v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/vcs v0.0.0-20240701075920-023ba275664a + github.com/crawlab-team/crawlab/db v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/fs v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/trace v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/vcs v0.0.0-20240731075841-7fe770ae9d15 github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 @@ -87,7 +87,6 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect @@ -163,7 +162,6 @@ require ( golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/core/go.sum b/core/go.sum index 92c0660e8..ee5b1249a 100644 --- a/core/go.sum +++ b/core/go.sum @@ -27,7 +27,6 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= @@ -311,8 +310,6 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -325,8 +322,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= -github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -709,22 +704,14 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1192,12 +1179,9 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f h1:RARaIm8pxYuxyNPbBQf5igT7XdOyCNtat1qAT2ZxjU4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1227,8 +1211,6 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= diff --git a/go.work.sum b/go.work.sum index 967f3c1b1..b91b7aa62 100644 --- a/go.work.sum +++ b/go.work.sum @@ -11,6 +11,7 @@ cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbD cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= cloud.google.com/go/accessapproval v1.7.5 h1:uzmAMSgYcnlHa9X9YSQZ4Q1wlfl4NNkZyQgho1Z6p04= cloud.google.com/go/accessapproval v1.7.5/go.mod h1:g88i1ok5dvQ9XJsxpUInWWvUBrIZhyPDPbk4T01OoJ0= @@ -843,7 +844,6 @@ github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -945,7 +945,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= @@ -1084,14 +1083,17 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0/go.mod h1: go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= @@ -1221,7 +1223,6 @@ google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJ google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= @@ -1285,6 +1286,7 @@ google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFL google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= From a7da3a04425b14d4645d691361a34adfa3b0bec6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 3 Aug 2024 19:43:11 +0800 Subject: [PATCH 088/106] feat: Update notification trigger patterns and add alert trigger --- core/controllers/router_v2.go | 5 +++ core/controllers/setting_v2.go | 45 +++++++++++++++++++++++--- core/models/common/index_service_v2.go | 2 +- core/system/service.go | 11 ++++--- go.work.sum | 2 ++ 5 files changed, 55 insertions(+), 10 deletions(-) diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 4f2ff1d6a..60d9ecd3c 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -326,6 +326,11 @@ func InitRoutes(app *gin.Engine) (err error) { Path: "/:id", HandlerFunc: GetSetting, }, + { + Method: http.MethodPost, + Path: "/:id", + HandlerFunc: PostSetting, + }, { Method: http.MethodPut, Path: "/:id", diff --git a/core/controllers/setting_v2.go b/core/controllers/setting_v2.go index d2d913391..c4be44cc3 100644 --- a/core/controllers/setting_v2.go +++ b/core/controllers/setting_v2.go @@ -1,11 +1,12 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "errors" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" ) func GetSetting(c *gin.Context) { @@ -13,8 +14,12 @@ func GetSetting(c *gin.Context) { key := c.Param("id") // setting - s, err := service.NewModelServiceV2[models2.SettingV2]().GetOne(bson.M{"key": key}, nil) + s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil) if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + HandleSuccess(c) + return + } HandleErrorInternalServerError(c, err) return } @@ -22,18 +27,48 @@ func GetSetting(c *gin.Context) { HandleSuccessWithData(c, s) } +func PostSetting(c *gin.Context) { + // key + key := c.Param("id") + + // settings + var s models.SettingV2 + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + if s.Key == "" { + s.Key = key + } + + u := GetUserFromContextV2(c) + + s.SetCreated(u.Id) + s.SetUpdated(u.Id) + + id, err := service.NewModelServiceV2[models.SettingV2]().InsertOne(s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + s.Id = id + + HandleSuccessWithData(c, s) +} + func PutSetting(c *gin.Context) { // key key := c.Param("id") // settings - var s models.Setting + var s models.SettingV2 if err := c.ShouldBindJSON(&s); err != nil { HandleErrorInternalServerError(c, err) return } - modelSvc := service.NewModelServiceV2[models2.SettingV2]() + modelSvc := service.NewModelServiceV2[models.SettingV2]() // setting _s, err := modelSvc.GetOne(bson.M{"key": key}, nil) diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index 2805ad3ed..0eae98336 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -68,7 +68,7 @@ func CreateIndexesV2() { // settings mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, }) // tokens diff --git a/core/system/service.go b/core/system/service.go index 891657bb5..94cd31403 100644 --- a/core/system/service.go +++ b/core/system/service.go @@ -25,7 +25,7 @@ func (svc *Service) Init() (err error) { func (svc *Service) initData() (err error) { total, err := svc.col.Count(bson.M{ - "key": "site_title", + "key": "customize", }) if err != nil { return err @@ -38,10 +38,13 @@ func (svc *Service) initData() (err error) { settings := []models.Setting{ { Id: primitive.NewObjectID(), - Key: "site_title", + Key: "customize", Value: bson.M{ - "customize_site_title": false, - "site_title": "", + "show_custom_title": false, + "custom_title": "", + "show_custom_logo": false, + "custom_logo": "", + "hide_platform_version": false, }, }, } diff --git a/go.work.sum b/go.work.sum index b91b7aa62..a737bf099 100644 --- a/go.work.sum +++ b/go.work.sum @@ -844,6 +844,7 @@ github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -945,6 +946,7 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= From e2cb99e56ac4341ba6bec66bd74c57fa53941451 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 4 Aug 2024 16:58:46 +0800 Subject: [PATCH 089/106] feat: Update models to use DatabaseV2 instead of DataSourceV2 The code changes update the models and related functions to use the new DatabaseV2 struct instead of the deprecated DataSourceV2 struct. This change ensures consistency and clarity in the codebase. --- .../{data_source_v2.go => database_v2.go} | 14 ++++++------- core/controllers/result_v2.go | 4 ++-- core/controllers/router_v2.go | 11 +++------- core/controllers/spider_v2.go | 4 ++-- core/ds/service_v2.go | 18 ++++++++--------- .../server/model_base_service_v2_server.go | 2 +- core/models/common/index_service_v2.go | 2 +- core/models/models/v2/data_source_v2.go | 20 ------------------- core/models/models/v2/database_v2.go | 20 +++++++++++++++++++ core/models/models/v2/spider_v2.go | 2 +- core/utils/cockroachdb.go | 4 ++-- core/utils/es.go | 4 ++-- core/utils/kafka.go | 4 ++-- core/utils/mongo.go | 4 ++-- core/utils/mssql.go | 4 ++-- core/utils/mysql.go | 4 ++-- core/utils/postgresql.go | 4 ++-- core/utils/sqlite.go | 4 ++-- 18 files changed, 62 insertions(+), 67 deletions(-) rename core/controllers/{data_source_v2.go => database_v2.go} (88%) delete mode 100644 core/models/models/v2/data_source_v2.go create mode 100644 core/models/models/v2/database_v2.go diff --git a/core/controllers/data_source_v2.go b/core/controllers/database_v2.go similarity index 88% rename from core/controllers/data_source_v2.go rename to core/controllers/database_v2.go index 84a3ac534..cd175cc60 100644 --- a/core/controllers/data_source_v2.go +++ b/core/controllers/database_v2.go @@ -9,7 +9,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) -func PostDataSource(c *gin.Context) { +func PostDatabase(c *gin.Context) { // data source var payload struct { Name string `json:"name"` @@ -35,7 +35,7 @@ func PostDataSource(c *gin.Context) { u := GetUserFromContextV2(c) // add data source to db - dataSource := models.DataSourceV2{ + dataSource := models.DatabaseV2{ Name: payload.Name, Type: payload.Type, Description: payload.Description, @@ -53,7 +53,7 @@ func PostDataSource(c *gin.Context) { } dataSource.SetCreated(u.Id) dataSource.SetUpdated(u.Id) - id, err := service.NewModelServiceV2[models.DataSourceV2]().InsertOne(dataSource) + id, err := service.NewModelServiceV2[models.DatabaseV2]().InsertOne(dataSource) if err != nil { HandleErrorInternalServerError(c, err) return @@ -68,7 +68,7 @@ func PostDataSource(c *gin.Context) { HandleSuccessWithData(c, dataSource) } -func PutDataSourceById(c *gin.Context) { +func PutDatabaseById(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { HandleErrorInternalServerError(c, err) @@ -76,13 +76,13 @@ func PutDataSourceById(c *gin.Context) { } // data source - var dataSource models.DataSourceV2 + var dataSource models.DatabaseV2 if err := c.ShouldBindJSON(&dataSource); err != nil { HandleErrorBadRequest(c, err) return } - err = service.NewModelServiceV2[models.DataSourceV2]().ReplaceById(id, dataSource) + err = service.NewModelServiceV2[models.DatabaseV2]().ReplaceById(id, dataSource) if err != nil { HandleErrorInternalServerError(c, err) return @@ -94,7 +94,7 @@ func PutDataSourceById(c *gin.Context) { }() } -func PostDataSourceChangePassword(c *gin.Context) { +func PostDatabaseChangePassword(c *gin.Context) { id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { HandleErrorBadRequest(c, err) diff --git a/core/controllers/result_v2.go b/core/controllers/result_v2.go index 2f7fb7372..662ef1f47 100644 --- a/core/controllers/result_v2.go +++ b/core/controllers/result_v2.go @@ -39,10 +39,10 @@ func GetResultList(c *gin.Context) { } // data source - ds, err := service.NewModelServiceV2[models2.DataSourceV2]().GetById(dsId) + ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId) if err != nil { if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models2.DataSourceV2{} + ds = &models2.DatabaseV2{} } else { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 60d9ecd3c..39ec57eb5 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -56,21 +56,16 @@ func InitRoutes(app *gin.Engine) (err error) { groups := NewRouterGroups(app) RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]()) - RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models2.DataSourceV2]([]Action{ + RegisterController(groups.AuthGroup, "/databases", NewControllerV2[models2.DatabaseV2]([]Action{ { Method: http.MethodPost, Path: "", - HandlerFunc: PostDataSource, + HandlerFunc: PostDatabase, }, { Method: http.MethodPut, Path: "/:id", - HandlerFunc: PutDataSourceById, - }, - { - Method: http.MethodPost, - Path: "/:id/change-password", - HandlerFunc: PostDataSourceChangePassword, + HandlerFunc: PutDatabaseById, }, }...)) RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]()) diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index a0abfb371..bcd0b5aab 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -699,7 +699,7 @@ func GetSpiderDataSource(c *gin.Context) { } // data source - ds, err := service.NewModelServiceV2[models2.DataSourceV2]().GetById(s.DataSourceId) + ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(s.DataSourceId) if err != nil { if err.Error() == mongo2.ErrNoDocuments.Error() { HandleSuccess(c) @@ -736,7 +736,7 @@ func PostSpiderDataSource(c *gin.Context) { // data source if !dsId.IsZero() { - _, err = service.NewModelServiceV2[models2.DataSourceV2]().GetById(dsId) + _, err = service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/ds/service_v2.go b/core/ds/service_v2.go index e96a96453..fbf8fe247 100644 --- a/core/ds/service_v2.go +++ b/core/ds/service_v2.go @@ -51,7 +51,7 @@ func (svc *ServiceV2) Stop() { } func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { - dataSource, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(id) + dataSource, err := service.NewModelServiceV2[models.DatabaseV2]().GetById(id) if err != nil { return err } @@ -60,7 +60,7 @@ func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by return err } dataSource.SetUpdated(by) - err = service.NewModelServiceV2[models.DataSourceV2]().ReplaceById(id, *dataSource) + err = service.NewModelServiceV2[models.DatabaseV2]().ReplaceById(id, *dataSource) if err != nil { return err } @@ -85,7 +85,7 @@ func (svc *ServiceV2) Monitor() { } func (svc *ServiceV2) CheckStatus(id primitive.ObjectID) (err error) { - ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(id) + ds, err := service.NewModelServiceV2[models.DatabaseV2]().GetById(id) if err != nil { return err } @@ -106,7 +106,7 @@ func (svc *ServiceV2) monitor() (err error) { log.Debugf("[DataSourceService] start monitoring") // data source list - dataSources, err := service.NewModelServiceV2[models.DataSourceV2]().GetMany(nil, nil) + dataSources, err := service.NewModelServiceV2[models.DatabaseV2]().GetMany(nil, nil) if err != nil { return err } @@ -118,7 +118,7 @@ func (svc *ServiceV2) monitor() (err error) { // iterate data source list for _, ds := range dataSources { // async operation - go func(ds *models.DataSourceV2) { + go func(ds *models.DatabaseV2) { // check status and save _ = svc.checkStatus(ds, true) @@ -137,7 +137,7 @@ func (svc *ServiceV2) monitor() (err error) { return nil } -func (svc *ServiceV2) checkStatus(ds *models.DataSourceV2, save bool) (err error) { +func (svc *ServiceV2) checkStatus(ds *models.DatabaseV2, save bool) (err error) { // check status if err := svc._checkStatus(ds); err != nil { ds.Status = constants2.DataSourceStatusOffline @@ -155,12 +155,12 @@ func (svc *ServiceV2) checkStatus(ds *models.DataSourceV2, save bool) (err error return nil } -func (svc *ServiceV2) _save(ds *models.DataSourceV2) (err error) { +func (svc *ServiceV2) _save(ds *models.DatabaseV2) (err error) { log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error) - return service.NewModelServiceV2[models.DataSourceV2]().ReplaceById(ds.Id, *ds) + return service.NewModelServiceV2[models.DatabaseV2]().ReplaceById(ds.Id, *ds) } -func (svc *ServiceV2) _checkStatus(ds *models.DataSourceV2) (err error) { +func (svc *ServiceV2) _checkStatus(ds *models.DatabaseV2) (err error) { switch ds.Type { case constants.DataSourceTypeMongo: _, err := utils2.GetMongoClientWithTimeoutV2(ds, svc.timeout) diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 90eec10dc..d067e11ca 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -18,7 +18,7 @@ var ( typeOneInstances = []any{ *new(models2.TestModelV2), *new(models2.DataCollectionV2), - *new(models2.DataSourceV2), + *new(models2.DatabaseV2), *new(models2.DependencyV2), *new(models2.DependencyLogV2), *new(models2.DependencySettingV2), diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index 0eae98336..20c278f26 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -82,7 +82,7 @@ func CreateIndexesV2() { }) // data sources - mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataSourceV2{})).MustCreateIndexes([]mongo2.IndexModel{ + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{ {Keys: bson.M{"name": 1}}, }) diff --git a/core/models/models/v2/data_source_v2.go b/core/models/models/v2/data_source_v2.go deleted file mode 100644 index 7e9201cd6..000000000 --- a/core/models/models/v2/data_source_v2.go +++ /dev/null @@ -1,20 +0,0 @@ -package models - -type DataSourceV2 struct { - any `collection:"data_sources"` - BaseModelV2[DataSourceV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - Description string `json:"description" bson:"description"` - Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` - Url string `json:"url" bson:"url"` - Hosts []string `json:"hosts" bson:"hosts"` - Database string `json:"database" bson:"database"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - ConnectType string `json:"connect_type" bson:"connect_type"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` -} diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go new file mode 100644 index 000000000..3604a335f --- /dev/null +++ b/core/models/models/v2/database_v2.go @@ -0,0 +1,20 @@ +package models + +type DatabaseV2 struct { + any `collection:"databases"` + BaseModelV2[DatabaseV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Type string `json:"type" bson:"type"` + Description string `json:"description" bson:"description"` + Host string `json:"host" bson:"host"` + Port string `json:"port" bson:"port"` + Url string `json:"url" bson:"url"` + Hosts []string `json:"hosts" bson:"hosts"` + Database string `json:"database" bson:"database"` + Username string `json:"username" bson:"username"` + Password string `json:"-,omitempty" bson:"password"` + ConnectType string `json:"connect_type" bson:"connect_type"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` +} diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go index 24b4cb059..5e91baf47 100644 --- a/core/models/models/v2/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -11,7 +11,7 @@ type SpiderV2 struct { ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id ColName string `json:"col_name,omitempty" bson:"-"` // data collection name DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source + DataSource *DatabaseV2 `json:"data_source,omitempty" bson:"-"` // data source Description string `json:"description" bson:"description"` // description ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id Mode string `json:"mode" bson:"mode"` // default Task.Mode diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go index 3534a1de0..e3debd81c 100644 --- a/core/utils/cockroachdb.go +++ b/core/utils/cockroachdb.go @@ -60,13 +60,13 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses return s, err } -func GetCockroachdbSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetCockroachdbSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getCockroachdbSessionV2(ctx, ds) } -func getCockroachdbSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { +func getCockroachdbSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/es.go b/core/utils/es.go index d2db5e56c..b3c7238ee 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -84,13 +84,13 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas return c, err } -func GetElasticsearchClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *elasticsearch.Client, err error) { +func GetElasticsearchClientWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *elasticsearch.Client, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getElasticsearchClientV2(ctx, ds) } -func getElasticsearchClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *elasticsearch.Client, err error) { +func getElasticsearchClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *elasticsearch.Client, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/kafka.go b/core/utils/kafka.go index e47d2107b..b6d50a808 100644 --- a/core/utils/kafka.go +++ b/core/utils/kafka.go @@ -41,13 +41,13 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co return kafka.DialLeader(ctx, network, address, topic, partition) } -func GetKafkaConnectionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *kafka.Conn, err error) { +func GetKafkaConnectionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *kafka.Conn, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getKafkaConnectionV2(ctx, ds) } -func getKafkaConnectionV2(ctx context.Context, ds *models2.DataSourceV2) (c *kafka.Conn, err error) { +func getKafkaConnectionV2(ctx context.Context, ds *models2.DatabaseV2) (c *kafka.Conn, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/mongo.go b/core/utils/mongo.go index a2fd3b030..f98dd3e51 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -55,7 +55,7 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c return getMongoClient(ctx, ds) } -func GetMongoClientWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (c *mongo2.Client, err error) { +func GetMongoClientWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *mongo2.Client, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMongoClientV2(ctx, ds) @@ -100,7 +100,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien return mongo.GetMongoClient(opts...) } -func getMongoClientV2(ctx context.Context, ds *models2.DataSourceV2) (c *mongo2.Client, err error) { +func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Client, err error) { // normalize settings if ds.Host == "" { ds.Host = constants.DefaultHost diff --git a/core/utils/mssql.go b/core/utils/mssql.go index 7333706ba..35d852b49 100644 --- a/core/utils/mssql.go +++ b/core/utils/mssql.go @@ -60,13 +60,13 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetMssqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetMssqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMssqlSessionV2(ctx, ds) } -func getMssqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { +func getMssqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/mysql.go b/core/utils/mysql.go index 4ea293cff..6b7015543 100644 --- a/core/utils/mysql.go +++ b/core/utils/mysql.go @@ -60,13 +60,13 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetMysqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetMysqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getMysqlSessionV2(ctx, ds) } -func getMysqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { +func getMysqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go index a3086762e..30ed5f642 100644 --- a/core/utils/postgresql.go +++ b/core/utils/postgresql.go @@ -60,13 +60,13 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess return s, err } -func GetPostgresqlSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetPostgresqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getPostgresqlSessionV2(ctx, ds) } -func getPostgresqlSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { +func getPostgresqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { // normalize settings host := ds.Host port := ds.Port diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go index b8e877867..8e39db2a3 100644 --- a/core/utils/sqlite.go +++ b/core/utils/sqlite.go @@ -45,13 +45,13 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } -func GetSqliteSessionWithTimeoutV2(ds *models2.DataSourceV2, timeout time.Duration) (s db.Session, err error) { +func GetSqliteSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return getSqliteSessionV2(ctx, ds) } -func getSqliteSessionV2(ctx context.Context, ds *models2.DataSourceV2) (s db.Session, err error) { +func getSqliteSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { // connect settings settings := sqlite.ConnectionURL{ Database: ds.Database, From e57655c67ebde71f15c6786643f425a6ebc2c17b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 5 Aug 2024 13:14:57 +0800 Subject: [PATCH 090/106] refactor: Update models to use DatabaseV2 instead of DataSourceV2 --- core/constants/ds.go | 14 +++++++------- core/controllers/database_v2.go | 3 +-- core/controllers/router_v2.go | 12 ------------ core/ds/cockroachdb.go | 2 +- core/ds/es.go | 2 +- core/ds/kafka.go | 2 +- core/ds/mongo.go | 2 +- core/ds/mssql.go | 2 +- core/ds/mysql.go | 2 +- core/ds/postgresql.go | 2 +- core/models/models/data_source.go | 2 +- core/models/models/v2/database_v2.go | 9 +++++++-- core/utils/cockroachdb.go | 4 ++-- core/utils/es.go | 4 ++-- core/utils/kafka.go | 4 ++-- core/utils/mongo.go | 6 ++---- core/utils/mssql.go | 4 ++-- core/utils/mysql.go | 4 ++-- core/utils/postgresql.go | 4 ++-- db/mongo/client.go | 10 +++++----- db/mongo/client_options.go | 4 ++-- 21 files changed, 44 insertions(+), 54 deletions(-) diff --git a/core/constants/ds.go b/core/constants/ds.go index 1a92a8ad4..268c57612 100644 --- a/core/constants/ds.go +++ b/core/constants/ds.go @@ -16,13 +16,13 @@ const ( ) const ( - DefaultMongoPort = "27017" - DefaultMysqlPort = "3306" - DefaultPostgresqlPort = "5432" - DefaultMssqlPort = "1433" - DefaultCockroachdbPort = "26257" - DefaultElasticsearchPort = "9200" - DefaultKafkaPort = "9092" + DefaultMongoPort = 27017 + DefaultMysqlPort = 3306 + DefaultPostgresqlPort = 5432 + DefaultMssqlPort = 1433 + DefaultCockroachdbPort = 26257 + DefaultElasticsearchPort = 9200 + DefaultKafkaPort = 9092 ) const ( diff --git a/core/controllers/database_v2.go b/core/controllers/database_v2.go index cd175cc60..b23eaf09c 100644 --- a/core/controllers/database_v2.go +++ b/core/controllers/database_v2.go @@ -16,7 +16,7 @@ func PostDatabase(c *gin.Context) { Type string `json:"type"` Description string `json:"description"` Host string `json:"host"` - Port string `json:"port"` + Port int `json:"port"` Url string `json:"url"` Hosts []string `json:"hosts"` Database string `json:"database"` @@ -42,7 +42,6 @@ func PostDatabase(c *gin.Context) { Host: payload.Host, Port: payload.Port, Url: payload.Url, - Hosts: payload.Hosts, Database: payload.Database, Username: payload.Username, Password: payload.Password, diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 39ec57eb5..36ee2a265 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -56,18 +56,6 @@ func InitRoutes(app *gin.Engine) (err error) { groups := NewRouterGroups(app) RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]()) - RegisterController(groups.AuthGroup, "/databases", NewControllerV2[models2.DatabaseV2]([]Action{ - { - Method: http.MethodPost, - Path: "", - HandlerFunc: PostDatabase, - }, - { - Method: http.MethodPut, - Path: "/:id", - HandlerFunc: PutDatabaseById, - }, - }...)) RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]()) RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models2.NodeV2]()) RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models2.ProjectV2]([]Action{ diff --git a/core/ds/cockroachdb.go b/core/ds/cockroachdb.go index 44aca2630..4af04720a 100644 --- a/core/ds/cockroachdb.go +++ b/core/ds/cockroachdb.go @@ -38,7 +38,7 @@ func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.Ob if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultCockroachdbPort } diff --git a/core/ds/es.go b/core/ds/es.go index e3910d195..de87ee7cb 100644 --- a/core/ds/es.go +++ b/core/ds/es.go @@ -179,7 +179,7 @@ func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive. if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultElasticsearchPort } diff --git a/core/ds/kafka.go b/core/ds/kafka.go index 4931b6326..0bf549400 100644 --- a/core/ds/kafka.go +++ b/core/ds/kafka.go @@ -81,7 +81,7 @@ func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultKafkaPort } diff --git a/core/ds/mongo.go b/core/ds/mongo.go index d6e69d3d4..29d912fa9 100644 --- a/core/ds/mongo.go +++ b/core/ds/mongo.go @@ -71,7 +71,7 @@ func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultMongoPort } diff --git a/core/ds/mssql.go b/core/ds/mssql.go index f73e4be65..516f7a329 100644 --- a/core/ds/mssql.go +++ b/core/ds/mssql.go @@ -39,7 +39,7 @@ func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultMssqlPort } diff --git a/core/ds/mysql.go b/core/ds/mysql.go index b9f840181..fc2f3bae0 100644 --- a/core/ds/mysql.go +++ b/core/ds/mysql.go @@ -38,7 +38,7 @@ func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultMysqlPort } diff --git a/core/ds/postgresql.go b/core/ds/postgresql.go index 174d004be..485a3a96f 100644 --- a/core/ds/postgresql.go +++ b/core/ds/postgresql.go @@ -39,7 +39,7 @@ func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.Obj if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultPostgresqlPort } diff --git a/core/models/models/data_source.go b/core/models/models/data_source.go index f444266f5..5693d9eb8 100644 --- a/core/models/models/data_source.go +++ b/core/models/models/data_source.go @@ -11,7 +11,7 @@ type DataSource struct { Type string `json:"type" bson:"type"` Description string `json:"description" bson:"description"` Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` + Port int `json:"port" bson:"port"` Url string `json:"url" bson:"url"` Hosts []string `json:"hosts" bson:"hosts"` Database string `json:"database" bson:"database"` diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go index 3604a335f..f990d00c2 100644 --- a/core/models/models/v2/database_v2.go +++ b/core/models/models/v2/database_v2.go @@ -4,12 +4,12 @@ type DatabaseV2 struct { any `collection:"databases"` BaseModelV2[DatabaseV2] `bson:",inline"` Name string `json:"name" bson:"name"` + DataSource string `json:"data_source" bson:"data_source"` Type string `json:"type" bson:"type"` Description string `json:"description" bson:"description"` Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` + Port int `json:"port" bson:"port"` Url string `json:"url" bson:"url"` - Hosts []string `json:"hosts" bson:"hosts"` Database string `json:"database" bson:"database"` Username string `json:"username" bson:"username"` Password string `json:"-,omitempty" bson:"password"` @@ -17,4 +17,9 @@ type DatabaseV2 struct { Status string `json:"status" bson:"status"` Error string `json:"error" bson:"error"` Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` + + MongoParams *struct { + AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` + AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty"` + } `json:"mongo_params,omitempty" bson:"mongo_params,omitempty"` } diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go index e3debd81c..012da3757 100644 --- a/core/utils/cockroachdb.go +++ b/core/utils/cockroachdb.go @@ -28,7 +28,7 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultCockroachdbPort } @@ -73,7 +73,7 @@ func getCockroachdbSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db. if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultCockroachdbPort } diff --git a/core/utils/es.go b/core/utils/es.go index b3c7238ee..ee70f1876 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -34,7 +34,7 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultElasticsearchPort } @@ -97,7 +97,7 @@ func getElasticsearchClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *e if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultElasticsearchPort } diff --git a/core/utils/kafka.go b/core/utils/kafka.go index b6d50a808..c07f19ea8 100644 --- a/core/utils/kafka.go +++ b/core/utils/kafka.go @@ -27,7 +27,7 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultKafkaPort } @@ -54,7 +54,7 @@ func getKafkaConnectionV2(ctx context.Context, ds *models2.DatabaseV2) (c *kafka if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultKafkaPort } diff --git a/core/utils/mongo.go b/core/utils/mongo.go index f98dd3e51..160706716 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -66,7 +66,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien if ds.Host == "" { ds.Host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { ds.Port = constants.DefaultMongoPort } @@ -79,7 +79,6 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien opts = append(opts, mongo.WithDb(ds.Database)) opts = append(opts, mongo.WithUsername(ds.Username)) opts = append(opts, mongo.WithPassword(ds.Password)) - opts = append(opts, mongo.WithHosts(ds.Hosts)) // extra if ds.Extra != nil { @@ -105,7 +104,7 @@ func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Cl if ds.Host == "" { ds.Host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { ds.Port = constants.DefaultMongoPort } @@ -118,7 +117,6 @@ func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Cl opts = append(opts, mongo.WithDb(ds.Database)) opts = append(opts, mongo.WithUsername(ds.Username)) opts = append(opts, mongo.WithPassword(ds.Password)) - opts = append(opts, mongo.WithHosts(ds.Hosts)) // extra if ds.Extra != nil { diff --git a/core/utils/mssql.go b/core/utils/mssql.go index 35d852b49..1190a28ec 100644 --- a/core/utils/mssql.go +++ b/core/utils/mssql.go @@ -28,7 +28,7 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultMssqlPort } @@ -73,7 +73,7 @@ func getMssqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Sessio if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultMssqlPort } diff --git a/core/utils/mysql.go b/core/utils/mysql.go index 6b7015543..34c8bcfb4 100644 --- a/core/utils/mysql.go +++ b/core/utils/mysql.go @@ -28,7 +28,7 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultMysqlPort } @@ -73,7 +73,7 @@ func getMysqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Sessio if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultMysqlPort } diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go index 30ed5f642..0971c10e5 100644 --- a/core/utils/postgresql.go +++ b/core/utils/postgresql.go @@ -28,7 +28,7 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultPostgresqlPort } @@ -73,7 +73,7 @@ func getPostgresqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.S if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { port = constants.DefaultPostgresqlPort } diff --git a/db/mongo/client.go b/db/mongo/client.go index e5c03f90d..f49a443ad 100644 --- a/db/mongo/client.go +++ b/db/mongo/client.go @@ -33,10 +33,10 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) { _opts.Host = "localhost" } } - if _opts.Port == "" { - _opts.Port = viper.GetString("mongo.port") - if _opts.Port == "" { - _opts.Port = "27017" + if _opts.Port == 0 { + _opts.Port = viper.GetInt("mongo.port") + if _opts.Port == 0 { + _opts.Port = 27017 } } if _opts.Db == "" { @@ -123,7 +123,7 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client, mongoOpts.SetHosts(_opts.Hosts) } else { // hosts are unset - mongoOpts.ApplyURI(fmt.Sprintf("mongodb://%s:%s/%s", _opts.Host, _opts.Port, _opts.Db)) + mongoOpts.ApplyURI(fmt.Sprintf("mongodb://%s:%d/%s", _opts.Host, _opts.Port, _opts.Db)) } } diff --git a/db/mongo/client_options.go b/db/mongo/client_options.go index ef664f2a0..3395876f5 100644 --- a/db/mongo/client_options.go +++ b/db/mongo/client_options.go @@ -8,7 +8,7 @@ type ClientOptions struct { Context context.Context Uri string Host string - Port string + Port int Db string Hosts []string Username string @@ -36,7 +36,7 @@ func WithHost(value string) ClientOption { } } -func WithPort(value string) ClientOption { +func WithPort(value int) ClientOption { return func(options *ClientOptions) { options.Port = value } From 421a06930c2ca3155f9eadc6781e6ae680a0fcf0 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 5 Aug 2024 16:07:19 +0800 Subject: [PATCH 091/106] refactor: Update models to use DatabaseV2 instead of DataSourceV2 --- core/controllers/database_v2.go | 30 ++++++++--------- core/models/models/v2/database_v2.go | 49 ++++++++++++++++++++-------- core/utils/mongo.go | 17 +--------- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/core/controllers/database_v2.go b/core/controllers/database_v2.go index b23eaf09c..771cee8d7 100644 --- a/core/controllers/database_v2.go +++ b/core/controllers/database_v2.go @@ -12,20 +12,19 @@ import ( func PostDatabase(c *gin.Context) { // data source var payload struct { - Name string `json:"name"` - Type string `json:"type"` - Description string `json:"description"` - Host string `json:"host"` - Port int `json:"port"` - Url string `json:"url"` - Hosts []string `json:"hosts"` - Database string `json:"database"` - Username string `json:"username"` - Password string `json:"-,omitempty"` - ConnectType string `json:"connect_type"` - Status string `json:"status"` - Error string `json:"error"` - Extra map[string]string `json:"extra,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + Description string `json:"description"` + Host string `json:"host"` + Port int `json:"port"` + Url string `json:"url"` + Hosts []string `json:"hosts"` + Database string `json:"database"` + Username string `json:"username"` + Password string `json:"-,omitempty"` + ConnectType string `json:"connect_type"` + Status string `json:"status"` + Error string `json:"error"` } if err := c.ShouldBindJSON(&payload); err != nil { HandleErrorBadRequest(c, err) @@ -41,14 +40,13 @@ func PostDatabase(c *gin.Context) { Description: payload.Description, Host: payload.Host, Port: payload.Port, - Url: payload.Url, + URI: payload.Url, Database: payload.Database, Username: payload.Username, Password: payload.Password, ConnectType: payload.ConnectType, Status: payload.Status, Error: payload.Error, - Extra: payload.Extra, } dataSource.SetCreated(u.Id) dataSource.SetUpdated(u.Id) diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go index f990d00c2..673bd92ae 100644 --- a/core/models/models/v2/database_v2.go +++ b/core/models/models/v2/database_v2.go @@ -1,25 +1,46 @@ package models +import "time" + type DatabaseV2 struct { any `collection:"databases"` BaseModelV2[DatabaseV2] `bson:",inline"` - Name string `json:"name" bson:"name"` - DataSource string `json:"data_source" bson:"data_source"` - Type string `json:"type" bson:"type"` - Description string `json:"description" bson:"description"` - Host string `json:"host" bson:"host"` - Port int `json:"port" bson:"port"` - Url string `json:"url" bson:"url"` - Database string `json:"database" bson:"database"` - Username string `json:"username" bson:"username"` - Password string `json:"-,omitempty" bson:"password"` - ConnectType string `json:"connect_type" bson:"connect_type"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` + Name string `json:"name" bson:"name"` + DataSource string `json:"data_source" bson:"data_source"` + Type string `json:"type" bson:"type"` + Description string `json:"description" bson:"description"` + Host string `json:"host" bson:"host"` + Port int `json:"port" bson:"port"` + URI string `json:"uri,omitempty" bson:"uri,omitempty"` + Database string `json:"database,omitempty" bson:"database,omitempty"` + Username string `json:"username,omitempty" bson:"username,omitempty"` + Password string `json:"-,omitempty" bson:"password,omitempty"` + ConnectType string `json:"connect_type,omitempty" bson:"connect_type,omitempty"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_ts" bson:"active_ts"` MongoParams *struct { AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty"` } `json:"mongo_params,omitempty" bson:"mongo_params,omitempty"` + PostgresParams *struct { + SSLMode string `json:"ssl_mode,omitempty" bson:"ssl_mode,omitempty"` + } `json:"postgres_params,omitempty" bson:"postgres_params,omitempty"` + SnowflakeParams *struct { + Account string `json:"account,omitempty" bson:"account,omitempty"` + Schema string `json:"schema,omitempty" bson:"schema,omitempty"` + Warehouse string `json:"warehouse,omitempty" bson:"warehouse,omitempty"` + Role string `json:"role,omitempty" bson:"role,omitempty"` + } `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty"` + CassandraParams *struct { + Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty"` + } + HiveParams *struct { + Auth string `json:"auth,omitempty" bson:"auth,omitempty"` + } + RedisParams *struct { + DB int `json:"db,omitempty" bson:"db,omitempty"` + } } diff --git a/core/utils/mongo.go b/core/utils/mongo.go index 160706716..f576cec57 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -111,28 +111,13 @@ func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Cl // options var opts []mongo.ClientOption opts = append(opts, mongo.WithContext(ctx)) - opts = append(opts, mongo.WithUri(ds.Url)) + opts = append(opts, mongo.WithUri(ds.URI)) opts = append(opts, mongo.WithHost(ds.Host)) opts = append(opts, mongo.WithPort(ds.Port)) opts = append(opts, mongo.WithDb(ds.Database)) opts = append(opts, mongo.WithUsername(ds.Username)) opts = append(opts, mongo.WithPassword(ds.Password)) - // extra - if ds.Extra != nil { - // auth source - authSource, ok := ds.Extra["auth_source"] - if ok { - opts = append(opts, mongo.WithAuthSource(authSource)) - } - - // auth mechanism - authMechanism, ok := ds.Extra["auth_mechanism"] - if ok { - opts = append(opts, mongo.WithAuthMechanism(authMechanism)) - } - } - // client return mongo.GetMongoClient(opts...) } From 5e7346788305ebb6c4c56633b68e04e061c03703 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 6 Aug 2024 11:34:16 +0800 Subject: [PATCH 092/106] feat: Update database models to use DatabaseV2 instead of DataSourceV2 This commit updates the database models and related functions to use the new DatabaseV2 struct instead of the deprecated DataSourceV2 struct. This change ensures consistency and clarity in the codebase. --- core/entity/database.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 core/entity/database.go diff --git a/core/entity/database.go b/core/entity/database.go new file mode 100644 index 000000000..e0033781c --- /dev/null +++ b/core/entity/database.go @@ -0,0 +1,25 @@ +package entity + +type DatabaseMetadata struct { + Databases []Database `json:"databases"` +} + +type Database struct { + Name string `json:"name"` + Tables []DatabaseTable `json:"tables"` +} + +type DatabaseTable struct { + Name string `json:"name"` + Columns []DatabaseColumn `json:"columns"` +} + +type DatabaseColumn struct { + Name string `json:"name"` + Type string `json:"type"` + Null bool `json:"null,omitempty"` + Key string `json:"key,omitempty"` + Default string `json:"default,omitempty"` + Extra string `json:"extra,omitempty"` + Children []DatabaseColumn `json:"children,omitempty"` +} From 1197c05316ebc4b9fe3b329c9b51ab6e6135497f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 6 Aug 2024 15:35:36 +0800 Subject: [PATCH 093/106] =?UTF-8?q?=F0=9F=9A=80=20feat:=20add=20IsDefault?= =?UTF-8?q?=20field=20to=20DatabaseV2=20model=20=F0=9F=94=A8=20refactor:?= =?UTF-8?q?=20modify=20GetDatabaseList,=20GetDatabaseById,=20PostDatabaseT?= =?UTF-8?q?estConnection,=20and=20GetDatabaseMetadata=20functions=20?= =?UTF-8?q?=F0=9F=90=9E=20fix:=20GetDatabaseById=20and=20GetMetadata=20fun?= =?UTF-8?q?ctions=20in=20MongoService=20=F0=9F=94=A7=20chore:=20add=20GetD?= =?UTF-8?q?efaultDatabase=20and=20GetDatabaseById=20functions=20to=20utils?= =?UTF-8?q?=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/controllers/database_v2.go | 117 ------------ core/ds/service_v2.go | 271 --------------------------- core/entity/database.go | 13 ++ core/models/models/v2/database_v2.go | 11 +- 4 files changed, 18 insertions(+), 394 deletions(-) delete mode 100644 core/controllers/database_v2.go delete mode 100644 core/ds/service_v2.go diff --git a/core/controllers/database_v2.go b/core/controllers/database_v2.go deleted file mode 100644 index 771cee8d7..000000000 --- a/core/controllers/database_v2.go +++ /dev/null @@ -1,117 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/ds" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -func PostDatabase(c *gin.Context) { - // data source - var payload struct { - Name string `json:"name"` - Type string `json:"type"` - Description string `json:"description"` - Host string `json:"host"` - Port int `json:"port"` - Url string `json:"url"` - Hosts []string `json:"hosts"` - Database string `json:"database"` - Username string `json:"username"` - Password string `json:"-,omitempty"` - ConnectType string `json:"connect_type"` - Status string `json:"status"` - Error string `json:"error"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - u := GetUserFromContextV2(c) - - // add data source to db - dataSource := models.DatabaseV2{ - Name: payload.Name, - Type: payload.Type, - Description: payload.Description, - Host: payload.Host, - Port: payload.Port, - URI: payload.Url, - Database: payload.Database, - Username: payload.Username, - Password: payload.Password, - ConnectType: payload.ConnectType, - Status: payload.Status, - Error: payload.Error, - } - dataSource.SetCreated(u.Id) - dataSource.SetUpdated(u.Id) - id, err := service.NewModelServiceV2[models.DatabaseV2]().InsertOne(dataSource) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - dataSource.Id = id - - // check data source status - go func() { - _ = ds.GetDataSourceServiceV2().CheckStatus(id) - }() - - HandleSuccessWithData(c, dataSource) -} - -func PutDatabaseById(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - var dataSource models.DatabaseV2 - if err := c.ShouldBindJSON(&dataSource); err != nil { - HandleErrorBadRequest(c, err) - return - } - - err = service.NewModelServiceV2[models.DatabaseV2]().ReplaceById(id, dataSource) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { - _ = ds.GetDataSourceServiceV2().CheckStatus(id) - }() -} - -func PostDatabaseChangePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload struct { - Password string `json:"password"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if payload.Password == "" { - HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields) - return - } - u := GetUserFromContextV2(c) - if err := ds.GetDataSourceServiceV2().ChangePassword(id, payload.Password, u.Id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} diff --git a/core/ds/service_v2.go b/core/ds/service_v2.go deleted file mode 100644 index fbf8fe247..000000000 --- a/core/ds/service_v2.go +++ /dev/null @@ -1,271 +0,0 @@ -package ds - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - constants2 "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/models/v2" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/utils" - utils2 "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" - "sync" - "time" -) - -type ServiceV2 struct { - // internals - timeout time.Duration - monitorInterval time.Duration - stopped bool -} - -func (svc *ServiceV2) Init() { - // result service registry - reg := result.GetResultServiceRegistry() - - // register result services - reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService) - reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService) - reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService) - reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService) - reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService) - reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService) - reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService) - reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService) -} - -func (svc *ServiceV2) Start() { - // start monitoring - go svc.Monitor() -} - -func (svc *ServiceV2) Wait() { - utils.DefaultWait() -} - -func (svc *ServiceV2) Stop() { - svc.stopped = true -} - -func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { - dataSource, err := service.NewModelServiceV2[models.DatabaseV2]().GetById(id) - if err != nil { - return err - } - dataSource.Password, err = utils.EncryptAES(password) - if err != nil { - return err - } - dataSource.SetUpdated(by) - err = service.NewModelServiceV2[models.DatabaseV2]().ReplaceById(id, *dataSource) - if err != nil { - return err - } - return nil -} - -func (svc *ServiceV2) Monitor() { - for { - // return if stopped - if svc.stopped { - return - } - - // monitor - if err := svc.monitor(); err != nil { - trace.PrintError(err) - } - - // wait - time.Sleep(svc.monitorInterval) - } -} - -func (svc *ServiceV2) CheckStatus(id primitive.ObjectID) (err error) { - ds, err := service.NewModelServiceV2[models.DatabaseV2]().GetById(id) - if err != nil { - return err - } - return svc.checkStatus(ds, true) -} - -func (svc *ServiceV2) SetTimeout(duration time.Duration) { - svc.timeout = duration -} - -func (svc *ServiceV2) SetMonitorInterval(duration time.Duration) { - svc.monitorInterval = duration -} - -func (svc *ServiceV2) monitor() (err error) { - // start - tic := time.Now() - log.Debugf("[DataSourceService] start monitoring") - - // data source list - dataSources, err := service.NewModelServiceV2[models.DatabaseV2]().GetMany(nil, nil) - if err != nil { - return err - } - - // waiting group - wg := sync.WaitGroup{} - wg.Add(len(dataSources)) - - // iterate data source list - for _, ds := range dataSources { - // async operation - go func(ds *models.DatabaseV2) { - // check status and save - _ = svc.checkStatus(ds, true) - - // release - wg.Done() - }(&ds) - } - - // wait - wg.Wait() - - // finish - toc := time.Now() - log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds()) - - return nil -} - -func (svc *ServiceV2) checkStatus(ds *models.DatabaseV2, save bool) (err error) { - // check status - if err := svc._checkStatus(ds); err != nil { - ds.Status = constants2.DataSourceStatusOffline - ds.Error = err.Error() - } else { - ds.Status = constants2.DataSourceStatusOnline - ds.Error = "" - } - - // save - if save { - return svc._save(ds) - } - - return nil -} - -func (svc *ServiceV2) _save(ds *models.DatabaseV2) (err error) { - log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error) - return service.NewModelServiceV2[models.DatabaseV2]().ReplaceById(ds.Id, *ds) -} - -func (svc *ServiceV2) _checkStatus(ds *models.DatabaseV2) (err error) { - switch ds.Type { - case constants.DataSourceTypeMongo: - _, err := utils2.GetMongoClientWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - case constants.DataSourceTypeMysql: - s, err := utils2.GetMysqlSessionWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - err := s.Close() - if err != nil { - return err - } - } - case constants.DataSourceTypePostgresql: - s, err := utils2.GetPostgresqlSessionWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - err := s.Close() - if err != nil { - return err - } - } - case constants.DataSourceTypeMssql: - s, err := utils2.GetMssqlSessionWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - err := s.Close() - if err != nil { - return err - } - } - case constants.DataSourceTypeSqlite: - s, err := utils2.GetSqliteSessionWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - err := s.Close() - if err != nil { - return err - } - } - case constants.DataSourceTypeCockroachdb: - s, err := utils2.GetCockroachdbSessionWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - if s != nil { - err := s.Close() - if err != nil { - return err - } - } - case constants.DataSourceTypeElasticSearch: - _, err := utils2.GetElasticsearchClientWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - case constants.DataSourceTypeKafka: - c, err := utils2.GetKafkaConnectionWithTimeoutV2(ds, svc.timeout) - if err != nil { - return err - } - if c != nil { - err := c.Close() - if err != nil { - return err - } - } - default: - log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type) - } - return nil -} - -func NewDataSourceServiceV2() *ServiceV2 { - // service - svc := &ServiceV2{ - monitorInterval: 15 * time.Second, - timeout: 10 * time.Second, - } - - // initialize - svc.Init() - - // start - svc.Start() - - return svc -} - -var _dsSvcV2 *ServiceV2 - -func GetDataSourceServiceV2() *ServiceV2 { - if _dsSvcV2 != nil { - return _dsSvcV2 - } - _dsSvcV2 = NewDataSourceServiceV2() - return _dsSvcV2 -} diff --git a/core/entity/database.go b/core/entity/database.go index e0033781c..0dee08558 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -12,6 +12,7 @@ type Database struct { type DatabaseTable struct { Name string `json:"name"` Columns []DatabaseColumn `json:"columns"` + Indexes []DatabaseIndex `json:"indexes"` } type DatabaseColumn struct { @@ -23,3 +24,15 @@ type DatabaseColumn struct { Extra string `json:"extra,omitempty"` Children []DatabaseColumn `json:"children,omitempty"` } + +type DatabaseIndex struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Columns []DatabaseIndexColumn `json:"columns"` + Unique bool `json:"unique"` +} + +type DatabaseIndexColumn struct { + Name string `json:"name"` + Order int `json:"order"` +} diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go index 673bd92ae..1d084a821 100644 --- a/core/models/models/v2/database_v2.go +++ b/core/models/models/v2/database_v2.go @@ -6,20 +6,19 @@ type DatabaseV2 struct { any `collection:"databases"` BaseModelV2[DatabaseV2] `bson:",inline"` Name string `json:"name" bson:"name"` - DataSource string `json:"data_source" bson:"data_source"` - Type string `json:"type" bson:"type"` Description string `json:"description" bson:"description"` + DataSource string `json:"data_source" bson:"data_source"` Host string `json:"host" bson:"host"` Port int `json:"port" bson:"port"` URI string `json:"uri,omitempty" bson:"uri,omitempty"` Database string `json:"database,omitempty" bson:"database,omitempty"` Username string `json:"username,omitempty" bson:"username,omitempty"` Password string `json:"-,omitempty" bson:"password,omitempty"` - ConnectType string `json:"connect_type,omitempty" bson:"connect_type,omitempty"` Status string `json:"status" bson:"status"` Error string `json:"error" bson:"error"` Active bool `json:"active" bson:"active"` ActiveAt time.Time `json:"active_ts" bson:"active_ts"` + IsDefault bool `json:"is_default" bson:"-"` MongoParams *struct { AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` @@ -36,11 +35,11 @@ type DatabaseV2 struct { } `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty"` CassandraParams *struct { Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty"` - } + } `json:"cassandra_params,omitempty" bson:"cassandra_params,omitempty"` HiveParams *struct { Auth string `json:"auth,omitempty" bson:"auth,omitempty"` - } + } `json:"hive_params,omitempty" bson:"hive_params,omitempty"` RedisParams *struct { DB int `json:"db,omitempty" bson:"db,omitempty"` - } + } `json:"redis_params,omitempty" bson:"redis_params,omitempty"` } From 6986a10246e339cee1dcd3c5f8e18bc285123e0a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 7 Aug 2024 18:05:35 +0800 Subject: [PATCH 094/106] feat: updated password handling logic --- core/models/models/v2/database_v2.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go index 1d084a821..e09a6fb36 100644 --- a/core/models/models/v2/database_v2.go +++ b/core/models/models/v2/database_v2.go @@ -1,6 +1,8 @@ package models -import "time" +import ( + "time" +) type DatabaseV2 struct { any `collection:"databases"` @@ -13,7 +15,8 @@ type DatabaseV2 struct { URI string `json:"uri,omitempty" bson:"uri,omitempty"` Database string `json:"database,omitempty" bson:"database,omitempty"` Username string `json:"username,omitempty" bson:"username,omitempty"` - Password string `json:"-,omitempty" bson:"password,omitempty"` + Password string `json:"password,omitempty" bson:"-"` + EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` Status string `json:"status" bson:"status"` Error string `json:"error" bson:"error"` Active bool `json:"active" bson:"active"` From fdea835d53885ea9815c7dd19fae4040d8f955b9 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 18 Aug 2024 22:01:55 +0800 Subject: [PATCH 095/106] feat: added modify table --- core/entity/database.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/core/entity/database.go b/core/entity/database.go index 0dee08558..bd89b7ce1 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -16,13 +16,14 @@ type DatabaseTable struct { } type DatabaseColumn struct { - Name string `json:"name"` - Type string `json:"type"` - Null bool `json:"null,omitempty"` - Key string `json:"key,omitempty"` - Default string `json:"default,omitempty"` - Extra string `json:"extra,omitempty"` - Children []DatabaseColumn `json:"children,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + PrimaryKey bool `json:"primary_key,omitempty"` + Null bool `json:"null,omitempty"` + Key string `json:"key,omitempty"` + Default string `json:"default,omitempty"` + Extra string `json:"extra,omitempty"` + Children []DatabaseColumn `json:"children,omitempty"` } type DatabaseIndex struct { @@ -36,3 +37,11 @@ type DatabaseIndexColumn struct { Name string `json:"name"` Order int `json:"order"` } + +func (col *DatabaseIndexColumn) OrderString() string { + if col.Order < 0 { + return "DESC" + } else { + return "ASC" + } +} From 08d1acb15eda01ac11c0a870ca48f672552e7a5b Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 26 Aug 2024 18:33:46 +0800 Subject: [PATCH 096/106] feat: added hash to models --- core/entity/database.go | 2 ++ core/utils/hash.go | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 core/utils/hash.go diff --git a/core/entity/database.go b/core/entity/database.go index bd89b7ce1..eccb7d4f5 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -24,6 +24,7 @@ type DatabaseColumn struct { Default string `json:"default,omitempty"` Extra string `json:"extra,omitempty"` Children []DatabaseColumn `json:"children,omitempty"` + Hash string `json:"hash,omitempty"` } type DatabaseIndex struct { @@ -31,6 +32,7 @@ type DatabaseIndex struct { Type string `json:"type,omitempty"` Columns []DatabaseIndexColumn `json:"columns"` Unique bool `json:"unique"` + Hash string `json:"hash,omitempty"` } type DatabaseIndexColumn struct { diff --git a/core/utils/hash.go b/core/utils/hash.go new file mode 100644 index 000000000..8fcbf0a15 --- /dev/null +++ b/core/utils/hash.go @@ -0,0 +1,12 @@ +package utils + +import "encoding/json" + +func GetObjectHash(obj any) string { + data, _ := json.Marshal(obj) + if data == nil { + // random hash + return EncryptMd5(NewUUIDString()) + } + return EncryptMd5(string(data)) +} From a8ab6af7b2e3afcfc1628841768fc5184fbff9d5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Thu, 29 Aug 2024 17:21:18 +0800 Subject: [PATCH 097/106] feat: updated database services --- core/entity/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/entity/database.go b/core/entity/database.go index eccb7d4f5..0b9a37d6d 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -19,9 +19,9 @@ type DatabaseColumn struct { Name string `json:"name"` Type string `json:"type"` PrimaryKey bool `json:"primary_key,omitempty"` - Null bool `json:"null,omitempty"` + Null bool `json:"null"` Key string `json:"key,omitempty"` - Default string `json:"default,omitempty"` + Default string `json:"default"` Extra string `json:"extra,omitempty"` Children []DatabaseColumn `json:"children,omitempty"` Hash string `json:"hash,omitempty"` From f5666897386ec119c950f59428fc3446a96d410e Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 2 Sep 2024 19:01:58 +0800 Subject: [PATCH 098/106] feat: updated database services --- core/entity/database.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/core/entity/database.go b/core/entity/database.go index 0b9a37d6d..073fc6a82 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -16,23 +16,27 @@ type DatabaseTable struct { } type DatabaseColumn struct { - Name string `json:"name"` - Type string `json:"type"` - PrimaryKey bool `json:"primary_key,omitempty"` - Null bool `json:"null"` - Key string `json:"key,omitempty"` - Default string `json:"default"` - Extra string `json:"extra,omitempty"` - Children []DatabaseColumn `json:"children,omitempty"` - Hash string `json:"hash,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + PrimaryKey bool `json:"primary_key,omitempty"` + Null bool `json:"null"` + Key string `json:"key,omitempty"` + Default string `json:"default"` + Extra string `json:"extra,omitempty"` + Children []DatabaseColumn `json:"children,omitempty"` + Hash string `json:"hash,omitempty"` + OriginalName string `json:"original_name,omitempty"` + Status string `json:"status,omitempty"` } type DatabaseIndex struct { - Name string `json:"name"` - Type string `json:"type,omitempty"` - Columns []DatabaseIndexColumn `json:"columns"` - Unique bool `json:"unique"` - Hash string `json:"hash,omitempty"` + Name string `json:"name"` + Type string `json:"type,omitempty"` + Columns []DatabaseIndexColumn `json:"columns"` + Unique bool `json:"unique"` + Hash string `json:"hash,omitempty"` + OriginalName string `json:"original_name,omitempty"` + Status string `json:"status,omitempty"` } type DatabaseIndexColumn struct { From aa0cb96936f9f5ddcc4ae3f9072c9ff1a2b739f4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 4 Sep 2024 17:07:32 +0800 Subject: [PATCH 099/106] feat: added index operations for mysql service --- core/entity/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/entity/database.go b/core/entity/database.go index 073fc6a82..1c067dd43 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -19,7 +19,7 @@ type DatabaseColumn struct { Name string `json:"name"` Type string `json:"type"` PrimaryKey bool `json:"primary_key,omitempty"` - Null bool `json:"null"` + NotNull bool `json:"not_null"` Key string `json:"key,omitempty"` Default string `json:"default"` Extra string `json:"extra,omitempty"` From 4807c0a497e993f7bfe6a00453827010af215af2 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 11 Sep 2024 17:45:04 +0800 Subject: [PATCH 100/106] feat: added database table rows crud --- core/entity/database.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/core/entity/database.go b/core/entity/database.go index 1c067dd43..009f871e0 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -16,17 +16,18 @@ type DatabaseTable struct { } type DatabaseColumn struct { - Name string `json:"name"` - Type string `json:"type"` - PrimaryKey bool `json:"primary_key,omitempty"` - NotNull bool `json:"not_null"` - Key string `json:"key,omitempty"` - Default string `json:"default"` - Extra string `json:"extra,omitempty"` - Children []DatabaseColumn `json:"children,omitempty"` - Hash string `json:"hash,omitempty"` - OriginalName string `json:"original_name,omitempty"` - Status string `json:"status,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + PrimaryKey bool `json:"primary_key,omitempty"` + NotNull bool `json:"not_null"` + Key string `json:"key,omitempty"` + Default string `json:"default"` + Extra string `json:"extra,omitempty"` + AutoIncrement bool `json:"auto_increment,omitempty"` + Children []DatabaseColumn `json:"children,omitempty"` + Hash string `json:"hash,omitempty"` + OriginalName string `json:"original_name,omitempty"` + Status string `json:"status,omitempty"` } type DatabaseIndex struct { From a3e1751eef7f7e862f16c4f784048cd3a61fbada Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 23 Sep 2024 13:23:18 +0800 Subject: [PATCH 101/106] chore: ci cleanup --- .github/workflows/qodana_code_quality.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/qodana_code_quality.yml diff --git a/.github/workflows/qodana_code_quality.yml b/.github/workflows/qodana_code_quality.yml deleted file mode 100644 index ebfb9343b..000000000 --- a/.github/workflows/qodana_code_quality.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Qodana -on: - workflow_dispatch: - pull_request: - push: - branches: - - main - - develop - -jobs: - qodana: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: 'Qodana Scan' - uses: JetBrains/qodana-action@v2024.1 - env: - QODANA_TOKEN: ${{ secrets.QODANA_TOKEN }} \ No newline at end of file From 204021d962830e9bba509fd1fd0a494582d851d4 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 23 Sep 2024 15:33:26 +0800 Subject: [PATCH 102/106] feat: updated database model --- core/entity/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/entity/database.go b/core/entity/database.go index 009f871e0..c3334abb1 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -18,7 +18,7 @@ type DatabaseTable struct { type DatabaseColumn struct { Name string `json:"name"` Type string `json:"type"` - PrimaryKey bool `json:"primary_key,omitempty"` + Primary bool `json:"primary,omitempty"` NotNull bool `json:"not_null"` Key string `json:"key,omitempty"` Default string `json:"default"` From 64d664b3071c2ed36d901e21e41a968fdb54288f Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sun, 29 Sep 2024 16:54:22 +0800 Subject: [PATCH 103/106] feat: updated database model --- core/entity/database.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/entity/database.go b/core/entity/database.go index c3334abb1..9c0d75696 100644 --- a/core/entity/database.go +++ b/core/entity/database.go @@ -19,9 +19,9 @@ type DatabaseColumn struct { Name string `json:"name"` Type string `json:"type"` Primary bool `json:"primary,omitempty"` - NotNull bool `json:"not_null"` + NotNull bool `json:"not_null,omitempty"` Key string `json:"key,omitempty"` - Default string `json:"default"` + Default string `json:"default,omitempty"` Extra string `json:"extra,omitempty"` AutoIncrement bool `json:"auto_increment,omitempty"` Children []DatabaseColumn `json:"children,omitempty"` @@ -52,3 +52,10 @@ func (col *DatabaseIndexColumn) OrderString() string { return "ASC" } } + +type DatabaseQueryResults struct { + Columns []DatabaseColumn `json:"columns,omitempty"` + Rows []map[string]interface{} `json:"rows,omitempty"` + Output string `json:"output,omitempty"` + Error string `json:"error,omitempty"` +} From e15c3c9b4f96dd0a2006895263ae1450640cba31 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 5 Oct 2024 14:46:38 +0800 Subject: [PATCH 104/106] feat: added performance monitoring for database --- .../server/model_base_service_v2_server.go | 1 + core/models/common/index_service_v2.go | 29 ++++ core/models/models/v2/database_metric_v2.go | 23 +++ core/models/service/base_service_v2.go | 137 +++++++++++++++++- 4 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 core/models/models/v2/database_metric_v2.go diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index d067e11ca..d695d1de0 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -19,6 +19,7 @@ var ( *new(models2.TestModelV2), *new(models2.DataCollectionV2), *new(models2.DatabaseV2), + *new(models2.DatabaseMetricV2), *new(models2.DependencyV2), *new(models2.DependencyLogV2), *new(models2.DependencySettingV2), diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go index 20c278f26..2247649f5 100644 --- a/core/models/common/index_service_v2.go +++ b/core/models/common/index_service_v2.go @@ -202,4 +202,33 @@ func CreateIndexesV2() { }, }, }) + + // databases + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"data_source_id", 1}, + }, + }, + }) + + // database metrics + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"created_ts", -1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30), + }, + { + Keys: bson.D{ + {"database_id", 1}, + }, + }, + { + Keys: bson.D{ + {"type", 1}, + }, + }, + }) } diff --git a/core/models/models/v2/database_metric_v2.go b/core/models/models/v2/database_metric_v2.go new file mode 100644 index 000000000..147c76d72 --- /dev/null +++ b/core/models/models/v2/database_metric_v2.go @@ -0,0 +1,23 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DatabaseMetricV2 struct { + any `collection:"database_metrics"` + BaseModelV2[DatabaseMetricV2] `bson:",inline"` + DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + Connections int `json:"connections" bson:"connections"` + QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` + TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` + CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` + ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` + LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` +} diff --git a/core/models/service/base_service_v2.go b/core/models/service/base_service_v2.go index 9ebe4c91c..3d03a28a3 100644 --- a/core/models/service/base_service_v2.go +++ b/core/models/service/base_service_v2.go @@ -1,13 +1,16 @@ package service import ( + "context" "fmt" + "go.mongodb.org/mongo-driver/mongo/options" + "reflect" + "sync" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" - "sync" ) var ( @@ -30,6 +33,15 @@ func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err erro return &result, nil } +func (svc *ModelServiceV2[T]) GetByIdContext(ctx context.Context, id primitive.ObjectID) (model *T, err error) { + var result T + err = svc.col.GetCollection().FindOne(ctx, bson.M{"_id": id}).Decode(&result) + if err != nil { + return nil, err + } + return &result, nil +} + func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) { var result T err = svc.col.Find(query, options).One(&result) @@ -39,6 +51,25 @@ func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) ( return &result, nil } +func (svc *ModelServiceV2[T]) GetOneContext(ctx context.Context, query bson.M, opts *mongo.FindOptions) (model *T, err error) { + var result T + _opts := &options.FindOneOptions{} + if opts != nil { + if opts.Skip != 0 { + skipInt64 := int64(opts.Skip) + _opts.Skip = &skipInt64 + } + if opts.Sort != nil { + _opts.Sort = opts.Sort + } + } + err = svc.col.GetCollection().FindOne(ctx, query, _opts).Decode(&result) + if err != nil { + return nil, err + } + return &result, nil +} + func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) { var result []T err = svc.col.Find(query, options).All(&result) @@ -48,44 +79,115 @@ func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) return result, nil } +func (svc *ModelServiceV2[T]) GetManyContext(ctx context.Context, query bson.M, opts *mongo.FindOptions) (models []T, err error) { + var result []T + _opts := &options.FindOptions{} + if opts != nil { + if opts.Skip != 0 { + skipInt64 := int64(opts.Skip) + _opts.Skip = &skipInt64 + } + if opts.Limit != 0 { + limitInt64 := int64(opts.Limit) + _opts.Limit = &limitInt64 + } + if opts.Sort != nil { + _opts.Sort = opts.Sort + } + } + cur, err := svc.col.GetCollection().Find(ctx, query, _opts) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + for cur.Next(ctx) { + var model T + if err := cur.Decode(&model); err != nil { + return nil, err + } + result = append(result, model) + } + return result, nil +} + func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) { return svc.col.DeleteId(id) } +func (svc *ModelServiceV2[T]) DeleteByIdContext(ctx context.Context, id primitive.ObjectID) (err error) { + _, err = svc.col.GetCollection().DeleteOne(ctx, bson.M{"_id": id}) + return err +} + func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) { _, err = svc.col.GetCollection().DeleteOne(svc.col.GetContext(), query) return err } +func (svc *ModelServiceV2[T]) DeleteOneContext(ctx context.Context, query bson.M) (err error) { + _, err = svc.col.GetCollection().DeleteOne(ctx, query) + return err +} + func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) { _, err = svc.col.GetCollection().DeleteMany(svc.col.GetContext(), query, nil) return err } +func (svc *ModelServiceV2[T]) DeleteManyContext(ctx context.Context, query bson.M) (err error) { + _, err = svc.col.GetCollection().DeleteMany(ctx, query, nil) + return err +} + func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) { return svc.col.UpdateId(id, update) } +func (svc *ModelServiceV2[T]) UpdateByIdContext(ctx context.Context, id primitive.ObjectID, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateOne(ctx, bson.M{"_id": id}, update) + return err +} + func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) { _, err = svc.col.GetCollection().UpdateOne(svc.col.GetContext(), query, update) return err } +func (svc *ModelServiceV2[T]) UpdateOneContext(ctx context.Context, query bson.M, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateOne(ctx, query, update) + return err +} + func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) { _, err = svc.col.GetCollection().UpdateMany(svc.col.GetContext(), query, update) return err } +func (svc *ModelServiceV2[T]) UpdateManyContext(ctx context.Context, query bson.M, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateMany(ctx, query, update) + return err +} + func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) { _, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), bson.M{"_id": id}, model) return err } +func (svc *ModelServiceV2[T]) ReplaceByIdContext(ctx context.Context, id primitive.ObjectID, model T) (err error) { + _, err = svc.col.GetCollection().ReplaceOne(ctx, bson.M{"_id": id}, model) + return err +} + func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) { _, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), query, model) return err } +func (svc *ModelServiceV2[T]) ReplaceOneContext(ctx context.Context, query bson.M, model T) (err error) { + _, err = svc.col.GetCollection().ReplaceOne(ctx, query, model) + return err +} + func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) { m := any(&model).(interfaces.Model) if m.GetId().IsZero() { @@ -98,6 +200,18 @@ func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err err return res.InsertedID.(primitive.ObjectID), nil } +func (svc *ModelServiceV2[T]) InsertOneContext(ctx context.Context, model T) (id primitive.ObjectID, err error) { + m := any(&model).(interfaces.Model) + if m.GetId().IsZero() { + m.SetId(primitive.NewObjectID()) + } + res, err := svc.col.GetCollection().InsertOne(ctx, m) + if err != nil { + return primitive.NilObjectID, err + } + return res.InsertedID.(primitive.ObjectID), nil +} + func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) { var _models []any for _, model := range models { @@ -117,6 +231,25 @@ func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, return ids, nil } +func (svc *ModelServiceV2[T]) InsertManyContext(ctx context.Context, models []T) (ids []primitive.ObjectID, err error) { + var _models []any + for _, model := range models { + m := any(&model).(interfaces.Model) + if m.GetId().IsZero() { + m.SetId(primitive.NewObjectID()) + } + _models = append(_models, m) + } + res, err := svc.col.GetCollection().InsertMany(ctx, _models) + if err != nil { + return nil, err + } + for _, v := range res.InsertedIDs { + ids = append(ids, v.(primitive.ObjectID)) + } + return ids, nil +} + func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) { return svc.col.Count(query) } From 0ed21509015a1ea5e91568292db6ec2f33635b78 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 7 Oct 2024 17:19:09 +0800 Subject: [PATCH 105/106] feat: added performance monitoring for elasticsearch --- core/models/models/v2/database_metric_v2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/models/models/v2/database_metric_v2.go b/core/models/models/v2/database_metric_v2.go index 147c76d72..0d5b00817 100644 --- a/core/models/models/v2/database_metric_v2.go +++ b/core/models/models/v2/database_metric_v2.go @@ -6,6 +6,7 @@ type DatabaseMetricV2 struct { any `collection:"database_metrics"` BaseModelV2[DatabaseMetricV2] `bson:",inline"` DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` TotalMemory uint64 `json:"total_memory" bson:"total_memory"` AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` UsedMemory uint64 `json:"used_memory" bson:"used_memory"` From 4f52936ad88a817b1e4bf9115f54894b41096c59 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Tue, 8 Oct 2024 18:41:36 +0800 Subject: [PATCH 106/106] refactor: integrated database services into task data insert --- core/controllers/spider_v2.go | 60 +--------- core/controllers/task_v2.go | 44 +++---- core/{ => database}/entity/database.go | 0 .../interfaces/database_registry_service.go | 11 ++ core/database/interfaces/database_service.go | 27 +++++ core/database/registry_service.go | 15 +++ core/grpc/server/task_server_v2.go | 2 +- core/models/models/v2/spider_v2.go | 4 +- core/result/service.go | 8 +- core/result/test/base.go | 76 ------------ core/result/test/service_test.go | 67 ----------- core/task/stats/service_v2.go | 111 +++++++++++++----- 12 files changed, 161 insertions(+), 264 deletions(-) rename core/{ => database}/entity/database.go (100%) create mode 100644 core/database/interfaces/database_registry_service.go create mode 100644 core/database/interfaces/database_service.go create mode 100644 core/database/registry_service.go delete mode 100644 core/result/test/base.go delete mode 100644 core/result/test/service_test.go diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index bcd0b5aab..76d270380 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -3,7 +3,6 @@ package controllers import ( "errors" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" models2 "github.com/crawlab-team/crawlab/core/models/models/v2" @@ -48,8 +47,8 @@ func GetSpiderById(c *gin.Context) { } } - // data collection - if !s.ColId.IsZero() { + // data collection (compatible to old version) # TODO: remove in the future + if s.ColName == "" && !s.ColId.IsZero() { col, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(s.ColId) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { @@ -252,12 +251,6 @@ func PostSpider(c *gin.Context) { return } - // upsert data collection - if err := upsertSpiderDataCollection(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - // user u := GetUserFromContextV2(c) @@ -311,12 +304,6 @@ func PutSpiderById(c *gin.Context) { return } - // upsert data collection - if err := upsertSpiderDataCollection(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - u := GetUserFromContextV2(c) modelSvc := service.NewModelServiceV2[models2.SpiderV2]() @@ -773,49 +760,6 @@ func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err return getSpiderFsSvc(s) } -func upsertSpiderDataCollection(s *models2.SpiderV2) (err error) { - modelSvc := service.NewModelServiceV2[models2.DataCollectionV2]() - if s.ColId.IsZero() { - // validate - if s.ColName == "" { - return errors.New("data collection name is required") - } - // no id - dc, err := modelSvc.GetOne(bson.M{"name": s.ColName}, nil) - if err != nil { - if errors.Is(err, mongo2.ErrNoDocuments) { - // not exists, add new - dc = &models2.DataCollectionV2{Name: s.ColName} - dcId, err := modelSvc.InsertOne(*dc) - if err != nil { - return err - } - dc.SetId(dcId) - } else { - // error - return err - } - } - s.ColId = dc.Id - - // create index - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) - } else { - // with id - dc, err := modelSvc.GetById(s.ColId) - if err != nil { - return err - } - s.ColId = dc.Id - } - return nil -} - -func UpsertSpiderDataCollection(s *models2.SpiderV2) (err error) { - return upsertSpiderDataCollection(s) -} - func getSpiderRootPath(c *gin.Context) (rootPath string, err error) { // spider id id, err := primitive.ObjectIDFromHex(c.Param("id")) diff --git a/core/controllers/task_v2.go b/core/controllers/task_v2.go index 5284210a4..4334d008f 100644 --- a/core/controllers/task_v2.go +++ b/core/controllers/task_v2.go @@ -5,7 +5,7 @@ import ( log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/spider/admin" @@ -34,7 +34,7 @@ func GetTaskById(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorNotFound(c, err) return @@ -45,7 +45,7 @@ func GetTaskById(c *gin.Context) { } // spider - t.Spider, _ = service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId) + t.Spider, _ = service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId) // skip if task status is pending if t.Status == constants.TaskStatusPending { @@ -54,7 +54,7 @@ func GetTaskById(c *gin.Context) { } // task stat - t.Stat, _ = service.NewModelServiceV2[models2.TaskStatV2]().GetById(id) + t.Stat, _ = service.NewModelServiceV2[models.TaskStatV2]().GetById(id) HandleSuccessWithData(c, t) } @@ -62,7 +62,7 @@ func GetTaskById(c *gin.Context) { func GetTaskList(c *gin.Context) { withStats := c.Query("stats") if withStats == "" { - NewControllerV2[models2.TaskV2]().GetList(c) + NewControllerV2[models.TaskV2]().GetList(c) return } @@ -72,7 +72,7 @@ func GetTaskList(c *gin.Context) { sort := MustGetSortOption(c) // get tasks - tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(query, &mongo.FindOptions{ + tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -101,14 +101,14 @@ func GetTaskList(c *gin.Context) { } // total count - total, err := service.NewModelServiceV2[models2.TaskV2]().Count(query) + total, err := service.NewModelServiceV2[models.TaskV2]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return } // stat list - stats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{ + stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ "_id": bson.M{ "$in": taskIds, }, @@ -119,13 +119,13 @@ func GetTaskList(c *gin.Context) { } // cache stat list to dict - statsDict := map[primitive.ObjectID]models2.TaskStatV2{} + statsDict := map[primitive.ObjectID]models.TaskStatV2{} for _, s := range stats { statsDict[s.Id] = s } // spider list - spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(bson.M{ + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{ "_id": bson.M{ "$in": spiderIds, }, @@ -136,7 +136,7 @@ func GetTaskList(c *gin.Context) { } // cache spider list to dict - spiderDict := map[primitive.ObjectID]models2.SpiderV2{} + spiderDict := map[primitive.ObjectID]models.SpiderV2{} for _, s := range spiders { spiderDict[s.Id] = s } @@ -170,22 +170,22 @@ func DeleteTaskById(c *gin.Context) { // delete in db if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete task - _, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id) + _, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) if err != nil { return err } - err = service.NewModelServiceV2[models2.TaskV2]().DeleteById(id) + err = service.NewModelServiceV2[models.TaskV2]().DeleteById(id) if err != nil { return err } // delete task stat - _, err = service.NewModelServiceV2[models2.TaskStatV2]().GetById(id) + _, err = service.NewModelServiceV2[models.TaskStatV2]().GetById(id) if err != nil { log2.Warnf("delete task stat error: %s", err.Error()) return nil } - err = service.NewModelServiceV2[models2.TaskStatV2]().DeleteById(id) + err = service.NewModelServiceV2[models.TaskStatV2]().DeleteById(id) if err != nil { log2.Warnf("delete task stat error: %s", err.Error()) return nil @@ -217,7 +217,7 @@ func DeleteList(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { // delete tasks - if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -226,7 +226,7 @@ func DeleteList(c *gin.Context) { } // delete task stats - if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -261,7 +261,7 @@ func DeleteList(c *gin.Context) { func PostTaskRun(c *gin.Context) { // task - var t models2.TaskV2 + var t models.TaskV2 if err := c.ShouldBindJSON(&t); err != nil { HandleErrorBadRequest(c, err) return @@ -274,7 +274,7 @@ func PostTaskRun(c *gin.Context) { } // spider - s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId) + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(t.SpiderId) if err != nil { HandleErrorInternalServerError(c, err) return @@ -319,7 +319,7 @@ func PostTaskRestart(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -363,7 +363,7 @@ func PostTaskCancel(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -446,7 +446,7 @@ func GetTaskData(c *gin.Context) { } // task - t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/entity/database.go b/core/database/entity/database.go similarity index 100% rename from core/entity/database.go rename to core/database/entity/database.go diff --git a/core/database/interfaces/database_registry_service.go b/core/database/interfaces/database_registry_service.go new file mode 100644 index 000000000..5fc666828 --- /dev/null +++ b/core/database/interfaces/database_registry_service.go @@ -0,0 +1,11 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DatabaseRegistryService interface { + Start() + CheckStatus() + GetDatabaseService(id primitive.ObjectID) (res DatabaseService, err error) +} diff --git a/core/database/interfaces/database_service.go b/core/database/interfaces/database_service.go new file mode 100644 index 000000000..621829c2c --- /dev/null +++ b/core/database/interfaces/database_service.go @@ -0,0 +1,27 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab/core/database/entity" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DatabaseService interface { + TestConnection(id primitive.ObjectID) (err error) + GetMetadata(id primitive.ObjectID) (m *entity.DatabaseMetadata, err error) + GetMetadataAllDb(id primitive.ObjectID) (m *entity.DatabaseMetadata, err error) + CreateDatabase(id primitive.ObjectID, databaseName string) (err error) + DropDatabase(id primitive.ObjectID, databaseName string) (err error) + GetTableMetadata(id primitive.ObjectID, databaseName, tableName string) (table *entity.DatabaseTable, err error) + CreateTable(id primitive.ObjectID, databaseName string, table *entity.DatabaseTable) (err error) + ModifyTable(id primitive.ObjectID, databaseName string, table *entity.DatabaseTable) (err error) + DropTable(id primitive.ObjectID, databaseName, tableName string) (err error) + RenameTable(id primitive.ObjectID, databaseName, oldTableName, newTableName string) (err error) + GetColumnTypes(query string) (types []string) + ReadRows(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}, skip, limit int) ([]map[string]interface{}, int64, error) + CreateRow(id primitive.ObjectID, databaseName, tableName string, row map[string]interface{}) error + UpdateRow(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}, update map[string]interface{}) error + DeleteRow(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}) error + Query(id primitive.ObjectID, databaseName, query string) (results *entity.DatabaseQueryResults, err error) + GetCurrentMetric(id primitive.ObjectID) (m *models.DatabaseMetricV2, err error) +} diff --git a/core/database/registry_service.go b/core/database/registry_service.go new file mode 100644 index 000000000..e05b78094 --- /dev/null +++ b/core/database/registry_service.go @@ -0,0 +1,15 @@ +package database + +import ( + "github.com/crawlab-team/crawlab/core/database/interfaces" +) + +var serviceInstance interfaces.DatabaseRegistryService + +func SetDatabaseRegistryService(svc interfaces.DatabaseRegistryService) { + serviceInstance = svc +} + +func GetDatabaseRegistryService() interfaces.DatabaseRegistryService { + return serviceInstance +} diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index 3ae0ddb08..050a5cd15 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -214,7 +214,7 @@ func (svr TaskServerV2) handleInsertData(msg *grpc.StreamMessage) (err error) { if err != nil { return err } - var records []interface{} + var records []map[string]interface{} for _, d := range data.Records { res, ok := d[constants.TaskKey] if ok { diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go index 5e91baf47..b743ede21 100644 --- a/core/models/models/v2/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -8,8 +8,8 @@ type SpiderV2 struct { any `collection:"spiders"` BaseModelV2[SpiderV2] `bson:",inline"` Name string `json:"name" bson:"name"` // spider name - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id - ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future + ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id DataSource *DatabaseV2 `json:"data_source,omitempty" bson:"-"` // data source Description string `json:"description" bson:"description"` // description diff --git a/core/result/service.go b/core/result/service.go index d1e946851..5497f3a6c 100644 --- a/core/result/service.go +++ b/core/result/service.go @@ -38,7 +38,7 @@ func NewResultService(registryKey string, s *models.Spider) (svc2 interfaces.Res var store = sync.Map{} -func GetResultService(spiderId primitive.ObjectID, opts ...Option) (svc2 interfaces.ResultService, err error) { +func GetResultService(spiderId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { // model service modelSvc, err := service.GetService() if err != nil { @@ -51,12 +51,6 @@ func GetResultService(spiderId primitive.ObjectID, opts ...Option) (svc2 interfa return nil, trace.TraceError(err) } - // apply options - _opts := &Options{} - for _, opt := range opts { - opt(_opts) - } - // store key storeKey := s.ColId.Hex() + ":" + s.DataSourceId.Hex() diff --git a/core/result/test/base.go b/core/result/test/base.go deleted file mode 100644 index a224fa299..000000000 --- a/core/result/test/base.go +++ /dev/null @@ -1,76 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/db/mongo" - "go.uber.org/dig" - "testing" -) - -func init() { - T = NewTest() -} - -var T *Test - -type Test struct { - // dependencies - modelSvc service.ModelService - resultSvc interfaces.ResultService - - // test data - TestColName string - TestCol *mongo.Col - TestDc *models.DataCollection -} - -func (t *Test) Setup(t2 *testing.T) { - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.modelSvc.DropAll() -} - -func NewTest() *Test { - var err error - - // test - t := &Test{ - TestColName: "test_results", - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.NewService); err != nil { - panic(err) - } - if err := c.Invoke(func( - modelSvc service.ModelService, - ) { - t.modelSvc = modelSvc - }); err != nil { - panic(err) - } - - // data collection - t.TestDc = &models.DataCollection{ - Name: t.TestColName, - } - if err := delegate.NewModelDelegate(t.TestDc).Add(); err != nil { - panic(err) - } - t.TestCol = mongo.GetMongoCol(t.TestColName) - - // result service - t.resultSvc, err = result.GetResultService(t.TestDc.GetId()) - if err != nil { - panic(err) - } - - return t -} diff --git a/core/result/test/service_test.go b/core/result/test/service_test.go deleted file mode 100644 index b44faf4e8..000000000 --- a/core/result/test/service_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/stretchr/testify/require" - "testing" -) - -func TestResultService_GetList(t *testing.T) { - var err error - T.Setup(t) - - n := 1000 - var docs []interface{} - for i := 0; i < n; i++ { - d := &models.Result{ - "i": i, - } - docs = append(docs, d) - } - _, err = T.TestCol.InsertMany(docs) - require.Nil(t, err) - - // get all - results, err := T.resultSvc.List(nil, nil) - require.Nil(t, err) - require.Equal(t, n, len(results)) - - //query := bson.M{ - // "i": bson.M{ - // "$lt": n / 2, - // }, - //} - //results, err = T.resultSvc.List(query, nil) - //require.Nil(t, err) - //require.Equal(t, n/2, len(results)) -} - -func TestResultService_Count(t *testing.T) { - var err error - T.Setup(t) - - n := 1000 - var docs []interface{} - for i := 0; i < n; i++ { - d := &models.Result{ - "i": i, - } - docs = append(docs, d) - } - _, err = T.TestCol.InsertMany(docs) - require.Nil(t, err) - - // get all - total, err := T.resultSvc.Count(nil) - require.Nil(t, err) - require.Equal(t, n, total) - - //query := bson.M{ - // "i": bson.M{ - // "$lt": n / 2, - // }, - //} - //total, err = T.resultSvc.Count(query) - //require.Nil(t, err) - //require.Equal(t, n/2, total) -} diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go index 0637e1e16..d5a7d73a5 100644 --- a/core/task/stats/service_v2.go +++ b/core/task/stats/service_v2.go @@ -1,12 +1,16 @@ package stats import ( + log2 "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/database" + interfaces2 "github.com/crawlab-team/crawlab/core/database/interfaces" "github.com/crawlab-team/crawlab/core/interfaces" models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -14,15 +18,23 @@ import ( "time" ) +type databaseServiceItem struct { + taskId primitive.ObjectID + dbId primitive.ObjectID + dbSvc interfaces2.DatabaseService + tableName string + time time.Time +} + type ServiceV2 struct { // dependencies nodeCfgSvc interfaces.NodeConfigService // internals - mu sync.Mutex - resultServices sync.Map - rsTtl time.Duration - logDriver log.Driver + mu sync.Mutex + databaseServiceItems map[string]*databaseServiceItem + databaseServiceTll time.Duration + logDriver log.Driver } func (svc *ServiceV2) Init() (err error) { @@ -30,15 +42,39 @@ func (svc *ServiceV2) Init() (err error) { return nil } -func (svc *ServiceV2) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { - resultSvc, err := svc.getResultService(id) +func (svc *ServiceV2) InsertData(taskId primitive.ObjectID, records ...map[string]interface{}) (err error) { + count := 0 + + item, err := svc.getDatabaseServiceItem(taskId) if err != nil { return err } - if err := resultSvc.Insert(records...); err != nil { - return err + dbId := item.dbId + dbSvc := item.dbSvc + tableName := item.tableName + if utils.IsPro() && dbSvc != nil { + for _, record := range records { + if err := dbSvc.CreateRow(dbId, "", tableName, record); err != nil { + log2.Errorf("failed to insert data: %v", err) + continue + } + count++ + } + } else { + var records2 []interface{} + for _, record := range records { + records2 = append(records2, record) + } + _, err = mongo.GetMongoCol(tableName).InsertMany(records2) + if err != nil { + log2.Errorf("failed to insert data: %v", err) + return err + } + count = len(records) } - go svc.updateTaskStats(id, len(records)) + + go svc.updateTaskStats(taskId, count) + return nil } @@ -46,38 +82,52 @@ func (svc *ServiceV2) InsertLogs(id primitive.ObjectID, logs ...string) (err err return svc.logDriver.WriteLines(id.Hex(), logs) } -func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { +func (svc *ServiceV2) getDatabaseServiceItem(taskId primitive.ObjectID) (item *databaseServiceItem, err error) { // atomic operation svc.mu.Lock() defer svc.mu.Unlock() // attempt to get from cache - res, _ := svc.resultServices.Load(id.Hex()) - if res != nil { + item, ok := svc.databaseServiceItems[taskId.Hex()] + if ok { // hit in cache - resultSvc, ok := res.(interfaces.ResultService) - resultSvc.SetTime(time.Now()) - if ok { - return resultSvc, nil - } + item.time = time.Now() + return item, nil } // task - t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(taskId) if err != nil { return nil, err } - // result service - resultSvc, err = result.GetResultService(t.SpiderId) + // spider + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId) if err != nil { return nil, err } + // database service + var dbSvc interfaces2.DatabaseService + if utils.IsPro() { + if dbRegSvc := database.GetDatabaseRegistryService(); dbRegSvc != nil { + dbSvc, err = dbRegSvc.GetDatabaseService(s.DataSourceId) + if err != nil { + return nil, err + } + } + } + // store in cache - svc.resultServices.Store(id.Hex(), resultSvc) + svc.databaseServiceItems[taskId.Hex()] = &databaseServiceItem{ + taskId: taskId, + dbId: s.DataSourceId, + dbSvc: dbSvc, + tableName: s.ColName, + time: time.Now(), + } - return resultSvc, nil + return item, nil } func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) { @@ -96,13 +146,11 @@ func (svc *ServiceV2) cleanup() { // atomic operation svc.mu.Lock() - svc.resultServices.Range(func(key, value interface{}) bool { - rs := value.(interfaces.ResultService) - if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { - svc.resultServices.Delete(key) + for k, v := range svc.databaseServiceItems { + if time.Now().After(v.time.Add(svc.databaseServiceTll)) { + delete(svc.databaseServiceItems, k) } - return true - }) + } svc.mu.Unlock() @@ -113,8 +161,9 @@ func (svc *ServiceV2) cleanup() { func NewTaskStatsServiceV2() (svc2 *ServiceV2, err error) { // service svc := &ServiceV2{ - mu: sync.Mutex{}, - resultServices: sync.Map{}, + mu: sync.Mutex{}, + databaseServiceItems: map[string]*databaseServiceItem{}, + databaseServiceTll: 10 * time.Minute, } svc.nodeCfgSvc = nodeconfig.GetNodeConfigService()