From e8bc32c6603a50a094e4f94f1657b0cfad1af108 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 00:24:10 +0800 Subject: [PATCH 01/15] feat: support google cloud storage as das --- das/das.go | 8 +- das/factory.go | 9 ++ das/google_cloud_storage_service.go | 126 ++++++++++++++++++++++++++++ go.mod | 56 +++++++++---- go.sum | 122 ++++++++++++++++++++------- 5 files changed, 272 insertions(+), 49 deletions(-) create mode 100644 das/google_cloud_storage_service.go diff --git a/das/das.go b/das/das.go index 6bd02fbc75..0b03c05ad6 100644 --- a/das/das.go +++ b/das/das.go @@ -41,9 +41,10 @@ type DataAvailabilityConfig struct { LocalCache CacheConfig `koanf:"local-cache"` RedisCache RedisConfig `koanf:"redis-cache"` - LocalDBStorage LocalDBStorageConfig `koanf:"local-db-storage"` - LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"` - S3Storage S3StorageServiceConfig `koanf:"s3-storage"` + LocalDBStorage LocalDBStorageConfig `koanf:"local-db-storage"` + LocalFileStorage LocalFileStorageConfig `koanf:"local-file-storage"` + S3Storage S3StorageServiceConfig `koanf:"s3-storage"` + GoogleCloudStorage GoogleCloudStorageServiceConfig `koanf:"google-cloud-storage"` MigrateLocalDBToFileStorage bool `koanf:"migrate-local-db-to-file-storage"` @@ -114,6 +115,7 @@ func dataAvailabilityConfigAddOptions(prefix string, f *flag.FlagSet, r role) { LocalDBStorageConfigAddOptions(prefix+".local-db-storage", f) LocalFileStorageConfigAddOptions(prefix+".local-file-storage", f) S3ConfigAddOptions(prefix+".s3-storage", f) + GoogleCloudConfigAddOptions(prefix+".google-cloud-storage", f) f.Bool(prefix+".migrate-local-db-to-file-storage", DefaultDataAvailabilityConfig.MigrateLocalDBToFileStorage, "daserver will migrate all data on startup from local-db-storage to local-file-storage, then mark local-db-storage as unusable") // Key config for storage diff --git a/das/factory.go b/das/factory.go index 7f696912b3..0d78a18246 100644 --- a/das/factory.go +++ b/das/factory.go @@ -65,6 +65,15 @@ func CreatePersistentStorageService( storageServices = append(storageServices, s) } + if config.GoogleCloudStorage.Enable { + s, err := NewGoogleCloudStorageService(config.GoogleCloudStorage) + if err != nil { + return nil, nil, err + } + lifecycleManager.Register(s) + storageServices = append(storageServices, s) + } + if len(storageServices) > 1 { s, err := NewRedundantStorageService(ctx, storageServices) if err != nil { diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go new file mode 100644 index 0000000000..8c8c4716db --- /dev/null +++ b/das/google_cloud_storage_service.go @@ -0,0 +1,126 @@ +package das + +import ( + googlestorage "cloud.google.com/go/storage" + "context" + "encoding/hex" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/google/go-cmp/cmp" + "github.com/offchainlabs/nitro/arbstate/daprovider" + "github.com/offchainlabs/nitro/das/dastree" + "github.com/offchainlabs/nitro/util/pretty" + flag "github.com/spf13/pflag" + "google.golang.org/api/option" + "io" + "sort" +) + +type GoogleCloudStorageServiceConfig struct { + Enable bool `koanf:"enable"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + DiscardAfterTimeout bool `koanf:"discard-after-timeout"` +} + +var DefaultGoogleCloudStorageServiceConfig = GoogleCloudStorageServiceConfig{} + +func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { + f.Bool(prefix+".enable", DefaultGoogleCloudStorageServiceConfig.Enable, "enable storage/retrieval of sequencer batch data from an Google Cloud Storage bucket") + f.String(prefix+".access-token", DefaultGoogleCloudStorageServiceConfig.AccessToken, "Google Cloud Storage access token") + f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") + f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") + f.Bool(prefix+".discard-after-timeout", DefaultGoogleCloudStorageServiceConfig.DiscardAfterTimeout, "discard data after its expiry timeout") + +} + +type GoogleCloudStorageService struct { + client *googlestorage.Client + bucket string + objectPrefix string + discardAfterTimeout bool +} + +func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { + client, err := googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + if err != nil { + return nil, err + } + return &GoogleCloudStorageService{ + client: client, + bucket: config.Bucket, + objectPrefix: config.ObjectPrefix, + discardAfterTimeout: config.DiscardAfterTimeout, + }, nil +} + +func (gcs *GoogleCloudStorageService) Put(ctx context.Context, value []byte, timeout uint64) error { + logPut("das.GoogleCloudStorageService.Store", value, timeout, gcs) + bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) + w := bucket.NewWriter(ctx) + if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + log.Error("das.GoogleCloudStorageService.Store", "err", err) + return err + } + return w.Close() +} + +func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common.Hash) ([]byte, error) { + log.Trace("das.GoogleCloudStorageService.GetByHash", "key", pretty.PrettyHash(key), "this", gcs) + bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(key)) + reader, err := bucket.NewReader(ctx) + if err != nil { + return nil, err + } + buf, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return hex.DecodeString(string(buf)) +} + +func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { + if gcs.discardAfterTimeout { + return daprovider.DiscardAfterDataTimeout, nil + } + return daprovider.KeepForever, nil +} + +func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { + return nil +} + +func (gcs *GoogleCloudStorageService) Close(ctx context.Context) error { + return gcs.client.Close() +} + +func (gcs *GoogleCloudStorageService) String() string { + return fmt.Sprintf("GoogleCloudStorageService(:%s)", gcs.bucket) +} + +func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { + bucket := gcs.client.Bucket(gcs.bucket) + // check if we have bucket permissions + permissions := []string{ + "storage.buckets.get", + "storage.buckets.list", + "storage.objects.create", + "storage.objects.delete", + "storage.objects.list", + "storage.objects.get", + } + perms, err := bucket.IAM().TestPermissions(ctx, permissions) + if err != nil { + return err + } + sort.Strings(permissions) + sort.Strings(perms) + if !cmp.Equal(perms, permissions) { + return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) + } + // check if bucket exists (and others) + _, err = bucket.Attrs(ctx) + return err +} diff --git a/go.mod b/go.mod index da49b0d8b9..6407eec991 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ replace github.com/VictoriaMetrics/fastcache => ./fastcache replace github.com/ethereum/go-ethereum => ./go-ethereum require ( + cloud.google.com/go/storage v1.43.0 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/Shopify/toxiproxy v2.1.4+incompatible github.com/alicebob/miniredis/v2 v2.32.1 @@ -30,7 +31,7 @@ require ( github.com/gobwas/ws-examples v0.0.0-20190625122829-a9e8908d9484 github.com/google/btree v1.1.2 github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.2.4 github.com/knadh/koanf v1.4.0 @@ -42,15 +43,38 @@ require ( github.com/spf13/pflag v1.0.5 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/wealdtech/go-merkletree v1.0.0 - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/sys v0.18.0 - golang.org/x/term v0.18.0 - golang.org/x/tools v0.16.0 + golang.org/x/sys v0.21.0 + golang.org/x/term v0.21.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -require github.com/google/go-querystring v1.1.0 // indirect +require ( + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.8 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + google.golang.org/api v0.187.0 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/grpc v1.64.0 // indirect +) require ( github.com/DataDog/zstd v1.4.5 // indirect @@ -107,9 +131,9 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-github/v62 v62.0.0 @@ -159,13 +183,13 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opencensus.io v0.22.5 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.22.0 - golang.org/x/sync v0.5.0 - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/sync v0.7.0 + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index c0193be769..6120382844 100644 --- a/go.sum +++ b/go.sum @@ -13,14 +13,26 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -30,6 +42,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -233,6 +247,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -269,6 +285,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -303,12 +324,13 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -332,8 +354,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= @@ -353,6 +375,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -368,8 +391,11 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -381,12 +407,18 @@ github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8q github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -670,14 +702,19 @@ github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobt github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -729,8 +766,20 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -743,8 +792,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -781,8 +830,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -819,6 +868,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -827,8 +877,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -851,8 +901,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -928,14 +978,14 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -947,14 +997,15 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1003,8 +1054,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1025,6 +1076,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1063,6 +1116,12 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1078,7 +1137,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1091,8 +1153,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e6039aaddc0f3a2efc7433306959077e7ce9566b Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 02:01:27 +0800 Subject: [PATCH 02/15] refactor: adjust interface for easily unit tests --- cmd/genericconf/config.go | 18 +++++++ das/google_cloud_storage_service.go | 62 ++++++++++++++++++------ das/google_cloud_storage_service_test.go | 57 ++++++++++++++++++++++ 3 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 das/google_cloud_storage_service_test.go diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 06e1fcd12d..89dd0f336e 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -65,6 +65,24 @@ var DefaultS3Config = S3Config{ SecretKey: "", } +type GoogleCloudStorageConfig struct { + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` +} + +var DefaultGoogleCloudStorageConfig = GoogleCloudStorageConfig{ + AccessToken: "", + Bucket: "", + ObjectPrefix: "", +} + +func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { + f.String(prefix+".access-token", DefaultGoogleCloudStorageConfig.AccessToken, "Google Cloud Storage access token") + f.String(prefix+".bucket", DefaultGoogleCloudStorageConfig.Bucket, "Google Cloud Storage bucket") + f.String(prefix+".object-prefix", DefaultGoogleCloudStorageConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") +} + func HandlerFromLogType(logType string, output io.Writer) (slog.Handler, error) { if logType == "plaintext" { return log.NewTerminalHandler(output, false), nil diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 8c8c4716db..62a26422a0 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -17,6 +17,45 @@ import ( "sort" ) +type GoogleCloudStorageOperator interface { + Bucket(name string) *googlestorage.BucketHandle + Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error + Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) + Close(ctx context.Context) error +} + +type GoogleCloudStorageClient struct { + client *googlestorage.Client +} + +func (g *GoogleCloudStorageClient) Bucket(name string) *googlestorage.BucketHandle { + return g.client.Bucket(name) +} + +func (g *GoogleCloudStorageClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) + w := obj.NewWriter(ctx) + + if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + return err + } + return w.Close() + +} + +func (g *GoogleCloudStorageClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { + obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(key)) + reader, err := obj.NewReader(ctx) + if err != nil { + return nil, err + } + return io.ReadAll(reader) +} + +func (g *GoogleCloudStorageClient) Close(ctx context.Context) error { + return g.client.Close() +} + type GoogleCloudStorageServiceConfig struct { Enable bool `koanf:"enable"` AccessToken string `koanf:"access-token"` @@ -33,11 +72,10 @@ func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") f.Bool(prefix+".discard-after-timeout", DefaultGoogleCloudStorageServiceConfig.DiscardAfterTimeout, "discard data after its expiry timeout") - } type GoogleCloudStorageService struct { - client *googlestorage.Client + operator GoogleCloudStorageOperator bucket string objectPrefix string discardAfterTimeout bool @@ -49,7 +87,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora return nil, err } return &GoogleCloudStorageService{ - client: client, + operator: &GoogleCloudStorageClient{client: client}, bucket: config.Bucket, objectPrefix: config.ObjectPrefix, discardAfterTimeout: config.DiscardAfterTimeout, @@ -58,24 +96,18 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora func (gcs *GoogleCloudStorageService) Put(ctx context.Context, value []byte, timeout uint64) error { logPut("das.GoogleCloudStorageService.Store", value, timeout, gcs) - bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) - w := bucket.NewWriter(ctx) - if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, value); err != nil { log.Error("das.GoogleCloudStorageService.Store", "err", err) return err } - return w.Close() + return nil } func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common.Hash) ([]byte, error) { log.Trace("das.GoogleCloudStorageService.GetByHash", "key", pretty.PrettyHash(key), "this", gcs) - bucket := gcs.client.Bucket(gcs.bucket).Object(gcs.objectPrefix + EncodeStorageServiceKey(key)) - reader, err := bucket.NewReader(ctx) - if err != nil { - return nil, err - } - buf, err := io.ReadAll(reader) + buf, err := gcs.operator.Download(ctx, gcs.bucket, gcs.objectPrefix, key) if err != nil { + log.Error("das.GoogleCloudStorageService.GetByHash", "err", err) return nil, err } return hex.DecodeString(string(buf)) @@ -93,7 +125,7 @@ func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { } func (gcs *GoogleCloudStorageService) Close(ctx context.Context) error { - return gcs.client.Close() + return gcs.operator.Close(ctx) } func (gcs *GoogleCloudStorageService) String() string { @@ -101,7 +133,7 @@ func (gcs *GoogleCloudStorageService) String() string { } func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { - bucket := gcs.client.Bucket(gcs.bucket) + bucket := gcs.operator.Bucket(gcs.bucket) // check if we have bucket permissions permissions := []string{ "storage.buckets.get", diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go new file mode 100644 index 0000000000..eeabf9bc36 --- /dev/null +++ b/das/google_cloud_storage_service_test.go @@ -0,0 +1,57 @@ +package das + +import ( + googlestorage "cloud.google.com/go/storage" + "context" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/offchainlabs/nitro/cmd/genericconf" + "github.com/offchainlabs/nitro/das/dastree" + "testing" +) + +type mockGCSClient struct { +} + +func (c *mockGCSClient) Bucket(name string) *googlestorage.BucketHandle { + //TODO implement me + panic("implement me") +} + +func (c *mockGCSClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { + return nil, ErrNotFound +} + +func (c *mockGCSClient) Close(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (*mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + return nil +} + +func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig genericconf.GoogleCloudStorageConfig) (StorageService, error) { + return &GoogleCloudStorageService{ + bucket: googleCloudStorageConfig.Bucket, + objectPrefix: googleCloudStorageConfig.ObjectPrefix, + operator: &mockGCSClient{}, + }, nil +} + +func TestNewGoogleCloudStorageService(t *testing.T) { + ctx := context.Background() + //timeout := uint64(time.Now().Add(time.Hour).Unix()) + googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) + Require(t, err) + + val1 := []byte("The first value") + val1CorrectKey := dastree.Hash(val1) + //val2IncorrectKey := dastree.Hash(append(val1, 0)) + + _, err = googleCloudService.GetByHash(ctx, val1CorrectKey) + if !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + +} From 9fe04abf8a59d1a49142ba3c136cbb63c09262a1 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 10:13:54 +0800 Subject: [PATCH 03/15] fix: use raw bytes instead of hex --- das/google_cloud_storage_service.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 62a26422a0..8f54a00efd 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -3,7 +3,6 @@ package das import ( googlestorage "cloud.google.com/go/storage" "context" - "encoding/hex" "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -36,7 +35,7 @@ func (g *GoogleCloudStorageClient) Upload(ctx context.Context, bucket, objectPre obj := g.client.Bucket(bucket).Object(objectPrefix + EncodeStorageServiceKey(dastree.Hash(value))) w := obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, hex.EncodeToString(value)); err != nil { + if _, err := fmt.Fprintln(w, value); err != nil { return err } return w.Close() @@ -110,7 +109,7 @@ func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common. log.Error("das.GoogleCloudStorageService.GetByHash", "err", err) return nil, err } - return hex.DecodeString(string(buf)) + return buf, nil } func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { From 91d7c724ca6a223f6fadeee3b333d46fbacbe56a Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 11:43:30 +0800 Subject: [PATCH 04/15] fix: test case --- das/google_cloud_storage_service_test.go | 41 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index eeabf9bc36..86c611e2c5 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -1,6 +1,7 @@ package das import ( + "bytes" googlestorage "cloud.google.com/go/storage" "context" "errors" @@ -8,26 +9,32 @@ import ( "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/das/dastree" "testing" + "time" ) type mockGCSClient struct { + storage map[string][]byte } func (c *mockGCSClient) Bucket(name string) *googlestorage.BucketHandle { - //TODO implement me - panic("implement me") + return nil } func (c *mockGCSClient) Download(ctx context.Context, bucket, objectPrefix string, key common.Hash) ([]byte, error) { - return nil, ErrNotFound + value, ok := c.storage[objectPrefix+EncodeStorageServiceKey(key)] + if !ok { + return nil, ErrNotFound + } + return value, nil } func (c *mockGCSClient) Close(ctx context.Context) error { - //TODO implement me - panic("implement me") + return nil } -func (*mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { +func (c *mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, value []byte) error { + key := objectPrefix + EncodeStorageServiceKey(dastree.Hash(value)) + c.storage[key] = value return nil } @@ -35,23 +42,39 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon return &GoogleCloudStorageService{ bucket: googleCloudStorageConfig.Bucket, objectPrefix: googleCloudStorageConfig.ObjectPrefix, - operator: &mockGCSClient{}, + operator: &mockGCSClient{ + storage: make(map[string][]byte), + }, }, nil } func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() - //timeout := uint64(time.Now().Add(time.Hour).Unix()) + timeout := uint64(time.Now().Add(time.Hour).Unix()) googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) Require(t, err) val1 := []byte("The first value") val1CorrectKey := dastree.Hash(val1) - //val2IncorrectKey := dastree.Hash(append(val1, 0)) + val2IncorrectKey := dastree.Hash(append(val1, 0)) _, err = googleCloudService.GetByHash(ctx, val1CorrectKey) if !errors.Is(err, ErrNotFound) { t.Fatal(err) } + err = googleCloudService.Put(ctx, val1, timeout) + Require(t, err) + + _, err = googleCloudService.GetByHash(ctx, val2IncorrectKey) + if !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + + val, err := googleCloudService.GetByHash(ctx, val1CorrectKey) + Require(t, err) + if !bytes.Equal(val, val1) { + t.Fatal(val, val1) + } + } From 38b0c60a4d616d0b9eed668fa18ccd739132d3a0 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 12:09:32 +0800 Subject: [PATCH 05/15] refactor: use EnableExpire and MaxRetention for expiration polciy --- das/google_cloud_storage_service.go | 65 +++++++++++++++++++---------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 8f54a00efd..1ace445eed 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -10,10 +10,12 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/util/pretty" + "github.com/offchainlabs/nitro/util/stopwaiter" flag "github.com/spf13/pflag" "google.golang.org/api/option" "io" "sort" + "time" ) type GoogleCloudStorageOperator interface { @@ -56,11 +58,12 @@ func (g *GoogleCloudStorageClient) Close(ctx context.Context) error { } type GoogleCloudStorageServiceConfig struct { - Enable bool `koanf:"enable"` - AccessToken string `koanf:"access-token"` - Bucket string `koanf:"bucket"` - ObjectPrefix string `koanf:"object-prefix"` - DiscardAfterTimeout bool `koanf:"discard-after-timeout"` + Enable bool `koanf:"enable"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + EnableExpiry bool `koanf:"enable-expiry"` + MaxRetention time.Duration `koanf:"max-retention"` } var DefaultGoogleCloudStorageServiceConfig = GoogleCloudStorageServiceConfig{} @@ -70,14 +73,18 @@ func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".access-token", DefaultGoogleCloudStorageServiceConfig.AccessToken, "Google Cloud Storage access token") f.String(prefix+".bucket", DefaultGoogleCloudStorageServiceConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageServiceConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") - f.Bool(prefix+".discard-after-timeout", DefaultGoogleCloudStorageServiceConfig.DiscardAfterTimeout, "discard data after its expiry timeout") + f.Bool(prefix+".enable-expiry", DefaultLocalFileStorageConfig.EnableExpiry, "enable expiry of batches") + f.Duration(prefix+".max-retention", DefaultLocalFileStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") + } type GoogleCloudStorageService struct { - operator GoogleCloudStorageOperator - bucket string - objectPrefix string - discardAfterTimeout bool + operator GoogleCloudStorageOperator + bucket string + objectPrefix string + enableExpiry bool + maxRetention time.Duration + stopWaiter stopwaiter.StopWaiterSafe } func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { @@ -86,10 +93,11 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora return nil, err } return &GoogleCloudStorageService{ - operator: &GoogleCloudStorageClient{client: client}, - bucket: config.Bucket, - objectPrefix: config.ObjectPrefix, - discardAfterTimeout: config.DiscardAfterTimeout, + operator: &GoogleCloudStorageClient{client: client}, + bucket: config.Bucket, + objectPrefix: config.ObjectPrefix, + enableExpiry: config.EnableExpiry, + maxRetention: config.MaxRetention, }, nil } @@ -113,10 +121,10 @@ func (gcs *GoogleCloudStorageService) GetByHash(ctx context.Context, key common. } func (gcs *GoogleCloudStorageService) ExpirationPolicy(ctx context.Context) (daprovider.ExpirationPolicy, error) { - if gcs.discardAfterTimeout { - return daprovider.DiscardAfterDataTimeout, nil + if gcs.enableExpiry { + return daprovider.KeepForever, nil } - return daprovider.KeepForever, nil + return daprovider.DiscardAfterDataTimeout, nil } func (gcs *GoogleCloudStorageService) Sync(ctx context.Context) error { @@ -144,14 +152,29 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { } perms, err := bucket.IAM().TestPermissions(ctx, permissions) if err != nil { - return err + return fmt.Errorf("could not check permissions: %v", err) } sort.Strings(permissions) sort.Strings(perms) if !cmp.Equal(perms, permissions) { return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) } - // check if bucket exists (and others) - _, err = bucket.Attrs(ctx) - return err + // check if bucket exists (and others), and update expiration policy if enabled + attrs, err := bucket.Attrs(ctx) + if err != nil { + return err + } + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return fmt.Errorf("failed to update bucket lifecycle: %v", err) + } + return nil } From d466ea5480fab256aec4d351991ab26d5341e37d Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 12:33:36 +0800 Subject: [PATCH 06/15] fix: enable expiry check --- das/google_cloud_storage_service.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 1ace445eed..58c15295af 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -164,17 +164,20 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { if err != nil { return err } - lifecycleRule := googlestorage.LifecycleRule{ - Action: googlestorage.LifecycleAction{Type: "Delete"}, - Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days + if gcs.enableExpiry { + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return fmt.Errorf("failed to update bucket lifecycle: %v", err) + } } - attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) - bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ - Lifecycle: &attrs.Lifecycle, - } - if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { - return fmt.Errorf("failed to update bucket lifecycle: %v", err) - } return nil } From a86c4ec0ada9d3ca96f990d3bf17eb223b76190c Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 16:16:01 +0800 Subject: [PATCH 07/15] chore: make init client error more readable --- contracts | 2 +- das/google_cloud_storage_service.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts b/contracts index 23fc796282..f7894d3a6d 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f +Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index 58c15295af..e66cc1cef8 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -90,7 +90,7 @@ type GoogleCloudStorageService struct { func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { client, err := googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) if err != nil { - return nil, err + return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) } return &GoogleCloudStorageService{ operator: &GoogleCloudStorageClient{client: client}, From e9e364356accfb392748c0ed565d0d82d5dc4581 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Sat, 7 Sep 2024 20:21:01 +0800 Subject: [PATCH 08/15] refactor: support ADC --- das/google_cloud_storage_service.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index e66cc1cef8..e9104f1334 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -88,7 +88,15 @@ type GoogleCloudStorageService struct { } func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { - client, err := googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + var client *googlestorage.Client + var err error + // Note that if the credentials are not specified, the client library will find credentials using ADC(Application Default Credentials) + // https://cloud.google.com/docs/authentication/provide-credentials-adc. + if config.AccessToken == "" { + client, err = googlestorage.NewClient(context.Background()) + } else { + client, err = googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) + } if err != nil { return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) } From 88a2d724ed144e25a9ad38fc9dca466e2fa3838a Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 10:20:59 +0800 Subject: [PATCH 09/15] fix: check expire time before pushing data --- cmd/genericconf/config.go | 9 ++++++--- das/google_cloud_storage_service.go | 16 +++++++++++++--- das/google_cloud_storage_service_test.go | 5 +++-- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 89dd0f336e..8bde4a972b 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -66,21 +66,24 @@ var DefaultS3Config = S3Config{ } type GoogleCloudStorageConfig struct { - AccessToken string `koanf:"access-token"` - Bucket string `koanf:"bucket"` - ObjectPrefix string `koanf:"object-prefix"` + AccessToken string `koanf:"access-token"` + Bucket string `koanf:"bucket"` + ObjectPrefix string `koanf:"object-prefix"` + MaxRetention time.Duration `koanf:"max-retention"` } var DefaultGoogleCloudStorageConfig = GoogleCloudStorageConfig{ AccessToken: "", Bucket: "", ObjectPrefix: "", + MaxRetention: time.Hour * 24, } func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".access-token", DefaultGoogleCloudStorageConfig.AccessToken, "Google Cloud Storage access token") f.String(prefix+".bucket", DefaultGoogleCloudStorageConfig.Bucket, "Google Cloud Storage bucket") f.String(prefix+".object-prefix", DefaultGoogleCloudStorageConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") + f.Duration(prefix+".max-retention", DefaultGoogleCloudStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") } func HandlerFromLogType(logType string, output io.Writer) (slog.Handler, error) { diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index e9104f1334..d5ee9c4ace 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -14,6 +14,7 @@ import ( flag "github.com/spf13/pflag" "google.golang.org/api/option" "io" + "math" "sort" "time" ) @@ -109,9 +110,18 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora }, nil } -func (gcs *GoogleCloudStorageService) Put(ctx context.Context, value []byte, timeout uint64) error { - logPut("das.GoogleCloudStorageService.Store", value, timeout, gcs) - if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, value); err != nil { +func (gcs *GoogleCloudStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { + logPut("das.GoogleCloudStorageService.Store", data, expiry, gcs) + if expiry > math.MaxInt64 { + return fmt.Errorf("request expiry time (%v) exceeds max int64", expiry) + } + // #nosec G115 + expiryTime := time.Unix(int64(expiry), 0) + currentTimePlusRetention := time.Now().Add(gcs.maxRetention) + if expiryTime.After(currentTimePlusRetention) { + return fmt.Errorf("requested expiry time (%v) exceeds current time plus maximum allowed retention period(%v)", expiryTime, currentTimePlusRetention) + } + if err := gcs.operator.Upload(ctx, gcs.bucket, gcs.objectPrefix, data); err != nil { log.Error("das.GoogleCloudStorageService.Store", "err", err) return err } diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index 86c611e2c5..5482b0c1c2 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -45,12 +45,13 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon operator: &mockGCSClient{ storage: make(map[string][]byte), }, + maxRetention: googleCloudStorageConfig.MaxRetention, }, nil } func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() - timeout := uint64(time.Now().Add(time.Hour).Unix()) + expiry := uint64(time.Now().Add(time.Hour).Unix()) googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) Require(t, err) @@ -63,7 +64,7 @@ func TestNewGoogleCloudStorageService(t *testing.T) { t.Fatal(err) } - err = googleCloudService.Put(ctx, val1, timeout) + err = googleCloudService.Put(ctx, val1, expiry) Require(t, err) _, err = googleCloudService.GetByHash(ctx, val2IncorrectKey) From 4ac2c217ef9b92e30917a9cfa7c7171fbb9966f4 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 10:35:25 +0800 Subject: [PATCH 10/15] fix: move health check to constructor --- das/google_cloud_storage_service.go | 45 +++++++++++++++-------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index d5ee9c4ace..d5804d3002 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -101,13 +101,35 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora if err != nil { return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) } - return &GoogleCloudStorageService{ + service := &GoogleCloudStorageService{ operator: &GoogleCloudStorageClient{client: client}, bucket: config.Bucket, objectPrefix: config.ObjectPrefix, enableExpiry: config.EnableExpiry, maxRetention: config.MaxRetention, - }, nil + } + if config.EnableExpiry { + lifecycleRule := googlestorage.LifecycleRule{ + Action: googlestorage.LifecycleAction{Type: "Delete"}, + Condition: googlestorage.LifecycleCondition{AgeInDays: int64(config.MaxRetention.Hours() / 24)}, // Objects older than 30 days + } + ctx := context.Background() + bucket := service.operator.Bucket(service.bucket) + // check if bucket exists (and others), and update expiration policy if enabled + attrs, err := bucket.Attrs(ctx) + if err != nil { + return nil, fmt.Errorf("error getting bucket attributes: %v", err) + } + attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) + + bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ + Lifecycle: &attrs.Lifecycle, + } + if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { + return nil, fmt.Errorf("failed to update bucket lifecycle: %v", err) + } + } + return service, nil } func (gcs *GoogleCloudStorageService) Put(ctx context.Context, data []byte, expiry uint64) error { @@ -177,25 +199,6 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { if !cmp.Equal(perms, permissions) { return fmt.Errorf("permissions mismatch (-want +got):\n%s", cmp.Diff(permissions, perms)) } - // check if bucket exists (and others), and update expiration policy if enabled - attrs, err := bucket.Attrs(ctx) - if err != nil { - return err - } - if gcs.enableExpiry { - lifecycleRule := googlestorage.LifecycleRule{ - Action: googlestorage.LifecycleAction{Type: "Delete"}, - Condition: googlestorage.LifecycleCondition{AgeInDays: int64(gcs.maxRetention.Hours() / 24)}, // Objects older than 30 days - } - attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) - - bucketAttrsToUpdate := googlestorage.BucketAttrsToUpdate{ - Lifecycle: &attrs.Lifecycle, - } - if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { - return fmt.Errorf("failed to update bucket lifecycle: %v", err) - } - } return nil } From 680ee95ddd3b123f237fe29ab586c6731c418f11 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 11:17:43 +0800 Subject: [PATCH 11/15] chore: fmt and lint --- das/google_cloud_storage_service.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/das/google_cloud_storage_service.go b/das/google_cloud_storage_service.go index d5804d3002..2c490f346c 100644 --- a/das/google_cloud_storage_service.go +++ b/das/google_cloud_storage_service.go @@ -10,7 +10,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/offchainlabs/nitro/das/dastree" "github.com/offchainlabs/nitro/util/pretty" - "github.com/offchainlabs/nitro/util/stopwaiter" flag "github.com/spf13/pflag" "google.golang.org/api/option" "io" @@ -85,7 +84,6 @@ type GoogleCloudStorageService struct { objectPrefix string enableExpiry bool maxRetention time.Duration - stopWaiter stopwaiter.StopWaiterSafe } func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (StorageService, error) { @@ -99,7 +97,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora client, err = googlestorage.NewClient(context.Background(), option.WithCredentialsJSON([]byte(config.AccessToken))) } if err != nil { - return nil, fmt.Errorf("error creating Google Cloud Storage client: %v", err) + return nil, fmt.Errorf("error creating Google Cloud Storage client: %w", err) } service := &GoogleCloudStorageService{ operator: &GoogleCloudStorageClient{client: client}, @@ -118,7 +116,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora // check if bucket exists (and others), and update expiration policy if enabled attrs, err := bucket.Attrs(ctx) if err != nil { - return nil, fmt.Errorf("error getting bucket attributes: %v", err) + return nil, fmt.Errorf("error getting bucket attributes: %w", err) } attrs.Lifecycle.Rules = append(attrs.Lifecycle.Rules, lifecycleRule) @@ -126,7 +124,7 @@ func NewGoogleCloudStorageService(config GoogleCloudStorageServiceConfig) (Stora Lifecycle: &attrs.Lifecycle, } if _, err := bucket.Update(ctx, bucketAttrsToUpdate); err != nil { - return nil, fmt.Errorf("failed to update bucket lifecycle: %v", err) + return nil, fmt.Errorf("failed to update bucket lifecycle: %w", err) } } return service, nil @@ -192,7 +190,7 @@ func (gcs *GoogleCloudStorageService) HealthCheck(ctx context.Context) error { } perms, err := bucket.IAM().TestPermissions(ctx, permissions) if err != nil { - return fmt.Errorf("could not check permissions: %v", err) + return fmt.Errorf("could not check permissions: %w", err) } sort.Strings(permissions) sort.Strings(perms) From bee25349260483fed2089ba17a5bf35cecc0fba1 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Tue, 10 Sep 2024 11:19:54 +0800 Subject: [PATCH 12/15] chore: undo submodule commit for contracts --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f7894d3a6d..23fc796282 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f7894d3a6d4035ba60f51a7f1334f0f2d4f02dce +Subproject commit 23fc79628292aa5d604d449fed48937ae7faeb2f From e63345b55813c1e50f16d7ef6f74c5e056635a27 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 3 Oct 2024 07:35:04 +0800 Subject: [PATCH 13/15] refactor: remove dumplicate config, fix unit test --- cmd/genericconf/config.go | 21 --------------------- das/google_cloud_storage_service_test.go | 8 +++++--- go.mod | 3 ++- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/cmd/genericconf/config.go b/cmd/genericconf/config.go index 8bde4a972b..06e1fcd12d 100644 --- a/cmd/genericconf/config.go +++ b/cmd/genericconf/config.go @@ -65,27 +65,6 @@ var DefaultS3Config = S3Config{ SecretKey: "", } -type GoogleCloudStorageConfig struct { - AccessToken string `koanf:"access-token"` - Bucket string `koanf:"bucket"` - ObjectPrefix string `koanf:"object-prefix"` - MaxRetention time.Duration `koanf:"max-retention"` -} - -var DefaultGoogleCloudStorageConfig = GoogleCloudStorageConfig{ - AccessToken: "", - Bucket: "", - ObjectPrefix: "", - MaxRetention: time.Hour * 24, -} - -func GoogleCloudConfigAddOptions(prefix string, f *flag.FlagSet) { - f.String(prefix+".access-token", DefaultGoogleCloudStorageConfig.AccessToken, "Google Cloud Storage access token") - f.String(prefix+".bucket", DefaultGoogleCloudStorageConfig.Bucket, "Google Cloud Storage bucket") - f.String(prefix+".object-prefix", DefaultGoogleCloudStorageConfig.ObjectPrefix, "prefix to add to Google Cloud Storage objects") - f.Duration(prefix+".max-retention", DefaultGoogleCloudStorageConfig.MaxRetention, "store requests with expiry times farther in the future than max-retention will be rejected") -} - func HandlerFromLogType(logType string, output io.Writer) (slog.Handler, error) { if logType == "plaintext" { return log.NewTerminalHandler(output, false), nil diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index 5482b0c1c2..21cca0abb2 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -6,7 +6,6 @@ import ( "context" "errors" "github.com/ethereum/go-ethereum/common" - "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/das/dastree" "testing" "time" @@ -38,7 +37,7 @@ func (c *mockGCSClient) Upload(ctx context.Context, bucket, objectPrefix string, return nil } -func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig genericconf.GoogleCloudStorageConfig) (StorageService, error) { +func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageConfig GoogleCloudStorageServiceConfig) (StorageService, error) { return &GoogleCloudStorageService{ bucket: googleCloudStorageConfig.Bucket, objectPrefix: googleCloudStorageConfig.ObjectPrefix, @@ -52,7 +51,10 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() expiry := uint64(time.Now().Add(time.Hour).Unix()) - googleCloudService, err := NewTestGoogleCloudStorageService(ctx, genericconf.DefaultGoogleCloudStorageConfig) + googleCloudStorageServiceConfig := DefaultGoogleCloudStorageServiceConfig + googleCloudStorageServiceConfig.Enable = true + googleCloudStorageServiceConfig.MaxRetention = time.Hour * 24 + googleCloudService, err := NewTestGoogleCloudStorageService(ctx, googleCloudStorageServiceConfig) Require(t, err) val1 := []byte("The first value") diff --git a/go.mod b/go.mod index f44203786e..b6180ee640 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( golang.org/x/sys v0.21.0 golang.org/x/term v0.21.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d + google.golang.org/api v0.187.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -58,6 +59,7 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fjl/memsize v0.0.2 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -69,7 +71,6 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - google.golang.org/api v0.187.0 // indirect google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect From 3df28169b187187c46dd6bdf0544963e38d06973 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Thu, 3 Oct 2024 08:20:24 +0800 Subject: [PATCH 14/15] chore: adjust go mod --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index b6180ee640..e6b82a0d42 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,6 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fjl/memsize v0.0.2 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 6120382844..fa44536f2f 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,6 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= From f0b477a6031cc599458ba63d7c559ccd11e5c744 Mon Sep 17 00:00:00 2001 From: xiaohuo Date: Fri, 4 Oct 2024 01:42:47 +0800 Subject: [PATCH 15/15] fix: lint --- das/google_cloud_storage_service_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/das/google_cloud_storage_service_test.go b/das/google_cloud_storage_service_test.go index 21cca0abb2..799d999bad 100644 --- a/das/google_cloud_storage_service_test.go +++ b/das/google_cloud_storage_service_test.go @@ -50,6 +50,7 @@ func NewTestGoogleCloudStorageService(ctx context.Context, googleCloudStorageCon func TestNewGoogleCloudStorageService(t *testing.T) { ctx := context.Background() + // #nosec G115 expiry := uint64(time.Now().Add(time.Hour).Unix()) googleCloudStorageServiceConfig := DefaultGoogleCloudStorageServiceConfig googleCloudStorageServiceConfig.Enable = true