From 1ea4d8340672d77205b9c26021af2e7a061c6c2b Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 19 Dec 2024 13:53:13 +0000 Subject: [PATCH 1/6] Open AWS storage to being used with non-AWS MySQL and S3 services --- cmd/conformance/aws/main.go | 26 ++++++++++++++++++++++++++ storage/aws/README.md | 16 ++++++++++++++++ storage/aws/aws.go | 16 ++++++++++++---- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/cmd/conformance/aws/main.go b/cmd/conformance/aws/main.go index f86c6043..7b3223b0 100644 --- a/cmd/conformance/aws/main.go +++ b/cmd/conformance/aws/main.go @@ -24,6 +24,9 @@ import ( "net/http" "time" + aaws "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/storage/aws" "golang.org/x/mod/sumdb/note" @@ -42,6 +45,10 @@ var ( dbPassword = flag.String("db_password", "", "AuroraDB user") dbMaxConns = flag.Int("db_max_conns", 0, "Maximum connections to the database, defaults to 0, i.e unlimited") dbMaxIdle = flag.Int("db_max_idle_conns", 2, "Maximum idle database connections in the connection pool, defaults to 2") + s3Endpoint = flag.String("s3_endpoint", "", "Endpoint for custom non-AWS S3 service") + s3AccessKeyID = flag.String("s3_access_key", "", "Access key ID for custom non-AWS S3 service") + s3SecretAccessKey = flag.String("s3_secret", "", "Secret access key for custom non-AWS S3 service") + s3UseSSL = flag.Bool("s3_use_ssl", false, "Whether to use SSL for custom non-AWS S3 service") signer = flag.String("signer", "", "Note signer to use to sign checkpoints") publishInterval = flag.Duration("publish_interval", 3*time.Second, "How frequently to publish updated checkpoints") additionalSigners = []string{} @@ -138,8 +145,27 @@ func storageConfigFromFlags() aws.Config { *dbUser, *dbPassword, dbEndpoint, *dbName, ) + // Configure to use MinIO Server + var awsConfig *aaws.Config + var s3Opts func(o *s3.Options) + if *s3Endpoint != "" { + const defaultRegion = "us-east-1" + s3Opts = func(o *s3.Options) { + o.BaseEndpoint = aaws.String(*s3Endpoint) + o.Credentials = credentials.NewStaticCredentialsProvider(*s3AccessKeyID, *s3SecretAccessKey, "") + o.Region = defaultRegion + o.UsePathStyle = true + } + + awsConfig = &aaws.Config{ + Region: defaultRegion, + } + } + return aws.Config{ Bucket: *bucket, + SDKConfig: awsConfig, + S3Options: s3Opts, DSN: dsn, MaxOpenConns: *dbMaxConns, MaxIdleConns: *dbMaxIdle, diff --git a/storage/aws/README.md b/storage/aws/README.md index 838eff14..ade470e3 100644 --- a/storage/aws/README.md +++ b/storage/aws/README.md @@ -58,6 +58,22 @@ Two experimental implementations have been tested which uses either Aurora MySQL or a local bbolt database to store the `` --> `sequence` mapping. They work well, but call for further stress testing and cost analysis. +## Compatibility + +This storage implementation is intended to be used with AWS services. + +However, given that it's based on services which are compatible with MySQL and +S3 protocols, it's possible that it will work with other non-AWS-based backends +which are compatible with these protocols. + +Given the vast array of combinations of backend implementations and versions, we +can't offer anything more than a "best effort" level of support for using this storage +implementation outside of AWS. + +Similarly, PRs raised against it relating to its use outside of AWS are unlikely to +be accepted unless it's shown that they have no detremental effect to the implementation's +performance on AWS. + ### Alternatives considered Other transactional storage systems are available on AWS, e.g. Redshift, RDS or diff --git a/storage/aws/aws.go b/storage/aws/aws.go index aeda7e4e..c9b7c71c 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -110,6 +110,11 @@ type consumeFunc func(ctx context.Context, from uint64, entries []storage.Sequen // Config holds AWS project and resource configuration for a storage instance. type Config struct { + // SDKConfig is an optional AWS config to use when configuring service clients, e.g. S3. + // If nil, the value from config.LoadDefaultConfig() will be used. + SDKConfig *aws.Config + // S3Options is an optional function which can be used to configure the S3 library. + S3Options func(*s3.Options) // Bucket is the name of the S3 bucket to use for storing log state. Bucket string // DSN is the DSN of the MySQL instance to use. @@ -133,11 +138,14 @@ func New(ctx context.Context, cfg Config, opts ...func(*options.StorageOptions)) return nil, fmt.Errorf("requested CheckpointInterval (%v) is less than minimum permitted %v", opt.CheckpointInterval, minCheckpointInterval) } - sdkConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to load default AWS configuration: %v", err) + if cfg.SDKConfig == nil { + sdkConfig, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load default AWS configuration: %v", err) + } + cfg.SDKConfig = &sdkConfig } - c := s3.NewFromConfig(sdkConfig) + c := s3.NewFromConfig(*cfg.SDKConfig, cfg.S3Options) seq, err := newMySQLSequencer(ctx, cfg.DSN, uint64(opt.PushbackMaxOutstanding), cfg.MaxOpenConns, cfg.MaxIdleConns) if err != nil { From 5546321d5b3dd5a1a207af0eb3a9f637f12ec13e Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 19 Dec 2024 17:02:52 +0000 Subject: [PATCH 2/6] Fix lint --- cmd/conformance/aws/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/conformance/aws/main.go b/cmd/conformance/aws/main.go index 7b3223b0..4294fdbb 100644 --- a/cmd/conformance/aws/main.go +++ b/cmd/conformance/aws/main.go @@ -48,7 +48,6 @@ var ( s3Endpoint = flag.String("s3_endpoint", "", "Endpoint for custom non-AWS S3 service") s3AccessKeyID = flag.String("s3_access_key", "", "Access key ID for custom non-AWS S3 service") s3SecretAccessKey = flag.String("s3_secret", "", "Secret access key for custom non-AWS S3 service") - s3UseSSL = flag.Bool("s3_use_ssl", false, "Whether to use SSL for custom non-AWS S3 service") signer = flag.String("signer", "", "Note signer to use to sign checkpoints") publishInterval = flag.Duration("publish_interval", 3*time.Second, "How frequently to publish updated checkpoints") additionalSigners = []string{} From 83f0f57c7f526db55933780a096c703b494791db Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 19 Dec 2024 17:08:11 +0000 Subject: [PATCH 3/6] Tweak wording --- storage/aws/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/aws/README.md b/storage/aws/README.md index ade470e3..a08bff66 100644 --- a/storage/aws/README.md +++ b/storage/aws/README.md @@ -66,9 +66,9 @@ However, given that it's based on services which are compatible with MySQL and S3 protocols, it's possible that it will work with other non-AWS-based backends which are compatible with these protocols. -Given the vast array of combinations of backend implementations and versions, we -can't offer anything more than a "best effort" level of support for using this storage -implementation outside of AWS. +Given the vast array of combinations of backend implementations and versions, +using this storage implementation outside of AWS isn't officially supported, although +there may be folks who can help with issues in the Transparency-Dev slack. Similarly, PRs raised against it relating to its use outside of AWS are unlikely to be accepted unless it's shown that they have no detremental effect to the implementation's From 07b7d0d29b8edb3bd9ef49ade6eac8900d1c211d Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 19 Dec 2024 17:12:04 +0000 Subject: [PATCH 4/6] Add warnings to Config docstrings --- storage/aws/aws.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/storage/aws/aws.go b/storage/aws/aws.go index c9b7c71c..def57805 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -110,10 +110,16 @@ type consumeFunc func(ctx context.Context, from uint64, entries []storage.Sequen // Config holds AWS project and resource configuration for a storage instance. type Config struct { - // SDKConfig is an optional AWS config to use when configuring service clients, e.g. S3. - // If nil, the value from config.LoadDefaultConfig() will be used. + // SDKConfig is an optional AWS config to use when configuring service clients, e.g. to + // use non-AWS S3 or MySQL services. + // + // If nil, the value from config.LoadDefaultConfig() will be used - this is the only + // supported configuration. SDKConfig *aws.Config // S3Options is an optional function which can be used to configure the S3 library. + // This is primarily useful when configuring the use of non-AWS S3 or MySQL services. + // + // If nil, the default options will be used - this is the only supported configuration. S3Options func(*s3.Options) // Bucket is the name of the S3 bucket to use for storing log state. Bucket string From c265a96e0ce13455bf7946f48a5455a553a5d6c7 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 19 Dec 2024 17:24:34 +0000 Subject: [PATCH 5/6] Add warning about running on non-AWS infra --- storage/aws/aws.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/storage/aws/aws.go b/storage/aws/aws.go index def57805..be06fbb1 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -30,8 +30,10 @@ package aws import ( "bytes" + "compress/gzip" "context" "database/sql" + "encoding/base64" "encoding/gob" "errors" "fmt" @@ -150,6 +152,8 @@ func New(ctx context.Context, cfg Config, opts ...func(*options.StorageOptions)) return nil, fmt.Errorf("failed to load default AWS configuration: %v", err) } cfg.SDKConfig = &sdkConfig + } else { + printDragonsWarning() } c := s3.NewFromConfig(*cfg.SDKConfig, cfg.S3Options) @@ -887,3 +891,13 @@ func (s *s3Storage) lastModified(ctx context.Context, obj string) (time.Time, er return *r.LastModified, r.Body.Close() } + +func printDragonsWarning() { + d := `H4sIAFZYZGcAA01QMQ7EIAzbeYXV5UCqkq1bf2IFtpNuPalj334hFQdkwLGNAwBzyXnKitOiqTYj +B7ZGplWEwZhZqxZ1aKuswcD0AA4GXPUhI0MEpSd5Ow09vJ+m6rVtF6m0GDccYXDZEdp9N/g1H9Pf +Qu80vNj7tiOe0lkdc8hwZK9YxavT0+FTP++vU6DUKvpEOr1+VGTk3IBXKSX9AHz5xXRgAQAA` + g, _ := base64.StdEncoding.DecodeString(d) + r, _ := gzip.NewReader(bytes.NewReader(g)) + t, _ := io.ReadAll(r) + klog.Infof("Running in non-AWS mode: here be dragons!\n%s", t) +} From 224c389558983dae426a4892c8515565d5c6f0f4 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 19 Dec 2024 17:58:21 +0000 Subject: [PATCH 6/6] Add pointer to README --- storage/aws/aws.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/aws/aws.go b/storage/aws/aws.go index be06fbb1..3e348325 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -899,5 +899,6 @@ Qu80vNj7tiOe0lkdc8hwZK9YxavT0+FTP++vU6DUKvpEOr1+VGTk3IBXKSX9AHz5xXRgAQAA` g, _ := base64.StdEncoding.DecodeString(d) r, _ := gzip.NewReader(bytes.NewReader(g)) t, _ := io.ReadAll(r) - klog.Infof("Running in non-AWS mode: here be dragons!\n%s", t) + klog.Infof("Running in non-AWS mode - see storage/aws/README.md for more details.") + klog.Infof("Here be dragons!\n%s", t) }