From a545af9fe81befc29bdf043bb604fced4ae28533 Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Mon, 9 Dec 2024 12:34:37 +0800 Subject: [PATCH 1/6] Set up minio locally --- docker-compose.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 131428b18a..61e9eb130f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -91,6 +91,30 @@ services: ports: - "4318:4318" + minio: + image: quay.io/minio/minio:RELEASE.2024-11-07T00-52-20Z + command: + - "minio" + - "server" + - "/data" + - "--console-address" + - ":9001" + volumes: + - minio_data:/data + ports: + - "9000:9000" + - "9001:9001" + environment: + # This can be used as access key ID. + # But do not do this in production. + # See https://min.io/docs/minio/linux/administration/identity-access-management/minio-user-management.html#minio-root-user + MINIO_ROOT_USER: "minio" + # This must be at least 8 characters + # this can be used as secret access key. + # But do not do this in production. + # See https://min.io/docs/minio/linux/administration/identity-access-management/minio-user-management.html#minio-root-user + MINIO_ROOT_PASSWORD: "secretpassword" + ldap: profiles: ["ldap"] image: bitnami/openldap:2.6 @@ -124,3 +148,5 @@ volumes: driver: local openldap_data: driver: local + minio_data: + driver: local From fb9ec0625ac322ebed22f7ecd28fc6e4a7905a60 Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Mon, 9 Dec 2024 14:13:53 +0800 Subject: [PATCH 2/6] Document how to set up MinIO --- CONTRIBUTING.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 612be5d576..d9195d38ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,7 @@ * [Install dependencies](#install-dependencies) * [Set up environment](#set-up-environment) * [Set up the database](#set-up-the-database) + * [Set up MinIO](#setup-minio) * [Run](#run) * [Create an account for yourselves and grant you access to the portal](#create-an-account-for-yourselves-and-grant-you-access-to-the-portal) * [Known issues](#known-issues) @@ -179,6 +180,20 @@ This project uses asdf, and there is a .tool-versions file at the project root. go run ./cmd/authgear search database migrate up ``` +## Set up MinIO + +```sh +docker compose up -d minio +docker compose exec -it bash + +# Inside the container +mc alias set local http://localhost:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" +# Create a bucket named "images" +mc mb local/images +# Create a bucket named "userexport" +mc mb local/userexport +``` + ## Run 1. In case you have made changes to authui, you run `make authui` to re-build the assets, or run `make authui-dev` to start the development server (Hot Reload/Hot Module Replacement supported). From 39e390ad49b8360e7b9f4391207441f1cf159b55 Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Mon, 9 Dec 2024 13:20:24 +0800 Subject: [PATCH 3/6] Introduce AbstractObjectStoreConfig to avoid code duplication --- cmd/authgear/images/server/config.go | 7 +- pkg/images/config/object_store.go | 146 +----------------- pkg/images/deps/deps.go | 7 +- ...rt_storage.go => abstract_object_store.go} | 9 +- pkg/lib/config/userexport_object_store.go | 3 + 5 files changed, 20 insertions(+), 152 deletions(-) rename pkg/lib/config/{export_storage.go => abstract_object_store.go} (92%) create mode 100644 pkg/lib/config/userexport_object_store.go diff --git a/cmd/authgear/images/server/config.go b/cmd/authgear/images/server/config.go index 084383de5e..bee4d800bc 100644 --- a/cmd/authgear/images/server/config.go +++ b/cmd/authgear/images/server/config.go @@ -6,6 +6,7 @@ import ( "github.com/kelseyhightower/envconfig" imagesconfig "github.com/authgear/authgear-server/pkg/images/config" + "github.com/authgear/authgear-server/pkg/lib/config" "github.com/authgear/authgear-server/pkg/util/validation" ) @@ -43,12 +44,14 @@ func LoadConfigFromEnv() (*Config, error) { func (c *Config) Initialize() error { ctx := &validation.Context{} - c.ObjectStore.Initialize(ctx.Child("IMAGES_OBJECT_STORE")) + objectStoreConfig := config.AbstractObjectStoreConfig(*c.ObjectStore) + objectStoreConfig.Initialize(ctx.Child("IMAGES_OBJECT_STORE")) return ctx.Error("failed to initialize server configuration") } func (c *Config) Validate() error { ctx := &validation.Context{} - c.ObjectStore.Validate(ctx.Child("IMAGES_OBJECT_STORE")) + objectStoreConfig := config.AbstractObjectStoreConfig(*c.ObjectStore) + objectStoreConfig.Validate(ctx.Child("IMAGES_OBJECT_STORE")) return ctx.Error("invalid server configuration") } diff --git a/pkg/images/config/object_store.go b/pkg/images/config/object_store.go index 58b920d6c7..4f5fcedd97 100644 --- a/pkg/images/config/object_store.go +++ b/pkg/images/config/object_store.go @@ -1,149 +1,7 @@ package config import ( - "fmt" - "io/ioutil" - "os" - - "github.com/authgear/authgear-server/pkg/util/validation" + "github.com/authgear/authgear-server/pkg/lib/config" ) -type ObjectStoreConfig struct { - Type ObjectStoreType `envconfig:"TYPE"` - AWSS3 AWSS3ObjectStoreConfig `envconfig:"AWS_S3"` - GCPGCS GCPGCSObjectStoreConfig `envconfig:"GCP_GCS"` - AzureBlobStorage AzureBlobStorageObjectStoreConfig `envconfig:"AZURE_BLOB_STORAGE"` -} - -func (c *ObjectStoreConfig) Initialize(ctx *validation.Context) { - switch c.Type { - case "": - break - case ObjectStoreTypeAWSS3: - c.AWSS3.Initialize(ctx.Child("AWS_S3")) - case ObjectStoreTypeGCPGCS: - c.GCPGCS.Initialize(ctx.Child("GCP_GCS")) - case ObjectStoreTypeAzureBlobStorage: - c.AzureBlobStorage.Initialize(ctx.Child("AZURE_BLOB_STORAGE")) - default: - ctx.Child("TYPE").EmitErrorMessage(fmt.Sprintf("invalid object store type: %v", c.Type)) - } -} - -func (c *ObjectStoreConfig) Validate(ctx *validation.Context) { - switch c.Type { - case "": - break - case ObjectStoreTypeAWSS3: - c.AWSS3.Validate(ctx.Child("AWS_S3")) - case ObjectStoreTypeGCPGCS: - c.GCPGCS.Validate(ctx.Child("GCP_GCS")) - case ObjectStoreTypeAzureBlobStorage: - c.AzureBlobStorage.Validate(ctx.Child("AZURE_BLOB_STORAGE")) - default: - ctx.Child("TYPE").EmitErrorMessage(fmt.Sprintf("invalid object store type: %v", c.Type)) - } -} - -type ObjectStoreType string - -const ( - ObjectStoreTypeAWSS3 ObjectStoreType = "AWS_S3" - ObjectStoreTypeGCPGCS ObjectStoreType = "GCP_GCS" - ObjectStoreTypeAzureBlobStorage ObjectStoreType = "AZURE_BLOB_STORAGE" -) - -type AWSS3ObjectStoreConfig struct { - BucketName string `envconfig:"BUCKET_NAME"` - Region string `envconfig:"REGION"` - AccessKeyID string `envconfig:"ACCESS_KEY_ID"` - SecretAccessKey string `envconfig:"SECRET_ACCESS_KEY"` -} - -func (c *AWSS3ObjectStoreConfig) Initialize(ctx *validation.Context) { -} - -func (c *AWSS3ObjectStoreConfig) Validate(ctx *validation.Context) { - if c.BucketName == "" { - ctx.Child("BUCKET_NAME").EmitErrorMessage("bucket name must be set") - } - if c.Region == "" { - ctx.Child("REGION").EmitErrorMessage("region must be set") - } - if c.AccessKeyID == "" { - ctx.Child("ACCESS_KEY_ID").EmitErrorMessage("access key id must be set") - } - if c.SecretAccessKey == "" { - ctx.Child("SECRET_ACCESS_KEY").EmitErrorMessage("secret key id must be set") - } -} - -type GCPGCSObjectStoreConfig struct { - BucketName string `envconfig:"BUCKET_NAME"` - ServiceAccount string `envconfig:"SERVICE_ACCOUNT"` - CredentialsJSONPath string `envconfig:"CREDENTIALS_JSON_PATH"` - CredentialsJSON []byte `ignored:"true"` -} - -func (c *GCPGCSObjectStoreConfig) Initialize(ctx *validation.Context) { - var err error - defer func() { - if err != nil { - ctx.Child("CREDENTIALS_JSON_PATH").EmitErrorMessage(err.Error()) - } - }() - - if c.CredentialsJSONPath == "" { - return - } - - p := c.CredentialsJSONPath - f, err := os.Open(p) - if err != nil { - return - } - defer f.Close() - - jsonBytes, err := ioutil.ReadAll(f) - if err != nil { - return - } - c.CredentialsJSON = jsonBytes -} - -func (c *GCPGCSObjectStoreConfig) Validate(ctx *validation.Context) { - if c.BucketName == "" { - ctx.Child("BUCKET_NAME").EmitErrorMessage("bucket name must be set") - } - if c.ServiceAccount == "" { - ctx.Child("SERVICE_ACCOUNT").EmitErrorMessage("service account must be set") - } - // In DEV-1689, we support Workload Identity, so service account key is no longer required. - // if c.CredentialsJSONPath == "" { - // ctx.Child("CREDENTIALS_JSON_PATH").EmitErrorMessage("credentials json must be set") - // } -} - -type AzureBlobStorageObjectStoreConfig struct { - StorageAccount string `envconfig:"STORAGE_ACCOUNT"` - Container string `envconfig:"CONTAINER"` - // ServiceURL is custom Azure blob storage URL. Empty for default URL. - ServiceURL string `envconfig:"SERVICE_URL"` - // AccessKey is encoded in standard BASE64. - AccessKey string `envconfig:"ACCESS_KEY"` -} - -func (c *AzureBlobStorageObjectStoreConfig) Initialize(ctx *validation.Context) { -} - -func (c *AzureBlobStorageObjectStoreConfig) Validate(ctx *validation.Context) { - if c.StorageAccount == "" { - ctx.Child("STORAGE_ACCOUNT").EmitErrorMessage("storage account must be set") - } - if c.Container == "" { - ctx.Child("CONTAINER").EmitErrorMessage("container must be set") - } - if c.AccessKey == "" { - ctx.Child("ACCESS_KEY").EmitErrorMessage("access key must be set") - } -} +type ObjectStoreConfig config.AbstractObjectStoreConfig diff --git a/pkg/images/deps/deps.go b/pkg/images/deps/deps.go index ee8f716d3b..ae7af5f701 100644 --- a/pkg/images/deps/deps.go +++ b/pkg/images/deps/deps.go @@ -6,6 +6,7 @@ import ( imagesconfig "github.com/authgear/authgear-server/pkg/images/config" imagesservice "github.com/authgear/authgear-server/pkg/images/service" "github.com/authgear/authgear-server/pkg/lib/cloudstorage" + "github.com/authgear/authgear-server/pkg/lib/config" "github.com/authgear/authgear-server/pkg/lib/deps" "github.com/authgear/authgear-server/pkg/lib/infra/db/appdb" "github.com/authgear/authgear-server/pkg/util/clock" @@ -13,7 +14,7 @@ import ( func NewCloudStorage(objectStoreConfig *imagesconfig.ObjectStoreConfig, c clock.Clock) imagesservice.ImagesCloudStorageServiceStorage { switch objectStoreConfig.Type { - case imagesconfig.ObjectStoreTypeAWSS3: + case config.ObjectStoreTypeAWSS3: s, err := cloudstorage.NewS3Storage( objectStoreConfig.AWSS3.AccessKeyID, objectStoreConfig.AWSS3.SecretAccessKey, @@ -24,7 +25,7 @@ func NewCloudStorage(objectStoreConfig *imagesconfig.ObjectStoreConfig, c clock. panic(err) } return s - case imagesconfig.ObjectStoreTypeGCPGCS: + case config.ObjectStoreTypeGCPGCS: s, err := cloudstorage.NewGCSStorage( objectStoreConfig.GCPGCS.CredentialsJSON, objectStoreConfig.GCPGCS.ServiceAccount, @@ -35,7 +36,7 @@ func NewCloudStorage(objectStoreConfig *imagesconfig.ObjectStoreConfig, c clock. panic(err) } return s - case imagesconfig.ObjectStoreTypeAzureBlobStorage: + case config.ObjectStoreTypeAzureBlobStorage: return cloudstorage.NewAzureStorage( objectStoreConfig.AzureBlobStorage.ServiceURL, objectStoreConfig.AzureBlobStorage.StorageAccount, diff --git a/pkg/lib/config/export_storage.go b/pkg/lib/config/abstract_object_store.go similarity index 92% rename from pkg/lib/config/export_storage.go rename to pkg/lib/config/abstract_object_store.go index f97c51f810..57b18b1313 100644 --- a/pkg/lib/config/export_storage.go +++ b/pkg/lib/config/abstract_object_store.go @@ -8,14 +8,17 @@ import ( "github.com/authgear/authgear-server/pkg/util/validation" ) -type UserExportObjectStoreConfig struct { +// AbstractObjectStoreConfig is a type to configure object store for a feature. +// This type IS NOT intended to be used directly. +// You should create a new type of this type. +type AbstractObjectStoreConfig struct { Type ObjectStoreType `envconfig:"TYPE"` AWSS3 AWSS3ObjectStoreConfig `envconfig:"AWS_S3"` GCPGCS GCPGCSObjectStoreConfig `envconfig:"GCP_GCS"` AzureBlobStorage AzureBlobStorageObjectStoreConfig `envconfig:"AZURE_BLOB_STORAGE"` } -func (c *UserExportObjectStoreConfig) Initialize(ctx *validation.Context) { +func (c *AbstractObjectStoreConfig) Initialize(ctx *validation.Context) { switch c.Type { case "": break @@ -30,7 +33,7 @@ func (c *UserExportObjectStoreConfig) Initialize(ctx *validation.Context) { } } -func (c *UserExportObjectStoreConfig) Validate(ctx *validation.Context) { +func (c *AbstractObjectStoreConfig) Validate(ctx *validation.Context) { switch c.Type { case "": break diff --git a/pkg/lib/config/userexport_object_store.go b/pkg/lib/config/userexport_object_store.go new file mode 100644 index 0000000000..20b3c521b4 --- /dev/null +++ b/pkg/lib/config/userexport_object_store.go @@ -0,0 +1,3 @@ +package config + +type UserExportObjectStoreConfig AbstractObjectStoreConfig From 0ad9ff4383bf9687c89c41d00cca2a84fa8c366f Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Mon, 9 Dec 2024 12:09:20 +0800 Subject: [PATCH 4/6] Install minio SDK v7 --- custombuild/go.mod | 7 +++++++ custombuild/go.sum | 15 +++++++++++++++ e2e/go.mod | 7 +++++++ e2e/go.sum | 15 +++++++++++++++ go.mod | 7 +++++++ go.sum | 15 +++++++++++++++ 6 files changed, 66 insertions(+) diff --git a/custombuild/go.mod b/custombuild/go.mod index 2d3a1768eb..f0f90862ae 100644 --- a/custombuild/go.mod +++ b/custombuild/go.mod @@ -54,6 +54,7 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect @@ -66,6 +67,7 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-gsm/charset v1.0.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-ldap/ldap/v3 v3.4.8 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -110,6 +112,8 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect @@ -123,6 +127,8 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.81 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -140,6 +146,7 @@ require ( github.com/redis/go-redis/v9 v9.7.0 // indirect github.com/relvacode/iso8601 v1.1.1-0.20210511065120-b30b151cc433 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/rubenv/sql-migrate v1.7.0 // indirect github.com/russellhaering/goxmldsig v1.4.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/custombuild/go.sum b/custombuild/go.sum index 0869e0f9da..fd1eb8883a 100644 --- a/custombuild/go.sum +++ b/custombuild/go.sum @@ -126,6 +126,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= @@ -163,6 +165,8 @@ github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-gsm/charset v1.0.0 h1:6k6LOKHtxgCPXE15X0unRewjOksyhmHBIp6x7qLQ6Ls= github.com/go-gsm/charset v1.0.0/go.mod h1:sC8+2VpAM2sZDlxv11MxWIZiuf8MipOgM/hCYsRQRps= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ= github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -330,6 +334,11 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -374,6 +383,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.81 h1:SzhMN0TQ6T/xSBu6Nvw3M5M8voM+Ht8RH3hE8S7zxaA= +github.com/minio/minio-go/v7 v7.0.81/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -436,6 +449,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/russellhaering/goxmldsig v1.4.0 h1:8UcDh/xGyQiyrW+Fq5t8f+l2DLB1+zlhYzkPUJ7Qhys= diff --git a/e2e/go.mod b/e2e/go.mod index 1d7c091698..65a41a5465 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -68,6 +68,7 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect @@ -78,6 +79,7 @@ require ( github.com/getsentry/sentry-go v0.29.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect github.com/go-gsm/charset v1.0.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-ldap/ldap/v3 v3.4.8 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -113,6 +115,8 @@ require ( github.com/jonboulle/clockwork v0.2.2 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect @@ -123,6 +127,8 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.81 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -139,6 +145,7 @@ require ( github.com/redis/go-redis/v9 v9.7.0 // indirect github.com/relvacode/iso8601 v1.1.1-0.20210511065120-b30b151cc433 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/russellhaering/goxmldsig v1.4.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect diff --git a/e2e/go.sum b/e2e/go.sum index 268fb13a28..05f3995cbd 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -129,6 +129,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -162,6 +164,8 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gsm/charset v1.0.0 h1:6k6LOKHtxgCPXE15X0unRewjOksyhmHBIp6x7qLQ6Ls= github.com/go-gsm/charset v1.0.0/go.mod h1:sC8+2VpAM2sZDlxv11MxWIZiuf8MipOgM/hCYsRQRps= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ= @@ -318,6 +322,11 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -361,6 +370,10 @@ github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqf github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.81 h1:SzhMN0TQ6T/xSBu6Nvw3M5M8voM+Ht8RH3hE8S7zxaA= +github.com/minio/minio-go/v7 v7.0.81/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -428,6 +441,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russellhaering/goxmldsig v1.4.0 h1:8UcDh/xGyQiyrW+Fq5t8f+l2DLB1+zlhYzkPUJ7Qhys= github.com/russellhaering/goxmldsig v1.4.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/go.mod b/go.mod index b70485ee18..9ae08b18c0 100644 --- a/go.mod +++ b/go.mod @@ -92,6 +92,7 @@ require ( github.com/coder/websocket v1.8.12 github.com/go-gsm/charset v1.0.0 github.com/go-ldap/ldap/v3 v3.4.8 + github.com/minio/minio-go/v7 v7.0.81 github.com/rivo/uniseg v0.4.7 github.com/russellhaering/goxmldsig v1.4.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 @@ -122,13 +123,19 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/rs/xid v1.6.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/otel/trace v1.32.0 // indirect diff --git a/go.sum b/go.sum index 4ac45695fe..d0a61dc0c4 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= @@ -163,6 +165,8 @@ github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-gsm/charset v1.0.0 h1:6k6LOKHtxgCPXE15X0unRewjOksyhmHBIp6x7qLQ6Ls= github.com/go-gsm/charset v1.0.0/go.mod h1:sC8+2VpAM2sZDlxv11MxWIZiuf8MipOgM/hCYsRQRps= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ= github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2jXlIXrk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -332,6 +336,11 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -376,6 +385,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.81 h1:SzhMN0TQ6T/xSBu6Nvw3M5M8voM+Ht8RH3hE8S7zxaA= +github.com/minio/minio-go/v7 v7.0.81/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -446,6 +459,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/russellhaering/goxmldsig v1.4.0 h1:8UcDh/xGyQiyrW+Fq5t8f+l2DLB1+zlhYzkPUJ7Qhys= From f8650e20dbbc00d294b0b86f5968c2d137753e61 Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Mon, 9 Dec 2024 13:30:15 +0800 Subject: [PATCH 5/6] Add MinIO implementation --- .env.example | 13 +++ pkg/images/deps/deps.go | 11 +++ pkg/lib/cloudstorage/minio.go | 108 ++++++++++++++++++++++++ pkg/lib/config/abstract_object_store.go | 31 +++++++ pkg/lib/userexport/deps.go | 11 +++ 5 files changed, 174 insertions(+) create mode 100644 pkg/lib/cloudstorage/minio.go diff --git a/.env.example b/.env.example index 37247a3de0..c4e7468594 100644 --- a/.env.example +++ b/.env.example @@ -100,6 +100,12 @@ SAML_IDP_ENTITY_ID_TEMPLATE=urn:{{.app_id}}.localhost #IMAGES_OBJECT_STORE_AZURE_BLOB_STORAGE_SERVICE_URL= #IMAGES_OBJECT_STORE_AZURE_BLOB_STORAGE_ACCESS_KEY= +IMAGES_OBJECT_STORE_TYPE=MINIO +IMAGES_OBJECT_STORE_MINIO_ENDPOINT=http://localhost:9000 +IMAGES_OBJECT_STORE_MINIO_BUCKET_NAME=images +IMAGES_OBJECT_STORE_MINIO_ACCESS_KEY_ID=minio +IMAGES_OBJECT_STORE_MINIO_SECRET_ACCESS_KEY=secretpassword + # User export server configs, uncomment accordingly depending on storage type @@ -120,6 +126,13 @@ SAML_IDP_ENTITY_ID_TEMPLATE=urn:{{.app_id}}.localhost #USEREXPORT_OBJECT_STORE_AZURE_BLOB_STORAGE_SERVICE_URL= #USEREXPORT_OBJECT_STORE_AZURE_BLOB_STORAGE_ACCESS_KEY= +USEREXPORT_OBJECT_STORE_TYPE=MINIO +USEREXPORT_OBJECT_STORE_MINIO_ENDPOINT=http://localhost:9000 +USEREXPORT_OBJECT_STORE_MINIO_BUCKET_NAME=userexport +USEREXPORT_OBJECT_STORE_MINIO_ACCESS_KEY_ID=minio +USEREXPORT_OBJECT_STORE_MINIO_SECRET_ACCESS_KEY=secretpassword + + UI_IMPLEMENTATION= UI_SETTINGS_IMPLEMENTATION= diff --git a/pkg/images/deps/deps.go b/pkg/images/deps/deps.go index ae7af5f701..5a4d71fadc 100644 --- a/pkg/images/deps/deps.go +++ b/pkg/images/deps/deps.go @@ -44,6 +44,17 @@ func NewCloudStorage(objectStoreConfig *imagesconfig.ObjectStoreConfig, c clock. objectStoreConfig.AzureBlobStorage.Container, c, ) + case config.ObjectStoreTypeMinIO: + s, err := cloudstorage.NewMinIOStorage( + objectStoreConfig.MinIO.Endpoint, + objectStoreConfig.MinIO.BucketName, + objectStoreConfig.MinIO.AccessKeyID, + objectStoreConfig.MinIO.SecretAccessKey, + ) + if err != nil { + panic(err) + } + return s default: return nil } diff --git a/pkg/lib/cloudstorage/minio.go b/pkg/lib/cloudstorage/minio.go new file mode 100644 index 0000000000..ef204791a2 --- /dev/null +++ b/pkg/lib/cloudstorage/minio.go @@ -0,0 +1,108 @@ +package cloudstorage + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type MinIOStorage struct { + Endpoint string + AccessKeyID string + SecretAccessKey string + Bucket string + + client *minio.Client +} + +var _ storage = (*MinIOStorage)(nil) + +// MinIOStorage takes endpoint, accessKeyID, and secretAccessKey to construct a minio.Client under the hood. +// Contradictory to minio.Client, endpoint MUST BE a http URL or https URL. +func NewMinIOStorage(endpoint string, bucketName string, accessKeyID string, secretAccessKey string) (*MinIOStorage, error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + var secure bool + switch u.Scheme { + case "http": + secure = false + case "https": + secure = true + default: + return nil, fmt.Errorf("minio: endpoint MUST BE a http URL or https URL: %v", endpoint) + } + + var emptyToken string + options := &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, emptyToken), + Secure: secure, + } + client, err := minio.New(u.Host, options) + if err != nil { + return nil, err + } + + return &MinIOStorage{ + Endpoint: endpoint, + Bucket: bucketName, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + client: client, + }, nil +} + +func (s *MinIOStorage) PresignPutObject(ctx context.Context, name string, header http.Header) (*http.Request, error) { + params := url.Values{} + u, err := s.client.PresignHeader(ctx, "PUT", s.Bucket, name, PresignPutExpires, params, header) + if err != nil { + return nil, fmt.Errorf("failed to presign put request: %w", err) + } + + return &http.Request{ + Method: "PUT", + URL: u, + Header: header, + }, nil +} + +func (s *MinIOStorage) PresignHeadObject(ctx context.Context, name string, expire time.Duration) (*url.URL, error) { + params := url.Values{} + u, err := s.client.PresignedHeadObject(ctx, s.Bucket, name, expire, params) + if err != nil { + return nil, fmt.Errorf("failed to presign head request: %w", err) + } + + return u, nil +} + +func (s *MinIOStorage) PresignGetObject(ctx context.Context, name string, expire time.Duration) (*url.URL, error) { + params := url.Values{} + u, err := s.client.PresignedGetObject(ctx, s.Bucket, name, expire, params) + if err != nil { + return nil, fmt.Errorf("failed to presign get request: %w", err) + } + + return u, nil +} + +func (s *MinIOStorage) MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { + return func(r *http.Request) { + key := extractKey(r) + params := url.Values{} + u, err := s.client.PresignedGetObject(r.Context(), s.Bucket, key, expire, params) + if err != nil { + panic(fmt.Errorf("failed to presign get request: %w", err)) + } + + r.Host = "" + r.URL = u + } +} diff --git a/pkg/lib/config/abstract_object_store.go b/pkg/lib/config/abstract_object_store.go index 57b18b1313..4ccba9d92a 100644 --- a/pkg/lib/config/abstract_object_store.go +++ b/pkg/lib/config/abstract_object_store.go @@ -16,6 +16,7 @@ type AbstractObjectStoreConfig struct { AWSS3 AWSS3ObjectStoreConfig `envconfig:"AWS_S3"` GCPGCS GCPGCSObjectStoreConfig `envconfig:"GCP_GCS"` AzureBlobStorage AzureBlobStorageObjectStoreConfig `envconfig:"AZURE_BLOB_STORAGE"` + MinIO MinIOObjectStoreConfig `envconfig:"MINIO"` } func (c *AbstractObjectStoreConfig) Initialize(ctx *validation.Context) { @@ -28,6 +29,8 @@ func (c *AbstractObjectStoreConfig) Initialize(ctx *validation.Context) { c.GCPGCS.Initialize(ctx.Child("GCP_GCS")) case ObjectStoreTypeAzureBlobStorage: c.AzureBlobStorage.Initialize(ctx.Child("AZURE_BLOB_STORAGE")) + case ObjectStoreTypeMinIO: + c.MinIO.Initialize(ctx.Child("MINIO")) default: ctx.Child("TYPE").EmitErrorMessage(fmt.Sprintf("invalid object store type: %v", c.Type)) } @@ -43,6 +46,8 @@ func (c *AbstractObjectStoreConfig) Validate(ctx *validation.Context) { c.GCPGCS.Validate(ctx.Child("GCP_GCS")) case ObjectStoreTypeAzureBlobStorage: c.AzureBlobStorage.Validate(ctx.Child("AZURE_BLOB_STORAGE")) + case ObjectStoreTypeMinIO: + c.MinIO.Validate(ctx.Child("MINIO")) default: ctx.Child("TYPE").EmitErrorMessage(fmt.Sprintf("invalid object store type: %v", c.Type)) } @@ -54,6 +59,7 @@ const ( ObjectStoreTypeAWSS3 ObjectStoreType = "AWS_S3" ObjectStoreTypeGCPGCS ObjectStoreType = "GCP_GCS" ObjectStoreTypeAzureBlobStorage ObjectStoreType = "AZURE_BLOB_STORAGE" + ObjectStoreTypeMinIO ObjectStoreType = "MINIO" ) type AWSS3ObjectStoreConfig struct { @@ -150,3 +156,28 @@ func (c *AzureBlobStorageObjectStoreConfig) Validate(ctx *validation.Context) { ctx.Child("ACCESS_KEY").EmitErrorMessage("access key must be set") } } + +type MinIOObjectStoreConfig struct { + Endpoint string `envconfig:"ENDPOINT"` + BucketName string `envconfig:"BUCKET_NAME"` + AccessKeyID string `envconfig:"ACCESS_KEY_ID"` + SecretAccessKey string `envconfig:"SECRET_ACCESS_KEY"` +} + +func (c *MinIOObjectStoreConfig) Initialize(ctx *validation.Context) { +} + +func (c *MinIOObjectStoreConfig) Validate(ctx *validation.Context) { + if c.Endpoint == "" { + ctx.Child("ENDPOINT").EmitErrorMessage("endpoint must be set") + } + if c.BucketName == "" { + ctx.Child("BUCKET_NAME").EmitErrorMessage("bucket name must be set") + } + if c.AccessKeyID == "" { + ctx.Child("ACCESS_KEY_ID").EmitErrorMessage("access key id must be set") + } + if c.SecretAccessKey == "" { + ctx.Child("SECRET_ACCESS_KEY").EmitErrorMessage("secret key id must be set") + } +} diff --git a/pkg/lib/userexport/deps.go b/pkg/lib/userexport/deps.go index 9915a91f2d..e913ca6f76 100644 --- a/pkg/lib/userexport/deps.go +++ b/pkg/lib/userexport/deps.go @@ -57,6 +57,17 @@ func NewCloudStorage(objectStoreConfig *config.UserExportObjectStoreConfig, c cl objectStoreConfig.AzureBlobStorage.Container, c, ) + case config.ObjectStoreTypeMinIO: + s, err := cloudstorage.NewMinIOStorage( + objectStoreConfig.MinIO.Endpoint, + objectStoreConfig.MinIO.BucketName, + objectStoreConfig.MinIO.AccessKeyID, + objectStoreConfig.MinIO.SecretAccessKey, + ) + if err != nil { + panic(err) + } + return s default: return nil } From e06cc06689aa7e4553c251e76258968bbce947ba Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Mon, 9 Dec 2024 15:38:37 +0800 Subject: [PATCH 6/6] Remove context from MakeDirector So that it is impossible to use an inappropriate context. We should use the context from the request. --- .vettedpositions | 4 +++- pkg/images/handler/get.go | 5 ++--- pkg/images/handler/get_mock_test.go | 9 ++++----- pkg/images/handler/get_test.go | 8 ++++---- pkg/images/service/images_cloud_storage_service.go | 6 +++--- pkg/lib/cloudstorage/azure.go | 4 ++-- pkg/lib/cloudstorage/cloudstorage.go | 2 +- pkg/lib/cloudstorage/gcs.go | 4 ++-- pkg/lib/cloudstorage/minio.go | 2 +- pkg/lib/cloudstorage/s3.go | 2 +- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.vettedpositions b/.vettedpositions index 69329285c2..50bd9cccc8 100644 --- a/.vettedpositions +++ b/.vettedpositions @@ -269,13 +269,15 @@ /pkg/images/deps/middleware_request.go:39:39: requestcontext /pkg/images/deps/middleware_request.go:40:34: requestcontext /pkg/images/deps/middleware_request.go:45:40: requestcontext -/pkg/images/handler/get.go:95:43: requestcontext /pkg/images/handler/post.go:162:83: requestcontext /pkg/images/handler/post.go:193:10: requestcontext /pkg/lib/accountmanagement/rate_limit_middleware.go:41:10: requestcontext /pkg/lib/admin/authz/middleware.go:85:18: requestcontext /pkg/lib/authenticationflow/intl_middleware.go:16:41: requestcontext /pkg/lib/authenticationflow/rate_limit_middleware.go:31:49: requestcontext +/pkg/lib/cloudstorage/azure.go:128:32: requestcontext +/pkg/lib/cloudstorage/gcs.go:139:38: requestcontext +/pkg/lib/cloudstorage/minio.go:100:41: requestcontext /pkg/lib/config/contextbackground.go:11:52: contextbackground /pkg/lib/deps/context.go:23:10: requestcontext /pkg/lib/deps/contextbackground.go:10:28: contextbackground diff --git a/pkg/images/handler/get.go b/pkg/images/handler/get.go index a619a60862..82aebc72ad 100644 --- a/pkg/images/handler/get.go +++ b/pkg/images/handler/get.go @@ -2,7 +2,6 @@ package handler import ( "bytes" - "context" "fmt" "io" "io/ioutil" @@ -63,7 +62,7 @@ func ParseImageVariant(s string) (ImageVariant, bool) { } type DirectorMaker interface { - MakeDirector(ctx context.Context, extractKey func(*http.Request) string) func(*http.Request) + MakeDirector(extractKey func(*http.Request) string) func(*http.Request) } type GetHandler struct { @@ -92,7 +91,7 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - director := h.DirectorMaker.MakeDirector(r.Context(), ExtractKey) + director := h.DirectorMaker.MakeDirector(ExtractKey) reverseProxy := httputil.ReverseProxy{ Director: director, diff --git a/pkg/images/handler/get_mock_test.go b/pkg/images/handler/get_mock_test.go index 630d299146..83f4ab6bd5 100644 --- a/pkg/images/handler/get_mock_test.go +++ b/pkg/images/handler/get_mock_test.go @@ -5,7 +5,6 @@ package handler import ( - context "context" http "net/http" reflect "reflect" @@ -75,15 +74,15 @@ func (m *MockDirectorMaker) EXPECT() *MockDirectorMakerMockRecorder { } // MakeDirector mocks base method. -func (m *MockDirectorMaker) MakeDirector(ctx context.Context, extractKey func(*http.Request) string) func(*http.Request) { +func (m *MockDirectorMaker) MakeDirector(extractKey func(*http.Request) string) func(*http.Request) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakeDirector", ctx, extractKey) + ret := m.ctrl.Call(m, "MakeDirector", extractKey) ret0, _ := ret[0].(func(*http.Request)) return ret0 } // MakeDirector indicates an expected call of MakeDirector. -func (mr *MockDirectorMakerMockRecorder) MakeDirector(ctx, extractKey interface{}) *gomock.Call { +func (mr *MockDirectorMakerMockRecorder) MakeDirector(extractKey interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeDirector", reflect.TypeOf((*MockDirectorMaker)(nil).MakeDirector), ctx, extractKey) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeDirector", reflect.TypeOf((*MockDirectorMaker)(nil).MakeDirector), extractKey) } diff --git a/pkg/images/handler/get_test.go b/pkg/images/handler/get_test.go index 2a576ba3ec..bb98af8648 100644 --- a/pkg/images/handler/get_test.go +++ b/pkg/images/handler/get_test.go @@ -37,7 +37,7 @@ func TestGetHandler(t *testing.T) { r, _ := http.NewRequest("GET", "http://localhost:3004/_images/app/image.jpg/profile", nil) w := httptest.NewRecorder() - directorMaker.EXPECT().MakeDirector(gomock.Any(), gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) + directorMaker.EXPECT().MakeDirector(gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) gock.New("http://localhost:3004").Reply(404) router.HTTPHandler().ServeHTTP(w, r) @@ -50,7 +50,7 @@ func TestGetHandler(t *testing.T) { r, _ := http.NewRequest("GET", "http://localhost:3004/_images/app/image.jpg/invalid", nil) w := httptest.NewRecorder() - directorMaker.EXPECT().MakeDirector(gomock.Any(), gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) + directorMaker.EXPECT().MakeDirector(gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) router.HTTPHandler().ServeHTTP(w, r) So(w.Result().StatusCode, ShouldEqual, 404) @@ -62,7 +62,7 @@ func TestGetHandler(t *testing.T) { r, _ := http.NewRequest("GET", "http://localhost:3004/_images/app/image.jpg/profile", nil) w := httptest.NewRecorder() - directorMaker.EXPECT().MakeDirector(gomock.Any(), gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) + directorMaker.EXPECT().MakeDirector(gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) vipsDaemon.EXPECT().Process(gomock.Any()).Times(1).Return(&vipsutil.Output{ Data: nil, FileExtension: "", @@ -81,7 +81,7 @@ func TestGetHandler(t *testing.T) { r, _ := http.NewRequest("GET", "http://localhost:3004/_images/app/image.jpg/profile", nil) w := httptest.NewRecorder() - directorMaker.EXPECT().MakeDirector(gomock.Any(), gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) + directorMaker.EXPECT().MakeDirector(gomock.Any()).AnyTimes().Return(func(r *http.Request) {}) vipsDaemon.EXPECT().Process(gomock.Any()).Times(1).Return(&vipsutil.Output{ Data: nil, FileExtension: ".jpeg", diff --git a/pkg/images/service/images_cloud_storage_service.go b/pkg/images/service/images_cloud_storage_service.go index a47c443c8b..aa30df55ac 100644 --- a/pkg/images/service/images_cloud_storage_service.go +++ b/pkg/images/service/images_cloud_storage_service.go @@ -20,7 +20,7 @@ const MaxContentLength = 10 * 1024 * 1024 type ImagesCloudStorageServiceStorage interface { PresignPutObject(ctx context.Context, name string, header http.Header) (*http.Request, error) PresignHeadObject(ctx context.Context, name string, expire time.Duration) (*url.URL, error) - MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) + MakeDirector(extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) } type ImagesCloudStorageServiceHTTPClient struct { @@ -79,6 +79,6 @@ func (p *ImagesCloudStorageService) checkDuplicate(ctx context.Context, key stri return apierrors.AlreadyExists.WithReason("DuplicatedImage").Errorf("duplicated image") } -func (p *ImagesCloudStorageService) MakeDirector(ctx context.Context, extractKey func(r *http.Request) string) func(r *http.Request) { - return p.Storage.MakeDirector(ctx, extractKey, PresignGetExpires) +func (p *ImagesCloudStorageService) MakeDirector(extractKey func(r *http.Request) string) func(r *http.Request) { + return p.Storage.MakeDirector(extractKey, PresignGetExpires) } diff --git a/pkg/lib/cloudstorage/azure.go b/pkg/lib/cloudstorage/azure.go index 6fc7f886e4..175d229801 100644 --- a/pkg/lib/cloudstorage/azure.go +++ b/pkg/lib/cloudstorage/azure.go @@ -122,10 +122,10 @@ func (s *AzureStorage) SignedURL(name string, now time.Time, duration time.Durat return &u, nil } -func (s *AzureStorage) MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { +func (s *AzureStorage) MakeDirector(extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { return func(r *http.Request) { key := extractKey(r) - u, err := s.PresignGetObject(ctx, key, expire) + u, err := s.PresignGetObject(r.Context(), key, expire) if err != nil { panic(err) } diff --git a/pkg/lib/cloudstorage/cloudstorage.go b/pkg/lib/cloudstorage/cloudstorage.go index 12f0a2972a..b41b805d3d 100644 --- a/pkg/lib/cloudstorage/cloudstorage.go +++ b/pkg/lib/cloudstorage/cloudstorage.go @@ -20,5 +20,5 @@ type storage interface { // PresignGetObject returns an URL that is ready for use. PresignGetObject(ctx context.Context, name string, expire time.Duration) (*url.URL, error) // MakeDirector takes extractKey and returns a Director of httputil.ReverseProxy. - MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) + MakeDirector(extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) } diff --git a/pkg/lib/cloudstorage/gcs.go b/pkg/lib/cloudstorage/gcs.go index 9abe59b44f..24fa8e565a 100644 --- a/pkg/lib/cloudstorage/gcs.go +++ b/pkg/lib/cloudstorage/gcs.go @@ -133,10 +133,10 @@ func (s *GCSStorage) PresignGetObject(ctx context.Context, name string, expire t return s.PresignGetOrHeadObject(ctx, name, "GET", expire) } -func (s *GCSStorage) MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { +func (s *GCSStorage) MakeDirector(extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { return func(r *http.Request) { key := extractKey(r) - u, err := s.PresignGetOrHeadObject(ctx, key, "GET", expire) + u, err := s.PresignGetOrHeadObject(r.Context(), key, "GET", expire) if err != nil { panic(err) } diff --git a/pkg/lib/cloudstorage/minio.go b/pkg/lib/cloudstorage/minio.go index ef204791a2..62ed2963ed 100644 --- a/pkg/lib/cloudstorage/minio.go +++ b/pkg/lib/cloudstorage/minio.go @@ -93,7 +93,7 @@ func (s *MinIOStorage) PresignGetObject(ctx context.Context, name string, expire return u, nil } -func (s *MinIOStorage) MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { +func (s *MinIOStorage) MakeDirector(extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { return func(r *http.Request) { key := extractKey(r) params := url.Values{} diff --git a/pkg/lib/cloudstorage/s3.go b/pkg/lib/cloudstorage/s3.go index 8445b2e70e..c09a37207d 100644 --- a/pkg/lib/cloudstorage/s3.go +++ b/pkg/lib/cloudstorage/s3.go @@ -126,7 +126,7 @@ func (s *S3Storage) PresignGetObject(ctx context.Context, name string, expire ti return u, nil } -func (s *S3Storage) MakeDirector(ctx context.Context, extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { +func (s *S3Storage) MakeDirector(extractKey func(r *http.Request) string, expire time.Duration) func(r *http.Request) { return func(r *http.Request) { key := extractKey(r) input := &s3.GetObjectInput{