blob: added url builder;

.github/workflows/go.yml

@@ -23,10 +23,10 @@ jobs:
     name: go fmt
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -72,10 +72,10 @@ jobs:
     name: go build
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -90,10 +90,10 @@ jobs:
     name: static code analysis (go vet)
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -108,10 +108,10 @@ jobs:
     name: static code analysis (golangci-lint)
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -126,10 +126,10 @@ jobs:
     name: go test
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -144,10 +144,10 @@ jobs:
     name: go test (race)
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -162,10 +162,10 @@ jobs:
     name: go test (integration)
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -180,10 +180,10 @@ jobs:
     name: go test (integration, race)
     runs-on: ubuntu-20.04
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.20
         uses: actions/setup-go@v3
         with:
-          go-version: "1.18"
+          go-version: "1.20"
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3

.tool-versions

@@ -1 +1,2 @@
 golangci-lint 1.45.2
+mockery 2.22.1

docs/log/usage.go

@@ -1,4 +1,4 @@
-//nolint
+// nolint
 package main
 
 import (

go.mod

@@ -184,4 +184,4 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
-go 1.18
+go 1.20

pkg/apiserver/handler.go

@@ -3,7 +3,7 @@ package apiserver
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -295,7 +295,7 @@ func handleWithMultipleBindings(handler HandlerWithMultipleBindings, errHandler
 
 func handleRaw(handler HandlerWithoutInput, errHandler ErrorHandler) gin.HandlerFunc {
 	return func(ginCtx *gin.Context) {
-		body, err := ioutil.ReadAll(ginCtx.Request.Body)
+		body, err := io.ReadAll(ginCtx.Request.Body)
 		if err != nil {
 			handleError(ginCtx, errHandler, http.StatusBadRequest, gin.Error{
 				Err:  err,

pkg/blob/store.go

@@ -122,15 +122,7 @@ func CreateKey() string {
 
 func NewStore(ctx context.Context, config cfg.Config, logger log.Logger, name string) (Store, error) {
 	channels := ProvideBatchRunnerChannels(config)
-
-	var settings Settings
-	key := fmt.Sprintf("blob.%s", name)
-	config.UnmarshalKey(key, &settings)
-	settings.AppId.PadFromConfig(config)
-
-	if settings.Bucket == "" {
-		settings.Bucket = fmt.Sprintf("%s-%s-%s", settings.Project, settings.Environment, settings.Family)
-	}
+	settings := getStoreSettings(config, name)
 
 	s3Client, err := gosoS3.ProvideClient(ctx, config, logger, settings.ClientName)
 	if err != nil {
@@ -149,7 +141,7 @@ func NewStore(ctx context.Context, config cfg.Config, logger log.Logger, name st
 	return store, nil
 }
 
-func NewStoreWithInterfaces(logger log.Logger, channels *BatchRunnerChannels, client gosoS3.Client, settings Settings) Store {
+func NewStoreWithInterfaces(logger log.Logger, channels *BatchRunnerChannels, client gosoS3.Client, settings *Settings) Store {
 	return &s3Store{
 		logger:   logger,
 		channels: channels,
@@ -358,3 +350,16 @@ func isBucketAlreadyExistsError(err error) bool {
 
 	return false
 }
+
+func getStoreSettings(config cfg.Config, name string) *Settings {
+	settings := &Settings{}
+	key := fmt.Sprintf("blob.%s", name)
+	config.UnmarshalKey(key, settings)
+	settings.AppId.PadFromConfig(config)
+
+	if settings.Bucket == "" {
+		settings.Bucket = fmt.Sprintf("%s-%s-%s", settings.Project, settings.Environment, settings.Family)
+	}
+
+	return settings
+}

pkg/blob/stream.go

@@ -3,7 +3,6 @@ package blob
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"sync/atomic"
 
 	"github.com/pkg/errors"
@@ -15,13 +14,15 @@ type ReadCloser interface {
 }
 
 // A reader that we can close and that can seek
+//
 //go:generate mockery --name ReadSeekerCloser
 type ReadSeekerCloser interface {
 	io.ReadSeeker
 	io.Closer
 }
 
 // A stream is a source of bytes you can either get as a full []byte or stream as a reader.
+//
 //go:generate mockery --name Stream
 type Stream interface {
 	// Read all data and close the reader.
@@ -86,7 +87,7 @@ type readerStream struct {
 }
 
 func (r readerStream) ReadAll() ([]byte, error) {
-	b, err := ioutil.ReadAll(r.reader)
+	b, err := io.ReadAll(r.reader)
 
 	if err == nil {
 		err = r.reader.Close()
@@ -106,7 +107,7 @@ type noSeekerReaderStream struct {
 }
 
 func (r noSeekerReaderStream) ReadAll() ([]byte, error) {
-	b, err := ioutil.ReadAll(r.reader)
+	b, err := io.ReadAll(r.reader)
 
 	if err == nil {
 		err = r.reader.Close()

pkg/blob/url_builder.go

@@ -0,0 +1,57 @@
+package blob
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/justtrackio/gosoline/pkg/cfg"
+	"github.com/justtrackio/gosoline/pkg/cloud/aws/s3"
+)
+
+//go:generate mockery --name UrlBuilder
+type UrlBuilder interface {
+	GetAbsoluteUrl(path string) (string, error)
+}
+
+type urlBuilder struct {
+	endpoint     string
+	usePathStyle bool
+	bucket       string
+}
+
+func NewUrlBuilder(config cfg.Config, name string) (UrlBuilder, error) {
+	storeSettings := getStoreSettings(config, name)
+	clientConfig := s3.GetClientConfig(config, storeSettings.ClientName)
+
+	var err error
+	var endpoint aws.Endpoint
+
+	if endpoint, err = s3.ResolveEndpoint(config, storeSettings.ClientName); err != nil {
+		return nil, fmt.Errorf("can not resolve s3 endpoint for client %s: %w", storeSettings.ClientName, err)
+	}
+
+	return &urlBuilder{
+		endpoint:     endpoint.URL,
+		usePathStyle: clientConfig.Settings.UsePathStyle,
+		bucket:       storeSettings.Bucket,
+	}, nil
+}
+
+func (b *urlBuilder) GetAbsoluteUrl(path string) (string, error) {
+	var err error
+	var blobUrl *url.URL
+
+	if blobUrl, err = blobUrl.Parse(b.endpoint); err != nil {
+		return "", fmt.Errorf("can not parse endpoint %s: %w", b.endpoint, err)
+	}
+
+	if b.usePathStyle {
+		blobUrl = blobUrl.JoinPath(b.bucket, path)
+	} else {
+		blobUrl = blobUrl.JoinPath(path)
+		blobUrl.Host = fmt.Sprintf("%s.%s", b.bucket, blobUrl.Host)
+	}
+
+	return blobUrl.String(), nil
+}