Skip to content

Commit

Permalink
Migrate to aws-sdk-go-v2 (#81)
Browse files Browse the repository at this point in the history
  • Loading branch information
@robertvolkmann
robertvolkmann authored Jan 17, 2025
1 parent c859648 commit b5ad3b2
Show file tree
Hide file tree
Showing 6 changed files with 245 additions and 264 deletions.
173 changes: 67 additions & 106 deletions cmd/internal/backup/providers/s3/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,21 @@ package s3

import (
"context"
"errors"
"log/slog"
"path/filepath"
"strings"

"errors"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/spf13/afero"

"github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers"
"github.com/metal-stack/backup-restore-sidecar/pkg/constants"
"github.com/spf13/afero"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

const (
Expand All @@ -28,8 +27,7 @@ const (
type BackupProviderS3 struct {
fs afero.Fs
log *slog.Logger
c *s3.S3
sess *session.Session
c *s3.Client
config *BackupProviderConfigS3
}

Expand All @@ -42,7 +40,7 @@ type BackupProviderConfigS3 struct {
SecretKey string
BackupName string
ObjectPrefix string
ObjectsToKeep int64
ObjectsToKeep int32
FS afero.Fs
}

Expand All @@ -64,120 +62,93 @@ func (c *BackupProviderConfigS3) validate() error {
}

// New returns a S3 backup provider
func New(log *slog.Logger, config *BackupProviderConfigS3) (*BackupProviderS3, error) {
if config == nil {
func New(log *slog.Logger, cfg *BackupProviderConfigS3) (*BackupProviderS3, error) {
if cfg == nil {
return nil, errors.New("s3 backup provider requires a provider config")
}

if config.ObjectsToKeep == 0 {
config.ObjectsToKeep = constants.DefaultObjectsToKeep
if cfg.ObjectsToKeep == 0 {
cfg.ObjectsToKeep = constants.DefaultObjectsToKeep
}
if config.BackupName == "" {
config.BackupName = defaultBackupName
if cfg.BackupName == "" {
cfg.BackupName = defaultBackupName
}
if config.FS == nil {
config.FS = afero.NewOsFs()
if cfg.FS == nil {
cfg.FS = afero.NewOsFs()
}

err := config.validate()
err := cfg.validate()
if err != nil {
return nil, err
}
s3Config := &aws.Config{
Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""),
Endpoint: aws.String(config.Endpoint),
Region: aws.String(config.Region),
S3ForcePathStyle: aws.Bool(true),
}
newSession, err := session.NewSession(s3Config)

s3Cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "")),
config.WithRegion(cfg.Region),
)
if err != nil {
return nil, err
}
client := s3.New(newSession)
client := s3.NewFromConfig(s3Cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(cfg.Endpoint)
o.UsePathStyle = true
})

return &BackupProviderS3{
c: client,
sess: newSession,
config: config,
config: cfg,
log: log,
fs: config.FS,
fs: cfg.FS,
}, nil
}

// EnsureBackupBucket ensures a backup bucket at the backup provider
func (b *BackupProviderS3) EnsureBackupBucket(ctx context.Context) error {
bucket := aws.String(b.config.BucketName)

// create bucket
cparams := &s3.CreateBucketInput{
Bucket: bucket,
}

_, err := b.c.CreateBucketWithContext(ctx, cparams)
_, err := b.c.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(b.config.BucketName),
})
if err != nil {
// FIXME check how to migrate to errors.As
//nolint
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeBucketAlreadyExists:
case s3.ErrCodeBucketAlreadyOwnedByYou:
default:
return err
}
} else {
var (
bucketAlreadyExists *types.BucketAlreadyExists
bucketAlreadyOwnerByYou *types.BucketAlreadyOwnedByYou
)
if !errors.As(err, &bucketAlreadyExists) && !errors.As(err, &bucketAlreadyOwnerByYou) {
return err
}
}

// enable versioning
versioning := &s3.PutBucketVersioningInput{
Bucket: bucket,
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String("Enabled"),
_, err = b.c.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: aws.String(b.config.BucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
}
_, err = b.c.PutBucketVersioningWithContext(ctx, versioning)
})
if err != nil {
// FIXME check how to migrate to errors.As
//nolint
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
default:
return err
}
} else {
return err
}
return err
}

// add lifecycle policy
lifecycle := &s3.PutBucketLifecycleConfigurationInput{
Bucket: bucket,
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: []*s3.LifecycleRule{
_, err = b.c.PutBucketLifecycleConfiguration(ctx, &s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(b.config.BucketName),
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
Rules: []types.LifecycleRule{
{
NoncurrentVersionExpiration: &s3.NoncurrentVersionExpiration{
NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
NoncurrentDays: &b.config.ObjectsToKeep,
},
Status: aws.String("Enabled"),
Status: types.ExpirationStatusEnabled,
ID: aws.String("backup-restore-lifecycle"),
Prefix: &b.config.ObjectPrefix,
Filter: &types.LifecycleRuleFilter{
Prefix: aws.String(b.config.ObjectPrefix),
},
},
},
},
}
_, err = b.c.PutBucketLifecycleConfigurationWithContext(ctx, lifecycle)
})
if err != nil {
// FIXME check how to migrate to errors.As
//nolint
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
default:
return err
}
} else {
return err
}
return err
}
return nil
}
Expand All @@ -188,10 +159,8 @@ func (b *BackupProviderS3) CleanupBackups(_ context.Context) error {
return nil
}

// DownloadBackup downloads the given backup version to the specified folder
// DownloadBackup downloads the given backup version to the restoration folder
func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *providers.BackupVersion, outDir string) (string, error) {
bucket := aws.String(b.config.BucketName)

downloadFileName := version.Name
if strings.Contains(downloadFileName, "/") {
downloadFileName = filepath.Base(downloadFileName)
Expand All @@ -205,16 +174,12 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider
}
defer f.Close()

downloader := s3manager.NewDownloader(b.sess)

_, err = downloader.DownloadWithContext(
ctx,
f,
&s3.GetObjectInput{
Bucket: bucket,
Key: &version.Name,
VersionId: &version.Version,
})
downloader := manager.NewDownloader(b.c)
_, err = downloader.Download(ctx, f, &s3.GetObjectInput{
Bucket: aws.String(b.config.BucketName),
Key: &version.Name,
VersionId: &version.Version,
})
if err != nil {
return "", err
}
Expand All @@ -224,8 +189,6 @@ func (b *BackupProviderS3) DownloadBackup(ctx context.Context, version *provider

// UploadBackup uploads a backup to the backup provider
func (b *BackupProviderS3) UploadBackup(ctx context.Context, sourcePath string) error {
bucket := aws.String(b.config.BucketName)

r, err := b.fs.Open(sourcePath)
if err != nil {
return err
Expand All @@ -239,9 +202,9 @@ func (b *BackupProviderS3) UploadBackup(ctx context.Context, sourcePath string)

b.log.Debug("uploading object", "src", sourcePath, "dest", destination)

uploader := s3manager.NewUploader(b.sess)
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: bucket,
uploader := manager.NewUploader(b.c)
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(b.config.BucketName),
Key: aws.String(destination),
Body: r,
})
Expand All @@ -260,10 +223,8 @@ func (b *BackupProviderS3) GetNextBackupName(_ context.Context) string {

// ListBackups lists the available backups of the backup provider
func (b *BackupProviderS3) ListBackups(ctx context.Context) (providers.BackupVersions, error) {
bucket := aws.String(b.config.BucketName)

it, err := b.c.ListObjectVersionsWithContext(ctx, &s3.ListObjectVersionsInput{
Bucket: bucket,
it, err := b.c.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
Bucket: aws.String(b.config.BucketName),
Prefix: &b.config.ObjectPrefix,
})
if err != nil {
Expand Down
5 changes: 3 additions & 2 deletions cmd/internal/backup/providers/s3/versions.go
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package s3

import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"

"github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers"
"github.com/metal-stack/backup-restore-sidecar/cmd/internal/backup/providers/common"
)

// backupVersionsS3 contains the list of available backup versions
type backupVersionsS3 struct {
objectAttrs []*s3.ObjectVersion
objectAttrs []types.ObjectVersion
}

// Latest returns latest backup version
Expand Down
2 changes: 1 addition & 1 deletion cmd/internal/encryption/encryption.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func (e *Encrypter) openOutputFile(output string) (afero.File, error) {
return e.fs.OpenFile(output, os.O_RDWR|os.O_CREATE, 0644)
}

// generateIV() returns unique initalization vector of same size as cipher block for encryption
// generateIV() returns unique initialization vector of same size as cipher block for encryption
func (e *Encrypter) generateIV(block cipher.Block) ([]byte, error) {
iv := make([]byte, block.BlockSize())
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ func initBackupProvider() error {
logger.WithGroup("backup"),
&s3.BackupProviderConfigS3{
ObjectPrefix: viper.GetString(objectPrefixFlg),
ObjectsToKeep: viper.GetInt64(objectsToKeepFlg),
ObjectsToKeep: viper.GetInt32(objectsToKeepFlg),
Region: viper.GetString(s3RegionFlg),
BucketName: viper.GetString(s3BucketNameFlg),
Endpoint: viper.GetString(s3EndpointFlg),
Expand Down
Loading

0 comments on commit b5ad3b2

Please sign in to comment.