add s3 storage wrappers

Signed-off-by: Amruta Kale <[email protected]>

pkg/kopialib/const.go

@@ -0,0 +1,45 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kopialib
+
+const (
+	// Common storage consts
+	BucketKey = "bucket"
+	PrefixKey = "prefix"
+
+	// S3 storage consts
+	S3EndpointKey     = "endpoint"
+	S3RegionKey       = "region"
+	SkipSSLVerifyKey  = "skipSSLVerify"
+	S3AccessKey       = "accessKeyID"
+	S3SecretAccessKey = "secretAccessKey"
+	S3TokenKey        = "sessionToken"
+	DoNotUseTLS       = "doNotUseTLS"
+	DoNotVerifyTLS    = "doNotVerifyTLS"
+
+	// Azure storage consts
+	AzureStorageAccount          = "storageAccount"
+	AzureStorageAccountAccessKey = "storageKey"
+	AzureSASToken                = "sasToken"
+
+	//Filestore storage consts
+	FilesystorePath    = "path"
+	DefaultFSMountPath = "/mnt/data"
+
+	//GCP storage consts
+	GCPServiceAccountCredentialsFile = "serviceAccountCredentialsFile"
+	GCPServiceAccountCredentialJSON  = "serviceAccountCredentialsJson"
+	GCPReadOnly                      = "readOnly"
+)

pkg/kopialib/storage/errors.go

@@ -0,0 +1,20 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+var (
+	ErrMissingRequiredFieldMsg      = "missing required field: %s for storage provider %s"
+	ErrStorageOptionsCannotBeNilMsg = "storage options cannot be nil for the provider %s"
+)

pkg/kopialib/storage/s3.go

@@ -0,0 +1,67 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+
+	"github.com/kanisterio/kanister/pkg/kopialib"
+	"github.com/kanisterio/kanister/pkg/utils"
+	"github.com/kopia/kopia/repo/blob"
+	"github.com/kopia/kopia/repo/blob/s3"
+)
+
+var _ Storage = &s3Storage{}
+
+var requiredS3Arguments = []string{
+	kopialib.BucketKey,
+}
+
+type s3Storage struct {
+	Options *s3.Options
+	Create  bool
+}
+
+func (s *s3Storage) Connect() (blob.Storage, error) {
+	return s3.New(context.Background(), s.Options, s.Create)
+}
+
+func (s *s3Storage) WithOptions(opts s3.Options) {
+	s.Options = &opts
+}
+
+func (s *s3Storage) WithCreate(create bool) {
+	s.Create = create
+}
+
+func (s *s3Storage) SetOptions(ctx context.Context, options map[string]string) error {
+	err := validateCommonStorageArgs(options, TypeS3, requiredS3Arguments)
+	if err != nil {
+		return err
+	}
+	s.Options = &s3.Options{
+		BucketName:      options[kopialib.BucketKey],
+		Endpoint:        options[kopialib.S3EndpointKey],
+		Prefix:          options[kopialib.PrefixKey],
+		Region:          options[kopialib.S3RegionKey],
+		SessionToken:    options[kopialib.S3TokenKey],
+		AccessKeyID:     options[kopialib.S3AccessKey],
+		SecretAccessKey: options[kopialib.S3SecretAccessKey],
+	}
+	s.Options.DoNotUseTLS, _ = utils.GetBoolOrDefault(options[kopialib.DoNotUseTLS], true)
+	s.Options.DoNotVerifyTLS, _ = utils.GetBoolOrDefault(options[kopialib.DoNotVerifyTLS], true)
+
+	return nil
+}

pkg/kopialib/storage/s3_test.go

@@ -0,0 +1,138 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/kanisterio/kanister/pkg/kopialib"
+	"github.com/kopia/kopia/repo/blob/s3"
+	. "gopkg.in/check.v1"
+)
+
+type s3StorageTestSuite struct{}
+
+var _ = Suite(&s3StorageTestSuite{})
+
+func (s *s3StorageTestSuite) TestSetOptions(c *C) {
+	for i, tc := range []struct {
+		name            string
+		options         map[string]string
+		expectedOptions s3.Options
+		expectedErr     string
+		errChecker      Checker
+	}{
+		{
+			name:        "options not set",
+			options:     map[string]string{},
+			expectedErr: fmt.Sprintf(ErrStorageOptionsCannotBeNilMsg, TypeS3),
+			errChecker:  NotNil,
+		},
+		{
+			name: "bucket name is required",
+			options: map[string]string{
+				kopialib.S3EndpointKey: "test-endpoint",
+			},
+			expectedErr: fmt.Sprintf(ErrMissingRequiredFieldMsg, kopialib.BucketKey, TypeS3),
+			errChecker:  NotNil,
+		},
+		{
+			name: "set correct options",
+			options: map[string]string{
+				kopialib.BucketKey:         "test-bucket",
+				kopialib.S3EndpointKey:     "test-endpoint",
+				kopialib.S3RegionKey:       "test-region",
+				kopialib.S3AccessKey:       "test-access-key",
+				kopialib.S3SecretAccessKey: "test-secret-access-key",
+				kopialib.S3TokenKey:        "test-s3-token",
+				kopialib.PrefixKey:         "test-prefix",
+			},
+			expectedOptions: s3.Options{
+				BucketName:      "test-bucket",
+				Endpoint:        "test-endpoint",
+				Region:          "test-region",
+				AccessKeyID:     "test-access-key",
+				SecretAccessKey: "test-secret-access-key",
+				Prefix:          "test-prefix",
+				SessionToken:    "test-s3-token",
+				DoNotUseTLS:     true,
+				DoNotVerifyTLS:  true,
+			},
+			errChecker: IsNil,
+		},
+		{
+			name: "set TLS",
+			options: map[string]string{
+				kopialib.BucketKey:         "test-bucket",
+				kopialib.S3EndpointKey:     "test-endpoint",
+				kopialib.S3RegionKey:       "test-region",
+				kopialib.S3AccessKey:       "test-access-key",
+				kopialib.S3SecretAccessKey: "test-secret-access-key",
+				kopialib.S3TokenKey:        "test-s3-token",
+				kopialib.PrefixKey:         "test-prefix",
+				kopialib.DoNotUseTLS:       "false",
+			},
+			expectedOptions: s3.Options{
+				BucketName:      "test-bucket",
+				Endpoint:        "test-endpoint",
+				Region:          "test-region",
+				AccessKeyID:     "test-access-key",
+				SecretAccessKey: "test-secret-access-key",
+				Prefix:          "test-prefix",
+				SessionToken:    "test-s3-token",
+				DoNotUseTLS:     false,
+				DoNotVerifyTLS:  true,
+			},
+			errChecker: IsNil,
+		},
+		{
+			name: "set verify TLS",
+			options: map[string]string{
+				kopialib.BucketKey:         "test-bucket",
+				kopialib.S3EndpointKey:     "test-endpoint",
+				kopialib.S3RegionKey:       "test-region",
+				kopialib.S3AccessKey:       "test-access-key",
+				kopialib.S3SecretAccessKey: "test-secret-access-key",
+				kopialib.S3TokenKey:        "test-s3-token",
+				kopialib.PrefixKey:         "test-prefix",
+				kopialib.DoNotUseTLS:       "false",
+				kopialib.DoNotVerifyTLS:    "false",
+			},
+			expectedOptions: s3.Options{
+				BucketName:      "test-bucket",
+				Endpoint:        "test-endpoint",
+				Region:          "test-region",
+				AccessKeyID:     "test-access-key",
+				SecretAccessKey: "test-secret-access-key",
+				Prefix:          "test-prefix",
+				SessionToken:    "test-s3-token",
+				DoNotUseTLS:     false,
+				DoNotVerifyTLS:  false,
+			},
+			errChecker: IsNil,
+		},
+	} {
+		s3Storage := s3Storage{}
+		err := s3Storage.SetOptions(context.Background(), tc.options)
+		c.Check(err, tc.errChecker)
+		if err != nil {
+			c.Check(err.Error(), Equals, tc.expectedErr, Commentf("test number: %d", i))
+		}
+
+		c.Assert(s3Storage.Options, DeepEquals, tc.expectedOptions)
+
+	}
+}

pkg/kopialib/storage/storage_interface.go

@@ -0,0 +1,58 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+
+	"github.com/kopia/kopia/repo/blob"
+	"github.com/pkg/errors"
+)
+
+type StorageType string
+
+const (
+	TypeS3        StorageType = "S3"
+	TypeAzure     StorageType = "Azure"
+	TypeFileStore StorageType = "FileStore"
+	TypeGCP       StorageType = "GCP"
+)
+
+type Storage interface {
+	Connect() (blob.Storage, error)
+	SetOptions(context.Context, map[string]string) error
+	WithCreate(bool)
+}
+
+func New(storageType StorageType) Storage {
+	switch storageType {
+	case TypeS3:
+		return &s3Storage{}
+	default:
+		return nil
+	}
+}
+
+func validateCommonStorageArgs(options map[string]string, storageType StorageType, requiredArgs []string) error {
+	if options == nil {
+		return errors.Errorf(ErrStorageOptionsCannotBeNilMsg, string(storageType))
+	}
+	for _, arg := range requiredArgs {
+		if _, ok := options[arg]; !ok {
+			return errors.Errorf(ErrMissingRequiredFieldMsg, arg, string(storageType))
+		}
+	}
+	return nil
+}

pkg/utils/utils.go

@@ -89,6 +89,19 @@ func GetIntOrDefault(value string, defaultValue int) (int, error) {
 	return v, nil
 }
 
+func GetBoolOrDefault(value string, defaultValue bool) (bool, error) {
+	if value != "" {
+		ret, err := strconv.ParseBool(value)
+		if err == nil {
+			return ret, nil
+		}
+
+		return defaultValue, errors.New("conversion to bool failed, using default value for the field")
+	}
+
+	return defaultValue, errors.New("field is empty, conversion to bool failed, using default value for the field")
+}
+
 // DurationToString formats the given duration into a short format which eludes trailing zero units in the string.
 func DurationToString(d time.Duration) string {
 	s := d.String()