From 680e782a072c4174ddd1f25837488a49d5c7803f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:08:02 -0800 Subject: [PATCH 01/54] Add safecli dependency --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index bb49e9f3df..0b81ec2854 100644 --- a/go.mod +++ b/go.mod @@ -216,6 +216,8 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) +require github.com/kanisterio/safecli v0.0.3 + require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect diff --git a/go.sum b/go.sum index 9793c1f653..fed0f3304c 100644 --- a/go.sum +++ b/go.sum @@ -359,6 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= +github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From 72011e75bc8478d1856b1e86a8f6bee896f55635 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:09:47 -0800 Subject: [PATCH 02/54] add new flag implementations based on the safecli package for the Kopia CLI --- pkg/kopia/cli/errors.go | 25 +++ pkg/kopia/cli/internal/flag/bool_flag.go | 45 ++++++ pkg/kopia/cli/internal/flag/flag.go | 81 ++++++++++ pkg/kopia/cli/internal/flag/flag_test.go | 171 +++++++++++++++++++++ pkg/kopia/cli/internal/flag/string_flag.go | 78 ++++++++++ pkg/kopia/cli/internal/test/flag_suite.go | 112 ++++++++++++++ 6 files changed, 512 insertions(+) create mode 100644 pkg/kopia/cli/errors.go create mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag_test.go create mode 100644 pkg/kopia/cli/internal/flag/string_flag.go create mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go new file mode 100644 index 0000000000..946dd04e87 --- /dev/null +++ b/pkg/kopia/cli/errors.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "github.com/pkg/errors" +) + +// flag errors +var ( + // ErrInvalidFlag is returned when the flag name is empty. + ErrInvalidFlag = errors.New("invalid flag") +) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go new file mode 100644 index 0000000000..4eea665cb2 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/bool_flag.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// boolFlag defines a boolean flag with a given flag name. +// If enabled is set to true, the flag is applied; otherwise, it is not. +type boolFlag struct { + flag string + enabled bool +} + +// Apply appends the flag to the command if the flag is enabled. +func (f boolFlag) Apply(cli safecli.CommandAppender) error { + if f.enabled { + cli.AppendLoggable(f.flag) + } + return nil +} + +// NewBoolFlag creates a new bool flag with a given flag name. +// If the flag name is empty, cli.ErrInvalidFlag is returned. +func NewBoolFlag(flag string, enabled bool) Applier { + if flag == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return boolFlag{flag, enabled} +} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go new file mode 100644 index 0000000000..898ae31e16 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" +) + +// Applier applies flags/args to the command. +type Applier interface { + // Apply applies the flags/args to the command. + Apply(cli safecli.CommandAppender) error +} + +// Apply appends multiple flags to the CLI. +// If any of the flags encounter an error during the Apply process, +// the error is returned and no changes are made to the CLI. +// If no error, the flags are appended to the CLI. +func Apply(cli safecli.CommandAppender, flags ...Applier) error { + // create a new sub builder which will be used to apply the flags + // to avoid mutating the CLI if an error is encountered. + sub := safecli.NewBuilder() + for _, flag := range flags { + if flag == nil { // if the flag is nil, skip it + continue + } + if err := flag.Apply(cli); err != nil { + return err + } + } + cli.Append(sub) + return nil +} + +// flags defines a collection of Flags. +type flags []Applier + +// Apply applies the flags to the CLI. +func (flags flags) Apply(cli safecli.CommandAppender) error { + return Apply(cli, flags...) +} + +// NewFlags creates a new collection of flags. +func NewFlags(fs ...Applier) Applier { + return flags(fs) +} + +// simpleFlag is a simple implementation of the Applier interface. +type simpleFlag struct { + err error +} + +// Apply does nothing except return an error if one is set. +func (f simpleFlag) Apply(safecli.CommandAppender) error { + return f.err +} + +// EmptyFlag creates a new flag that does nothing. +// It is useful for creating a no-op flag when a condition is not met +// but Applier interface is required. +func EmptyFlag() Applier { + return simpleFlag{} +} + +// ErrorFlag creates a new flag that returns an error when applied. +// It is useful for creating a flag validation if a condition is not met. +func ErrorFlag(err error) Applier { + return simpleFlag{err} +} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go new file mode 100644 index 0000000000..c20a46b450 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag_test.go @@ -0,0 +1,171 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_test + +import ( + "errors" + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +var ( + ErrFlag = errors.New("flag error") +) + +// MockFlagApplier is a mock implementation of the FlagApplier interface. +type MockFlagApplier struct { + flagName string + applyErr error +} + +func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { + cli.AppendLoggable(m.flagName) + return m.applyErr +} + +func TestApply(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ + { + Name: "Apply with no flags should generate only the command", + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with nil flags should generate only the command", + Flag: flag.NewFlags(nil, nil), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with flags should generate the command and flags", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "--flag1", applyErr: nil}, + &MockFlagApplier{flagName: "--flag2", applyErr: nil}, + ), + ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, + }, + { + Name: "Apply with one error flag should not modify the command and return the error", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "flag1", applyErr: nil}, + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "NewBoolFlag", + Flag: flag.NewFlags( + flag.NewBoolFlag("--flag1", true), + flag.NewBoolFlag("--flag2", false), + ), + ExpectedCLI: []string{"cmd", "--flag1"}, + }, + { + Name: "NewBoolFlag with empty flag name should return an error", + Flag: flag.NewFlags( + flag.NewBoolFlag("", true), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringFlag", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("--flag2", ""), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1"}, + }, + { + Name: "NewStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewRedactedStringFlag", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", ""), + flag.NewRedactedStringFlag("", "value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, + ExpectedLog: "cmd --flag1=<****> <****>", + }, + { + Name: "NewRedactedStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringValue", + Flag: flag.NewFlags( + flag.NewStringArgument("value1"), + ), + ExpectedCLI: []string{"cmd", "value1"}, + }, + { + Name: "NewStringValue with empty value should return an error", + Flag: flag.NewFlags( + flag.NewStringArgument(""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewFlags should generate multiple flags", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", "value2"), + flag.NewStringArgument("value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, + ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", + }, + { + Name: "NewFlags should generate no flags if one of them returns an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "EmptyFlag should not generate any flags", + Flag: flag.EmptyFlag(), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "ErrorFlag should return an error", + Flag: flag.ErrorFlag(ErrFlag), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, +}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go new file mode 100644 index 0000000000..a94c29722c --- /dev/null +++ b/pkg/kopia/cli/internal/flag/string_flag.go @@ -0,0 +1,78 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// stringFlag defines a string flag with a given flag name and value. +// If the value is empty, the flag is not applied. +type stringFlag struct { + flag string // flag name + value string // flag value + redacted bool // output the value as redacted +} + +// appenderFunc is a function that appends strings to a command. +type appenderFunc func(...string) *safecli.Builder + +// Apply appends the flag to the command if the value is not empty. +// If the value is redacted, it is appended as redacted. +func (f stringFlag) Apply(cli safecli.CommandAppender) error { + if f.value == "" { + return nil + } + appendValue, appendFlagValue := f.selectAppenderFuncs(cli) + if f.flag == "" { + appendValue(f.value) + } else { + appendFlagValue(f.flag, f.value) + } + return nil +} + +// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. +func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { + if f.redacted { + return cli.AppendRedacted, cli.AppendRedactedKV + } + return cli.AppendLoggable, cli.AppendLoggableKV +} + +// newStringFlag creates a new string flag with a given flag name and value. +func newStringFlag(flag, val string, redacted bool) Applier { + if flag == "" && val == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return stringFlag{flag: flag, value: val, redacted: redacted} +} + +// NewStringFlag creates a new string flag with a given flag name and value. +func NewStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, false) +} + +// NewRedactedStringFlag creates a new string flag with a given flag name and value. +func NewRedactedStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, true) +} + +// NewStringArgument creates a new string argument with a given value. +func NewStringArgument(val string) Applier { + return newStringFlag("", val, false) +} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go new file mode 100644 index 0000000000..7c41a1ea3a --- /dev/null +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -0,0 +1,112 @@ +package test + +import ( + "strings" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// FlagTest defines a single test for a flag. +type FlagTest struct { + // Name of the test. (required) + Name string + + // Flag to test. (required) + Flag flag.Applier + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be set to ExpectedCLI joined with space. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error +} + +// CheckCommentString implements check.CommentInterface +func (t *FlagTest) CheckCommentString() string { + return t.Name +} + +// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. +func (t *FlagTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") + } +} + +// assertError checks the error against ExpectedErr. +func (t *FlagTest) assertError(c *check.C, err error) { + if actualErr := errors.Cause(err); actualErr != nil { + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) + } else { + c.Assert(err, check.Equals, t.ExpectedErr, t) + } +} + +// assertNoError makes sure there is no error. +func (t *FlagTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +// assertCLI asserts the builder's CLI output against ExpectedCLI. +func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) +} + +// assertLog asserts the builder's log output against ExpectedLog. +func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { + t.setDefaultExpectedLog() + c.Check(b.String(), check.Equals, t.ExpectedLog, t) +} + +// Test runs the flag test. +func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { + err := flag.Apply(b, ft.Flag) + if ft.ExpectedErr != nil { + ft.assertError(c, err) + } else { + ft.assertNoError(c, err) + ft.assertCLI(c, b) + ft.assertLog(c, b) + } +} + +// FlagSuite defines a test suite for flags. +type FlagSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Tests []FlagTest // Tests to run. +} + +// TestFlags runs all tests in the flag suite. +func (s *FlagSuite) TestFlags(c *check.C) { + for _, test := range s.Tests { + b := newBuilder(s.Cmd) + test.Test(c, b) + } +} + +// NewFlagSuite creates a new FlagSuite. +func NewFlagSuite(tests []FlagTest) *FlagSuite { + return &FlagSuite{Tests: tests} +} + +// newBuilder creates a new safecli.Builder with the given command. +func newBuilder(cmd string) *safecli.Builder { + builder := safecli.NewBuilder() + if cmd != "" { + builder.AppendLoggable(cmd) + } + return builder +} From 0f635ad33638621c3e81849de07983c60192c73e Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 3 Feb 2024 16:03:14 -0800 Subject: [PATCH 03/54] apply go fmt Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 898ae31e16..66148034a1 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -68,7 +68,7 @@ func (f simpleFlag) Apply(safecli.CommandAppender) error { } // EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met +// It is useful for creating a no-op flag when a condition is not met // but Applier interface is required. func EmptyFlag() Applier { return simpleFlag{} From d0a6dd13afeb5f54cff4448c74131522680dda5c Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 5 Feb 2024 17:51:24 -0800 Subject: [PATCH 04/54] Add common Kopia args and flags --- pkg/kopia/cli/args.go | 34 +++ pkg/kopia/cli/errors.go | 6 + .../cli/internal/flag/common/common_flags.go | 205 ++++++++++++++++++ .../internal/flag/common/common_flags_test.go | 196 +++++++++++++++++ pkg/kopia/cli/internal/test/flag_suite.go | 25 +-- .../cli/internal/test/flag_suite_test.go | 146 +++++++++++++ pkg/kopia/cli/internal/test/redact.go | 47 ++++ pkg/kopia/cli/internal/test/redact_test.go | 51 +++++ 8 files changed, 695 insertions(+), 15 deletions(-) create mode 100644 pkg/kopia/cli/args.go create mode 100644 pkg/kopia/cli/internal/flag/common/common_flags.go create mode 100644 pkg/kopia/cli/internal/flag/common/common_flags_test.go create mode 100644 pkg/kopia/cli/internal/test/flag_suite_test.go create mode 100644 pkg/kopia/cli/internal/test/redact.go create mode 100644 pkg/kopia/cli/internal/test/redact_test.go diff --git a/pkg/kopia/cli/args.go b/pkg/kopia/cli/args.go new file mode 100644 index 0000000000..78b697b7a9 --- /dev/null +++ b/pkg/kopia/cli/args.go @@ -0,0 +1,34 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +// The common arguments for Kopia CLI. + +// CommonArgs provides the common arguments for Kopia CLI. +type CommonArgs struct { + ConfigFilePath string // the path to the config file. + LogDirectory string // the directory where logs are stored. + LogLevel string // the level of logging. Default is "error". + RepoPassword string // the password for the repository. +} + +// CacheArgs provides the cache arguments for Kopia CLI. +type CacheArgs struct { + CacheDirectory string // the directory where cache is stored. Default is "/tmp/kopia-cache". + ContentCacheSizeMB int // the size of the content cache in MB. + ContentCacheSizeLimitMB int // the maximum size of the content cache in MB. + MetadataCacheSizeMB int // the size of the metadata cache in MB. + MetadataCacheSizeLimitMB int // the maximum size of the metadata cache in MB. +} diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 946dd04e87..265c694d29 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -22,4 +22,10 @@ import ( var ( // ErrInvalidFlag is returned when the flag name is empty. ErrInvalidFlag = errors.New("invalid flag") + // ErrInvalidCommonArgs is returned when the common flag expects at most one cli.CommonArgs argument. + ErrInvalidCommonArgs = errors.New("common flag expects at most one cli.CommonArgs argument") + // ErrInvalidCacheArgs is returned when the cache flag expects at most one cli.CacheArgs argument. + ErrInvalidCacheArgs = errors.New("cache flag expects at most one cli.CacheArgs argument") + // ErrInvalidID is returned when the ID is empty. + ErrInvalidID = errors.New("invalid ID") ) diff --git a/pkg/kopia/cli/internal/flag/common/common_flags.go b/pkg/kopia/cli/internal/flag/common/common_flags.go new file mode 100644 index 0000000000..6df9b1bb5c --- /dev/null +++ b/pkg/kopia/cli/internal/flag/common/common_flags.go @@ -0,0 +1,205 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "strconv" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// Flags without conditions which applied to different kopia commands. +var ( + All = flag.NewBoolFlag("--all", true) + Delta = flag.NewBoolFlag("--delta", true) + ShowIdentical = flag.NewBoolFlag("--show-identical", true) + NoGRPC = flag.NewBoolFlag("--no-grpc", true) +) + +// predefined flags +var ( + CheckForUpdates = checkForUpdates{CheckForUpdates: true} + NoCheckForUpdates = checkForUpdates{CheckForUpdates: false} +) + +// flag defaults +var ( + defaultLogLevel = "error" + defaultCacheDirectory = "/tmp/kopia-cache" +) + +// LogDirectory creates a new log directory flag with a given directory. +func LogDirectory(dir string) flag.Applier { + return flag.NewStringFlag("--log-dir", dir) +} + +// LogLevel creates a new log level flag with a given level. +// If the level is empty, the default log level is used. +func LogLevel(level string) flag.Applier { + if level == "" { + level = defaultLogLevel + } + return flag.NewStringFlag("--log-level", level) +} + +// CacheDirectory creates a new cache directory flag with a given directory. +// If the directory is empty, the default cache directory is used. +func CacheDirectory(dir string) flag.Applier { + if dir == "" { + dir = defaultCacheDirectory + } + return flag.NewStringFlag("--cache-directory", dir) +} + +// ConfigFilePath creates a new config file path flag with a given path. +func ConfigFilePath(path string) flag.Applier { + return flag.NewStringFlag("--config-file", path) +} + +// RepoPassword creates a new repository password flag with a given password. +func RepoPassword(password string) flag.Applier { + return flag.NewRedactedStringFlag("--password", password) +} + +// checkForUpdates is the flag for checking for updates. +type checkForUpdates struct { + CheckForUpdates bool +} + +func (f checkForUpdates) Flag() string { + if f.CheckForUpdates { + return "--check-for-updates" + } + return "--no-check-for-updates" +} + +func (f checkForUpdates) Apply(cli safecli.CommandAppender) error { + cli.AppendLoggable(f.Flag()) + return nil +} + +// ReadOnly creates a new read only flag. +func ReadOnly(readOnly bool) flag.Applier { + return flag.NewBoolFlag("--readonly", readOnly) +} + +// ContentCacheSizeLimitMB creates a new content cache size flag with a given size. +func ContentCacheSizeLimitMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--content-cache-size-limit-mb", val) +} + +// ContentCacheSizeMB creates a new content cache size flag with a given size. +func ContentCacheSizeMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--content-cache-size-mb", val) +} + +// MetadataCacheSizeLimitMB creates a new metadata cache size flag with a given size. +func MetadataCacheSizeLimitMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--metadata-cache-size-limit-mb", val) +} + +// MetadataCacheSizeMB creates a new metadata cache size flag with a given size. +func MetadataCacheSizeMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--metadata-cache-size-mb", val) +} + +// common are the global flags for Kopia. +type common struct { + cli.CommonArgs +} + +// Apply applies the global flags to the command. +func (f common) Apply(cmd safecli.CommandAppender) error { + return flag.Apply(cmd, + ConfigFilePath(f.ConfigFilePath), + LogLevel(f.LogLevel), + LogDirectory(f.LogDirectory), + RepoPassword(f.RepoPassword), + ) +} + +// Common creates a new common flag. +// If no arguments are provided, the default common flags are used. +// If one argument is provided, the common flags are used. +// If more than one argument is provided, ErrInvalidCommonArgs is returned. +func Common(args ...cli.CommonArgs) flag.Applier { + switch len(args) { + case 0: + return common{cli.CommonArgs{}} + case 1: + return common{args[0]} + default: + return flag.ErrorFlag(cli.ErrInvalidCommonArgs) + } +} + +// cache defines cache flags and implements Applier interface for the cache flags. +type cache struct { + cli.CacheArgs +} + +// Apply applies the cache flags to the command. +func (f cache) Apply(cmd safecli.CommandAppender) error { + return flag.Apply(cmd, + CacheDirectory(f.CacheDirectory), + ContentCacheSizeLimitMB(f.ContentCacheSizeLimitMB), + MetadataCacheSizeLimitMB(f.MetadataCacheSizeLimitMB), + // ContentCacheSizeMB(f.ContentCacheSizeMB), + // MetadataCacheSizeMB(f.MetadataCacheSizeMB), + ) +} + +// Cache creates a new cache flag. +// If no arguments are provided, the default cache flags are used. +// If one argument is provided, the cache flags are used. +// If more than one argument is provided, ErrInvalidCacheArgs is returned. +func Cache(args ...cli.CacheArgs) flag.Applier { + switch len(args) { + case 0: + return cache{cli.CacheArgs{}} + case 1: + return cache{args[0]} + default: + return flag.ErrorFlag(cli.ErrInvalidCacheArgs) + } +} + +// JSONOutput creates a new JSON output flag. +func JSONOutput(enable bool) flag.Applier { + return flag.NewBoolFlag("--json", enable) +} + +// JSON flag enables JSON output for different kopia commands. +var JSON = JSONOutput(true) + +// Delete creates a new delete flag. +func Delete(enable bool) flag.Applier { + return flag.NewBoolFlag("--delete", enable) +} + +// ID create the Kopia ID argument for different commands. +func ID(id string) flag.Applier { + if id == "" { + return flag.ErrorFlag(cli.ErrInvalidID) + } + return flag.NewStringArgument(id) +} diff --git a/pkg/kopia/cli/internal/flag/common/common_flags_test.go b/pkg/kopia/cli/internal/flag/common/common_flags_test.go new file mode 100644 index 0000000000..2e8ac8cfc1 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/common/common_flags_test.go @@ -0,0 +1,196 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "fmt" + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestCommonFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty LogDirectory should generate a flag with default value", + Flag: LogDirectory(""), + }, + { + Name: "LogDirectory with value should generate a flag with the given directory", + Flag: LogDirectory("/path/to/logs"), + ExpectedCLI: []string{"--log-dir=/path/to/logs"}, + }, + { + Name: "Empty LogLevel should generate a flag with default value", + Flag: LogLevel(""), + ExpectedCLI: []string{fmt.Sprintf("--log-level=%s", defaultLogLevel)}, + }, + { + Name: "LogLevel with value should generate a flag with the given level", + Flag: LogLevel("info"), + ExpectedCLI: []string{"--log-level=info"}, + }, + { + Name: "Empty CacheDirectory should generate a flag with default value", + Flag: CacheDirectory(""), + ExpectedCLI: []string{fmt.Sprintf("--cache-directory=%s", defaultCacheDirectory)}, + }, + { + Name: "CacheDirectory with value should generate a flag with the given directory", + Flag: CacheDirectory("/home/user/.cache/kopia"), + ExpectedCLI: []string{"--cache-directory=/home/user/.cache/kopia"}, + }, + { + Name: "Empty ConfigFilePath should not generate a flag", + Flag: ConfigFilePath(""), + }, + { + Name: "ConfigFilePath with value should generate a flag with the given config file path", + Flag: ConfigFilePath("/var/kopia/config"), + ExpectedCLI: []string{"--config-file=/var/kopia/config"}, + }, + { + Name: "Empty RepoPassword should not generate a flag", + Flag: RepoPassword(""), + }, + { + Name: "RepoPassword with value should generate a flag with the given value and redact it for logs", + Flag: RepoPassword("pass12345"), + ExpectedCLI: []string{"--password=pass12345"}, + }, + { + Name: "CheckForUpdates should always generate a flag", + Flag: CheckForUpdates, + ExpectedCLI: []string{"--check-for-updates"}, + }, + { + Name: "NoCheckForUpdates should always generate a flag", + Flag: NoCheckForUpdates, + ExpectedCLI: []string{"--no-check-for-updates"}, + }, + { + Name: "ReadOnly(false)should not generate a flag", + Flag: ReadOnly(false), + }, + { + Name: "ReadOnly(true) should generate a flag", + Flag: ReadOnly(true), + ExpectedCLI: []string{"--readonly"}, + }, + { + Name: "NoGRPC should always generate '--no-grpc' flag", + Flag: NoGRPC, + ExpectedCLI: []string{"--no-grpc"}, + }, + { + Name: "JSON should always generate a flag", + Flag: JSON, + ExpectedCLI: []string{"--json"}, + }, + { + Name: "ContentCacheSizeLimitMB with value should generate a flag with the given value", + Flag: ContentCacheSizeLimitMB(1024), + ExpectedCLI: []string{"--content-cache-size-limit-mb=1024"}, + }, + { + Name: "ContentCacheSizeMB with value should generate a flag with the given value", + Flag: ContentCacheSizeMB(1024), + ExpectedCLI: []string{"--content-cache-size-mb=1024"}, + }, + { + Name: "MetadataCacheSizeLimitMB with value should generate a flag with the given value", + Flag: MetadataCacheSizeLimitMB(1024), + ExpectedCLI: []string{"--metadata-cache-size-limit-mb=1024"}, + }, + { + Name: "MetadataCacheSizeMB with value should generate a flag with the given value", + Flag: MetadataCacheSizeMB(1024), + ExpectedCLI: []string{"--metadata-cache-size-mb=1024"}, + }, + { + Name: "Empty Common should generate a flag with default value(s)", + Flag: Common(), + ExpectedCLI: []string{"--log-level=error"}, + }, + { + Name: "Common with more than one cli.CommonArgs should generate an error", + Flag: Common(cli.CommonArgs{}, cli.CommonArgs{}), + ExpectedErr: cli.ErrInvalidCommonArgs, + }, + { + Name: "Common with values should generate multiple flags with the given values and redact password for logs", + Flag: Common(cli.CommonArgs{ + ConfigFilePath: "/var/kopia/config", + LogDirectory: "/var/log/kopia", + LogLevel: "info", + RepoPassword: "pass12345", + }), + ExpectedCLI: []string{ + "--config-file=/var/kopia/config", + "--log-level=info", + "--log-dir=/var/log/kopia", + "--password=pass12345", + }, + }, + { + Name: "Empty FlagCacheArgs should generate multiple flags with default values", + Flag: Cache(), + ExpectedCLI: []string{ + "--cache-directory=/tmp/kopia-cache", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + }, + }, + { + Name: "Cache with more than one cli.CacheArgs should generate an error", + Flag: Cache(cli.CacheArgs{}, cli.CacheArgs{}), + ExpectedErr: cli.ErrInvalidCacheArgs, + }, + { + Name: "Cache with CacheArgs should generate multiple cache related flags", + Flag: Cache(cli.CacheArgs{ + CacheDirectory: "/home/user/.cache/kopia", + ContentCacheSizeLimitMB: 1024, + MetadataCacheSizeLimitMB: 2048, + }), + ExpectedCLI: []string{ + "--cache-directory=/home/user/.cache/kopia", + "--content-cache-size-limit-mb=1024", + "--metadata-cache-size-limit-mb=2048", + }, + }, + { + Name: "Delete(false) should not generate a flag", + Flag: Delete(false), + }, + { + Name: "Delete(true) should generate a flag", + Flag: Delete(true), + ExpectedCLI: []string{"--delete"}, + }, + { + Name: "Empty ID should generate an ErrInvalidID error", + Flag: ID(""), + ExpectedErr: cli.ErrInvalidID, + }, + { + Name: "ID with value should generate an argument with the given value", + Flag: ID("id12345"), + ExpectedCLI: []string{"id12345"}, + }, +})) diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index 7c41a1ea3a..c930573466 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -1,8 +1,6 @@ package test import ( - "strings" - "gopkg.in/check.v1" "github.com/pkg/errors" @@ -42,17 +40,14 @@ func (t *FlagTest) CheckCommentString() string { // setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. func (t *FlagTest) setDefaultExpectedLog() { if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") + t.ExpectedLog = RedactCLI(t.ExpectedCLI) } } // assertError checks the error against ExpectedErr. func (t *FlagTest) assertError(c *check.C, err error) { - if actualErr := errors.Cause(err); actualErr != nil { - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) - } else { - c.Assert(err, check.Equals, t.ExpectedErr, t) - } + actualErr := errors.Cause(err) + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) } // assertNoError makes sure there is no error. @@ -72,14 +67,14 @@ func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { } // Test runs the flag test. -func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { - err := flag.Apply(b, ft.Flag) - if ft.ExpectedErr != nil { - ft.assertError(c, err) +func (t *FlagTest) Test(c *check.C, b *safecli.Builder) { + err := flag.Apply(b, t.Flag) + if t.ExpectedErr != nil { + t.assertError(c, err) } else { - ft.assertNoError(c, err) - ft.assertCLI(c, b) - ft.assertLog(c, b) + t.assertNoError(c, err) + t.assertCLI(c, b) + t.assertLog(c, b) } } diff --git a/pkg/kopia/cli/internal/test/flag_suite_test.go b/pkg/kopia/cli/internal/test/flag_suite_test.go new file mode 100644 index 0000000000..307ff62533 --- /dev/null +++ b/pkg/kopia/cli/internal/test/flag_suite_test.go @@ -0,0 +1,146 @@ +package test_test + +import ( + "strings" + "testing" + + "github.com/pkg/errors" + + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/safecli" +) + +func TestCustomFlag(t *testing.T) { check.TestingT(t) } + +// CustomFlagTest is a test for FlagTest. +// it has a custom flag that can be used to test the flag. +// and implements flag.Applier. +type CustomFlagTest struct { + name string + flag string + flagErr error + expectedErr error +} + +func (t *CustomFlagTest) Apply(cli safecli.CommandAppender) error { + if t.flagErr == nil { + cli.AppendLoggable(t.flag) + } + return t.flagErr +} + +func (t *CustomFlagTest) Test(c *check.C) { + flagTest := test.FlagTest{ + Name: t.name, + Flag: t, + ExpectedErr: t.expectedErr, + } + if t.flag != "" { + flagTest.ExpectedCLI = []string{t.flag} + } + b := safecli.NewBuilder() + flagTest.Test(c, b) +} + +type CustomFlagSuite struct { + cmd string + tests []test.FlagTest +} + +func (s *CustomFlagSuite) Test(c *check.C) { + suite := test.NewFlagSuite(s.tests) + suite.Cmd = s.cmd + suite.TestFlags(c) +} + +// TestRunnerWithConfig is a test suite for CustomFlagTest. +type TestRunnerWithConfig struct { + out strings.Builder // output buffer for the test results + cfg *check.RunConf // custom test configuration +} + +// register the test suite +var _ = check.Suite(&TestRunnerWithConfig{}) + +// SetUpTest sets up the test suite for running. +// it initializes the output buffer and the test configuration. +func (s *TestRunnerWithConfig) SetUpTest(c *check.C) { + s.out = strings.Builder{} + s.cfg = &check.RunConf{ + Output: &s.out, + Verbose: true, + } +} + +// TestFlagTestOK tests the FlagTest with no errors. +func (s *TestRunnerWithConfig) TestFlagTestOK(c *check.C) { + cft := CustomFlagTest{ + name: "TestFlagOK", + flag: "--test", + } + res := check.Run(&cft, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagTest\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} + +// TestFlagTestErr tests the FlagTest with an error. +func (s *TestRunnerWithConfig) TestFlagTestErr(c *check.C) { + err := errors.New("test error") + cft := CustomFlagTest{ + name: "TestFlagErr", + flagErr: err, + expectedErr: err, + } + res := check.Run(&cft, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagTest\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} + +// TestFlagTestWrapperErr tests the FlagTest with a wrapped error. +func (s *TestRunnerWithConfig) TestFlagTestWrapperErr(c *check.C) { + err := errors.New("test error") + werr := errors.Wrap(err, "wrapper error") + cft := CustomFlagTest{ + name: "TestFlagTestWrapperErr", + flagErr: werr, + expectedErr: err, + } + res := check.Run(&cft, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagTest\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} + +// TestFlagTestUnexpectedErr tests the FlagTest with an unexpected error. +func (s *TestRunnerWithConfig) TestFlagTestUnexpectedErr(c *check.C) { + err := errors.New("test error") + cft := CustomFlagTest{ + name: "TestFlagUnexpectedErr", + flag: "--test", + flagErr: err, + expectedErr: nil, + } + res := check.Run(&cft, s.cfg) + ss := s.out.String() + c.Assert(strings.Contains(ss, "TestFlagUnexpectedErr"), check.Equals, true) + c.Assert(strings.Contains(ss, "test error"), check.Equals, true) + c.Assert(res.Passed(), check.Equals, false) +} + +// TestFlagSuiteOK tests the FlagSuite with no errors. +func (s *TestRunnerWithConfig) TestFlagSuiteOK(c *check.C) { + cfs := CustomFlagSuite{ + cmd: "cmd", + tests: []test.FlagTest{ + { + Name: "TestFlagOK", + Flag: &CustomFlagTest{name: "TestFlagOK", flag: "--test"}, + ExpectedCLI: []string{"cmd", "--test"}, + }, + }, + } + res := check.Run(&cfs, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagSuite\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} diff --git a/pkg/kopia/cli/internal/test/redact.go b/pkg/kopia/cli/internal/test/redact.go new file mode 100644 index 0000000000..2622877d13 --- /dev/null +++ b/pkg/kopia/cli/internal/test/redact.go @@ -0,0 +1,47 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "strings" +) + +const ( + redactField = "<****>" +) + +var redactedFlags = []string{ + "--password", + "--user-password", + "--server-password", + "--server-control-password", + "--server-cert-fingerprint", +} + +// RedactCLI redacts sensitive information from the CLI command for tests. +func RedactCLI(cli []string) string { + redactedCLI := make([]string, len(cli)) + for i, arg := range cli { + redactedCLI[i] = arg + for _, flag := range redactedFlags { + if strings.HasPrefix(arg, flag+"=") { + redactedCLI[i] = fmt.Sprintf("%s=%s", flag, redactField) + break // redacted flag found, no need to check further + } + } + } + return strings.Join(redactedCLI, " ") +} \ No newline at end of file diff --git a/pkg/kopia/cli/internal/test/redact_test.go b/pkg/kopia/cli/internal/test/redact_test.go new file mode 100644 index 0000000000..4bce54b009 --- /dev/null +++ b/pkg/kopia/cli/internal/test/redact_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "strings" + "testing" + + "gopkg.in/check.v1" +) + +func TestRedactCLI(t *testing.T) { check.TestingT(t) } + +type RedactSuite struct{} + +var _ = check.Suite(&RedactSuite{}) + +func (s *RedactSuite) TestRedactCLI(c *check.C) { + cli := []string{ + "--password=secret", + "--user-password=123456", + "--server-password=pass123", + "--server-control-password=abc123", + "--server-cert-fingerprint=abcd1234", + "--other-flag=value", + "argument", + } + expected := []string{ + "--password=<****>", + "--user-password=<****>", + "--server-password=<****>", + "--server-control-password=<****>", + "--server-cert-fingerprint=<****>", + "--other-flag=value", + "argument", + } + result := RedactCLI(cli) + c.Assert(result, check.Equals, strings.Join(expected, " ")) +} From ca853c1700d714f5615ad6bb7015d042b0bf315b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 5 Feb 2024 18:14:49 -0800 Subject: [PATCH 05/54] Add Kopia storage core flags --- pkg/kopia/cli/errors.go | 6 ++ .../internal/flag/storage/model/factory.go | 56 +++++++++++++ .../flag/storage/model/factory_test.go | 81 +++++++++++++++++++ .../internal/flag/storage/model/location.go | 61 ++++++++++++++ .../flag/storage/model/location_test.go | 81 +++++++++++++++++++ .../cli/internal/flag/storage/model/path.go | 28 +++++++ .../internal/flag/storage/model/path_test.go | 50 ++++++++++++ .../flag/storage/model/storage_flag.go | 61 ++++++++++++++ .../flag/storage/model/storage_flag_test.go | 69 ++++++++++++++++ pkg/kopia/cli/internal/log/log.go | 45 +++++++++++ 10 files changed, 538 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/location.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/location_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/path.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/path_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go create mode 100644 pkg/kopia/cli/internal/log/log.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 265c694d29..0f42f4c37d 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -29,3 +29,9 @@ var ( // ErrInvalidID is returned when the ID is empty. ErrInvalidID = errors.New("invalid ID") ) + +// storage errors +var ( + // ErrUnsupportedStorage is returned when the storage is not supported. + ErrUnsupportedStorage = errors.New("unsupported storage") +) diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory.go b/pkg/kopia/cli/internal/flag/storage/model/factory.go new file mode 100644 index 0000000000..8a77fbf7f6 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/factory.go @@ -0,0 +1,56 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// StorageBuilder defines a function that creates +// a safecli.Builder for the storage sub command. +type StorageBuilder func(StorageFlag) (*safecli.Builder, error) + +// StorageBuilderFactory defines a factory interface +// for creating a StorageBuilder by type. +type StorageBuilderFactory interface { + Create(rs.LocType) StorageBuilder +} + +// BuildersFactory defines a map of StorageBuilder by LocType. +type BuildersFactory map[rs.LocType]StorageBuilder + +// Create returns a StorageBuilder by LocType and +// implements the StorageBuilderFactory interface. +func (sb BuildersFactory) Create(locType rs.LocType) StorageBuilder { + if b, found := sb[locType]; found { + return b + } + return sb.unsupportedStorageType(locType) +} + +// unsupportedStorageType returns an error for an unsupported location type. +func (sb BuildersFactory) unsupportedStorageType(locType rs.LocType) StorageBuilder { + return func(StorageFlag) (*safecli.Builder, error) { + return nil, errors.Wrap(cli.ErrUnsupportedStorage, fmt.Sprintf("unsupported location type: '%v'", locType)) + } +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory_test.go b/pkg/kopia/cli/internal/flag/storage/model/factory_test.go new file mode 100644 index 0000000000..e0bee04d65 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/factory_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +func TestFactory(t *testing.T) { check.TestingT(t) } + +type FactorySuite struct{} + +var ( + _ = check.Suite(&FactorySuite{}) + ltBlue = rs.LocType("blue") + ltRed = rs.LocType("red") + ltUnknown = rs.LocType("unknown") +) + +type mockBuilder struct { + cmd string + err error +} + +func (m *mockBuilder) New() StorageBuilder { + return func(sf StorageFlag) (*safecli.Builder, error) { + if m.err != nil { + return nil, m.err + } + b := safecli.NewBuilder() + b.AppendLoggable(m.cmd) + return b, nil + } +} + +func mockFactory() StorageBuilderFactory { + factory := make(BuildersFactory) + + blue := mockBuilder{cmd: "blue"} + red := mockBuilder{cmd: "red"} + + factory[ltBlue] = blue.New() + factory[ltRed] = red.New() + return factory +} + +func (s *FactorySuite) TestBuildersFactory(c *check.C) { + factory := mockFactory() + + b, err := factory.Create(ltBlue)(StorageFlag{}) + c.Assert(err, check.IsNil) + c.Check(b, check.NotNil) + c.Check(b.Build(), check.DeepEquals, []string{"blue"}) + + b, err = factory.Create(ltRed)(StorageFlag{}) + c.Assert(err, check.IsNil) + c.Check(b, check.NotNil) + c.Check(b.Build(), check.DeepEquals, []string{"red"}) + + b, err = factory.Create(ltUnknown)(StorageFlag{}) + c.Assert(err, check.ErrorMatches, ".*unsupported location type: 'unknown'.*") + c.Check(b, check.IsNil) +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/location.go b/pkg/kopia/cli/internal/flag/storage/model/location.go new file mode 100644 index 0000000000..3a9a4e4fc9 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/location.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "strconv" + "strings" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +// Location is a map of key-value pairs that represent different storage properties. +type Location map[string][]byte + +// Type returns the location type. +func (l Location) Type() rs.LocType { + return rs.LocType(string(l[rs.TypeKey])) +} + +// Region returns the location region. +func (l Location) Region() string { + return string(l[rs.RegionKey]) +} + +// BucketName returns the location bucket name. +func (l Location) BucketName() string { + return string(l[rs.BucketKey]) +} + +// Endpoint returns the location endpoint. +func (l Location) Endpoint() string { + return string(l[rs.EndpointKey]) +} + +// Prefix returns the location prefix. +func (l Location) Prefix() string { + return string(l[rs.PrefixKey]) +} + +// IsInsecureEndpoint returns true if the location endpoint is insecure/http. +func (l Location) IsInsecureEndpoint() bool { + return strings.HasPrefix(l.Endpoint(), "http:") +} + +// HasSkipSSLVerify returns true if the location has skip SSL verification. +func (l Location) HasSkipSSLVerify() bool { + v, _ := strconv.ParseBool(string(l[rs.SkipSSLVerifyKey])) + return v +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/location_test.go b/pkg/kopia/cli/internal/flag/storage/model/location_test.go new file mode 100644 index 0000000000..1051bce642 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/location_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + "gopkg.in/check.v1" +) + +func TestLocation(t *testing.T) { check.TestingT(t) } + +type LocationSuite struct{} + +var _ = check.Suite(&LocationSuite{}) + +func (s *LocationSuite) TestLocation(c *check.C) { + type expected struct { + Type rs.LocType + Region string + BucketName string + Endpoint string + Prefix string + IsInsecure bool + HasSkipSSLVerify bool + } + + tests := []struct { + name string + location Location + expected expected + }{ + { + name: "Test with no fields", + location: Location{}, + expected: expected{}, + }, + { + name: "Test with all fields", + location: Location{ + rs.TypeKey: []byte("Type1"), + rs.RegionKey: []byte("Region1"), + rs.BucketKey: []byte("Bucket1"), + rs.EndpointKey: []byte("http://Endpoint1"), + rs.PrefixKey: []byte("Prefix1"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + expected: expected{ + Type: "Type1", + Region: "Region1", + BucketName: "Bucket1", + Endpoint: "http://Endpoint1", + Prefix: "Prefix1", + IsInsecure: true, + HasSkipSSLVerify: true, + }, + }, + } + for _, test := range tests { + c.Check(test.location.Type(), check.Equals, test.expected.Type) + c.Check(test.location.Region(), check.Equals, test.expected.Region) + c.Check(test.location.BucketName(), check.Equals, test.expected.BucketName) + c.Check(test.location.Endpoint(), check.Equals, test.expected.Endpoint) + c.Check(test.location.Prefix(), check.Equals, test.expected.Prefix) + c.Check(test.location.IsInsecureEndpoint(), check.Equals, test.expected.IsInsecure) + c.Check(test.location.HasSkipSSLVerify(), check.Equals, test.expected.HasSkipSSLVerify) + } +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/path.go b/pkg/kopia/cli/internal/flag/storage/model/path.go new file mode 100644 index 0000000000..5e18da2cb8 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/path.go @@ -0,0 +1,28 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "path" +) + +// GenerateFullRepoPath generates the full repository path. +// If the location-specific prefix is empty, the repository-specific prefix is returned. +func GenerateFullRepoPath(locPrefix, repoPathPrefix string) string { + if locPrefix != "" { + return path.Join(locPrefix, repoPathPrefix) + "/" + } + return repoPathPrefix +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/path_test.go b/pkg/kopia/cli/internal/flag/storage/model/path_test.go new file mode 100644 index 0000000000..2fd1762362 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/path_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "gopkg.in/check.v1" +) + +func TestPath(t *testing.T) { check.TestingT(t) } + +type PathSuite struct{} + +var _ = check.Suite(&PathSuite{}) + +func (s *PathSuite) TestGenerateFullRepoPath(c *check.C) { + tests := []struct { + locPrefix string + repoPathPrefix string + expected string + }{ + { + locPrefix: "", + repoPathPrefix: "repo", + expected: "repo", + }, + { + locPrefix: "loc", + repoPathPrefix: "repo", + expected: "loc/repo/", + }, + } + for _, test := range tests { + got := GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) + c.Check(got, check.Equals, test.expected) + } +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go new file mode 100644 index 0000000000..89ac32cb6f --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + log "github.com/kanisterio/kanister/pkg/log" +) + +var ( + // ErrInvalidFactory is returned when the factory is nil. + ErrInvalidFactory = errors.New("factory cannot be nil") +) + +// StorageFlag is a set of flags that are used to create a StorageFlag sub command. +type StorageFlag struct { + Location Location + RepoPathPrefix string + + Factory StorageBuilderFactory + Logger log.Logger +} + +// GetLogger returns the logger. +// If the logger is nil, it returns a NopLogger. +func (s StorageFlag) GetLogger() log.Logger { + if s.Logger == nil { + s.Logger = &cmdlog.NopLogger{} + } + return s.Logger +} + +// Apply applies the storage flags to the command. +func (s StorageFlag) Apply(cli safecli.CommandAppender) error { + if s.Factory == nil { + return ErrInvalidFactory + } + storageBuilder := s.Factory.Create(s.Location.Type()) + storageCLI, err := storageBuilder(s) + if err != nil { + return errors.Wrap(err, "failed to apply storage args") + } + cli.Append(storageCLI) + return nil +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go new file mode 100644 index 0000000000..dd211eef4a --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go @@ -0,0 +1,69 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +func TestStorageFlag(t *testing.T) { check.TestingT(t) } + +type StorageFlagSuite struct{} + +var _ = check.Suite(&StorageFlagSuite{}) + +func (s *StorageFlagSuite) TestGetLogger(c *check.C) { + sf := StorageFlag{} + c.Check(sf.GetLogger(), check.NotNil) + sf.Logger = nil + c.Check(sf.GetLogger(), check.NotNil) +} + +func (s *StorageFlagSuite) TestApplyNoFactory(c *check.C) { + sf := StorageFlag{} + err := sf.Apply(nil) + c.Check(err, check.Equals, ErrInvalidFactory) +} + +func (s *StorageFlagSuite) TestApply(c *check.C) { + sf := StorageFlag{ + Location: Location{ + rs.TypeKey: []byte("blue"), + }, + Factory: mockFactory(), + } + b := safecli.NewBuilder() + err := sf.Apply(b) + c.Check(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{"blue"}) +} + +func (s *StorageFlagSuite) TestApplyUnknowType(c *check.C) { + sf := StorageFlag{ + Location: Location{ + rs.TypeKey: []byte("unknow"), + }, + Factory: mockFactory(), + } + b := safecli.NewBuilder() + err := sf.Apply(b) + c.Check(err, check.ErrorMatches, ".*failed to apply storage args.*") +} diff --git a/pkg/kopia/cli/internal/log/log.go b/pkg/kopia/cli/internal/log/log.go new file mode 100644 index 0000000000..4d2b29ffe3 --- /dev/null +++ b/pkg/kopia/cli/internal/log/log.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "io" + + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/log" +) + +// NopLogger is a logger that does nothing. +// TODO: Move to log package +type NopLogger struct{} + +// Print does nothing. +func (NopLogger) Print(msg string, fields ...field.M) { +} + +// PrintTo does nothing. +func (NopLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { +} + +// WithContext does nothing. +func (NopLogger) WithContext(ctx context.Context) log.Logger { + return &NopLogger{} +} + +// WithError does nothing. +func (NopLogger) WithError(err error) log.Logger { + return &NopLogger{} +} From 2d44961f212e55a2c25efb5b1f08ddd510576e5c Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 14:51:59 -0800 Subject: [PATCH 06/54] Add kopia filesystem storage flags --- pkg/kopia/cli/errors.go | 2 + pkg/kopia/cli/internal/command/command.go | 45 ++++ .../cli/internal/command/command_test.go | 97 ++++++++ pkg/kopia/cli/internal/command/commands.go | 6 + pkg/kopia/cli/internal/flag/storage/fs/fs.go | 39 +++ .../cli/internal/flag/storage/fs/fs_flags.go | 26 ++ .../internal/flag/storage/fs/fs_flags_test.go | 36 +++ .../cli/internal/flag/storage/fs/fs_test.go | 58 +++++ .../cli/internal/flag/storage/storage.go | 70 ++++++ .../cli/internal/flag/storage/storage_test.go | 231 ++++++++++++++++++ pkg/kopia/cli/internal/test/command_suite.go | 134 ++++++++++ pkg/kopia/cli/internal/test/string_logger.go | 58 +++++ 12 files changed, 802 insertions(+) create mode 100644 pkg/kopia/cli/internal/command/command.go create mode 100644 pkg/kopia/cli/internal/command/command_test.go create mode 100644 pkg/kopia/cli/internal/command/commands.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/storage.go create mode 100644 pkg/kopia/cli/internal/flag/storage/storage_test.go create mode 100644 pkg/kopia/cli/internal/test/command_suite.go create mode 100644 pkg/kopia/cli/internal/test/string_logger.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 0f42f4c37d..c4cc8393f1 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -28,6 +28,8 @@ var ( ErrInvalidCacheArgs = errors.New("cache flag expects at most one cli.CacheArgs argument") // ErrInvalidID is returned when the ID is empty. ErrInvalidID = errors.New("invalid ID") + // ErrInvalidCommand is returned when the command is empty. + ErrInvalidCommand = errors.New("invalid command") ) // storage errors diff --git a/pkg/kopia/cli/internal/command/command.go b/pkg/kopia/cli/internal/command/command.go new file mode 100644 index 0000000000..04acbaecd4 --- /dev/null +++ b/pkg/kopia/cli/internal/command/command.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/kanisterio/safecli" + + clierrors "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// Command is a CLI command/subcommand. +type Command struct { + name string +} + +// Apply applies the command to the CLI. +func (c Command) Apply(cli safecli.CommandAppender) error { + if len(c.name) == 0 { + return clierrors.ErrInvalidCommand + } + cli.AppendLoggable(c.name) + return nil +} + +// NewCommandBuilder returns a new safecli.Builder for the storage sub command. +func NewCommandBuilder(cmd flag.Applier, flags ...flag.Applier) (*safecli.Builder, error) { + b := safecli.NewBuilder() + if err := flag.Apply(b, append([]flag.Applier{cmd}, flags...)...); err != nil { + return nil, err + } + return b, nil +} diff --git a/pkg/kopia/cli/internal/command/command_test.go b/pkg/kopia/cli/internal/command/command_test.go new file mode 100644 index 0000000000..f4ae074d37 --- /dev/null +++ b/pkg/kopia/cli/internal/command/command_test.go @@ -0,0 +1,97 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "errors" + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli" +) + +func TestCommand(t *testing.T) { check.TestingT(t) } + +type CommandSuite struct{} + +var _ = check.Suite(&CommandSuite{}) + +var ( + errInvalidCommand = errors.New("invalid command") + errInvalidFlag = errors.New("invalid flag") +) + +type mockCommandAndFlag struct { + flagName string + err error +} + +func (m *mockCommandAndFlag) Apply(cli safecli.CommandAppender) error { + if m.err == nil { + cli.AppendLoggable(m.flagName) + } + return m.err +} + +func (s *CommandSuite) TestCommand(c *check.C) { + b := safecli.NewBuilder() + cmd := Command{"cmd"} + err := cmd.Apply(b) + c.Assert(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{"cmd"}) +} + +func (s *CommandSuite) TestEmptyCommand(c *check.C) { + b := safecli.NewBuilder() + cmd := Command{} + err := cmd.Apply(b) + c.Assert(err, check.Equals, cli.ErrInvalidCommand) +} + +func (s *CommandSuite) TestNewCommandBuilderWithFailedCommand(c *check.C) { + // test if command is invalid + b, err := NewCommandBuilder( + &mockCommandAndFlag{err: errInvalidCommand}, + ) + c.Assert(b, check.IsNil) + c.Assert(err, check.Equals, errInvalidCommand) +} + +func (s *CommandSuite) TestNewCommandBuilderWithFailedFlag(c *check.C) { + // test if flag is invalid + b, err := NewCommandBuilder( + &mockCommandAndFlag{flagName: "cmd"}, + &mockCommandAndFlag{err: errInvalidFlag}, + ) + c.Assert(b, check.IsNil) + c.Assert(err, check.Equals, errInvalidFlag) +} + +func (s *CommandSuite) TestNewCommandBuilder(c *check.C) { + // test if command and flag are valid + b, err := NewCommandBuilder( + &mockCommandAndFlag{flagName: "cmd"}, + &mockCommandAndFlag{flagName: "--flag1"}, + &mockCommandAndFlag{flagName: "--flag2"}, + ) + c.Assert(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{ + "cmd", + "--flag1", + "--flag2", + }) +} diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go new file mode 100644 index 0000000000..17fa517524 --- /dev/null +++ b/pkg/kopia/cli/internal/command/commands.go @@ -0,0 +1,6 @@ +package command + +// Repository storage sub commands. +var ( + FileSystem = Command{"filesystem"} +) diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs.go b/pkg/kopia/cli/internal/flag/storage/fs/fs.go new file mode 100644 index 0000000000..20a8b0fac7 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs.go @@ -0,0 +1,39 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +const ( + // DefaultFSMountPath is the default mount path for the filesystem subcommand storage. + DefaultFSMountPath = "/mnt/data" +) + +// New returns a builder for the filesystem subcommand storage. +func New(f model.StorageFlag) (*safecli.Builder, error) { + path := generateFileSystemMountPath(f.Location.Prefix(), f.RepoPathPrefix) + return command.NewCommandBuilder(command.FileSystem, + Path(path), + ) +} + +func generateFileSystemMountPath(locPrefix, repoPathPrefix string) string { + return DefaultFSMountPath + "/" + model.GenerateFullRepoPath(locPrefix, repoPathPrefix) +} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go new file mode 100644 index 0000000000..67c252e1e4 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go @@ -0,0 +1,26 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + +// +// Filestore flags. +// + +// Path creates a new path flag with a given path. +func Path(path string) flag.Applier { + return flag.NewStringFlag("--path", path) +} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go new file mode 100644 index 0000000000..c0631f0a75 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestFilestoreFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Path should not generate a flag", + Flag: Path(""), + }, + { + Name: "Path with value should generate a flag with the given value", + Flag: Path("/path/to/file"), + ExpectedCLI: []string{"--path=/path/to/file"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go new file mode 100644 index 0000000000..13d992abdb --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageFS(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty FS storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{ + "filesystem", + "--path=/mnt/data/", + }, + }, + { + Name: "FS with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + }, + }) + }, + ExpectedCLI: []string{ + "filesystem", + "--path=/mnt/data/prefix/repo/path/prefix/", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go new file mode 100644 index 0000000000..48431e6a97 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -0,0 +1,70 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "sync" + + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// Option is a function that sets a storage option. +type Option func(*model.StorageFlag) + +// WithLogger sets the logger for the storage. +func WithLogger(logger log.Logger) Option { + return func(s *model.StorageFlag) { + s.Logger = logger + } +} + +// WithFactory sets the storage args builder factory for the storage. +func WithFactory(factory model.StorageBuilderFactory) Option { + return func(s *model.StorageFlag) { + s.Factory = factory + } +} + +var ( + // factoryOnce is used to initialize the factory once. + factoryOnce sync.Once + // factory creates a new StorageBuilder by LocType. + factory = model.BuildersFactory{} +) + +// Storage creates a new storage with the given location, repo path prefix and options. +func Storage(location model.Location, repoPathPrefix string, opts ...Option) model.StorageFlag { + factoryOnce.Do(func() { + // Register storage builders. + factory[rs.LocTypeFilestore] = fs.New + }) + // create a new storage with the given location, repo path prefix and defaults. + s := model.StorageFlag{ + Location: location, + RepoPathPrefix: repoPathPrefix, + Logger: &cmdlog.NopLogger{}, + Factory: &factory, + } + // apply storage options. + for _, opt := range opts { + opt(&s) + } + return s +} diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go new file mode 100644 index 0000000000..3341156ac4 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -0,0 +1,231 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "testing" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" +) + +func TestStorageFlags(t *testing.T) { check.TestingT(t) } + +type StorageSuite struct{} + +var _ = check.Suite(&StorageSuite{}) + +func (s *StorageSuite) TestLocationMethods(c *check.C) { + type expected struct { + Type rs.LocType + Region string + BucketName string + Endpoint string + Prefix string + IsInsecure bool + HasSkipSSLVerify bool + } + + tests := []struct { + name string + location model.Location + expected expected + }{ + { + name: "Test1", + location: model.Location{ + rs.TypeKey: []byte("Type1"), + rs.RegionKey: []byte("Region1"), + rs.BucketKey: []byte("Bucket1"), + rs.EndpointKey: []byte("http://Endpoint1"), + rs.PrefixKey: []byte("Prefix1"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + expected: expected{ + Type: "Type1", + Region: "Region1", + BucketName: "Bucket1", + Endpoint: "http://Endpoint1", + Prefix: "Prefix1", + IsInsecure: true, + HasSkipSSLVerify: true, + }, + }, + { + name: "Test2", + location: model.Location{ + rs.TypeKey: []byte("Type2"), + rs.RegionKey: []byte("Region2"), + rs.BucketKey: []byte("Bucket2"), + rs.EndpointKey: []byte("https://Endpoint2"), + rs.PrefixKey: []byte("Prefix2"), + rs.SkipSSLVerifyKey: []byte("false"), + }, + expected: expected{ + Type: "Type2", + Region: "Region2", + BucketName: "Bucket2", + Endpoint: "https://Endpoint2", + Prefix: "Prefix2", + IsInsecure: false, + HasSkipSSLVerify: false, + }, + }, + } + + for _, tt := range tests { + c.Assert(tt.location.Type(), check.Equals, tt.expected.Type) + c.Assert(tt.location.Region(), check.Equals, tt.expected.Region) + c.Assert(tt.location.BucketName(), check.Equals, tt.expected.BucketName) + c.Assert(tt.location.Endpoint(), check.Equals, tt.expected.Endpoint) + c.Assert(tt.location.Prefix(), check.Equals, tt.expected.Prefix) + c.Assert(tt.location.IsInsecureEndpoint(), check.Equals, tt.expected.IsInsecure) + c.Assert(tt.location.HasSkipSSLVerify(), check.Equals, tt.expected.HasSkipSSLVerify) + } +} + +func (s *StorageSuite) TestStorageFlag(c *check.C) { + tests := []struct { + name string + storage flag.Applier + expCLI []string + err error + errMsg string + }{ + { + name: "Empty Storage should generate an error", + storage: Storage(nil, ""), + err: cli.ErrUnsupportedStorage, + }, + { + name: "Filesystem without prefix and with repo path should generate repo path", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte(""), + }, + "dir1/subdir/", + ), + expCLI: []string{ + "filesystem", + fmt.Sprintf("--path=%s/dir1/subdir/", fs.DefaultFSMountPath), + }, + }, + { + name: "Filesystem with prefix and repo path should generate merged prefix and repo path", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte("test-prefix"), + }, + "dir1/subdir/", + ), + expCLI: []string{ + "filesystem", + fmt.Sprintf("--path=%s/test-prefix/dir1/subdir/", fs.DefaultFSMountPath), + }, + }, + { + name: "Unsupported storage type should generate an error", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("ftp"), + }, + "prefixfs", + ), + errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", + err: cli.ErrUnsupportedStorage, + }, + } + + for _, tt := range tests { + b := safecli.NewBuilder() + err := tt.storage.Apply(b) + + cmt := check.Commentf("FAIL: %v", tt.name) + if tt.errMsg != "" { + c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) + } + + if tt.err == nil { + c.Assert(err, check.IsNil, cmt) + } else { + if errors.Cause(err) != nil { + c.Assert(errors.Cause(err), check.DeepEquals, tt.err, cmt) + } else { + c.Assert(err, check.Equals, tt.err, cmt) + } + } + c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) + } +} + +type MockFlagWithError struct{} + +func (f MockFlagWithError) Flag() string { + return "mock" +} + +var errMock = fmt.Errorf("mock error") + +func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { + return errMock +} + +func (s *StorageSuite) TestNewStorageBuilderWithErrorFlag(c *check.C) { + b, err := command.NewCommandBuilder(command.FileSystem, MockFlagWithError{}) + c.Assert(b, check.IsNil) + c.Assert(err, check.NotNil) + c.Assert(err, check.DeepEquals, errMock) +} + +func (s *StorageSuite) TestStorageGetLogger(c *check.C) { + storage := Storage(nil, "prefix") + c.Assert(storage.GetLogger(), check.NotNil) + + nopLog := &cmdlog.NopLogger{} + storage = Storage(nil, "prefix", WithLogger(nopLog)) + c.Assert(storage.GetLogger(), check.Equals, nopLog) +} + +type MockFactory struct{} + +func (f MockFactory) Create(locType rs.LocType) model.StorageBuilder { + return func(s model.StorageFlag) (*safecli.Builder, error) { + return safecli.NewBuilder("mock"), nil + } +} + +func (s *StorageSuite) TestStorageFactory(c *check.C) { + storage := Storage(nil, "prefix") + c.Assert(storage.GetLogger(), check.NotNil) + + mockFactory := &MockFactory{} + storage = Storage(nil, "prefix", WithFactory(mockFactory)) + c.Assert(storage.Factory, check.Equals, mockFactory) +} diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go new file mode 100644 index 0000000000..480c1fcba5 --- /dev/null +++ b/pkg/kopia/cli/internal/test/command_suite.go @@ -0,0 +1,134 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// CommonArgs is a set of common arguments for the tests. +var CommonArgs = cli.CommonArgs{ + RepoPassword: "encr-key", + ConfigFilePath: "path/kopia.config", + LogDirectory: "cache/log", +} + +// CommandTest defines a single test for a command. +type CommandTest struct { + // Name of the test. (required) + Name string + + // CLI to test. (required) + CLI func() (safecli.CommandBuilder, error) + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be derived from ExpectedCLI by redacting sensitive information. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error + + // LoggerRegex is a list of regular expressions to match against the log output. (optional) + Logger log.Logger + LoggerRegex []string +} + +// CheckCommentString implements check.CommentInterface +func (t *CommandTest) CheckCommentString() string { + return t.Name +} + +func (t *CommandTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = RedactCLI(t.ExpectedCLI) + } +} + +func (t *CommandTest) assertError(c *check.C, err error) { + actualErr := errors.Cause(err) + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) +} + +func (t *CommandTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +func (t *CommandTest) assertCLI(c *check.C, b safecli.CommandBuilder) { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) +} + +func (t *CommandTest) assertLog(c *check.C, b safecli.CommandBuilder) { + t.setDefaultExpectedLog() + c.Check(fmt.Sprint(b), check.Equals, t.ExpectedLog, t) +} + +func (t *CommandTest) assertLogger(c *check.C) { + log, ok := t.Logger.(*StringLogger) + if !ok { + c.Fatalf("t.Logger is not a StringLogger") + } + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.Name, log, t.LoggerRegex) + for _, regex := range t.LoggerRegex { + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// Test runs the command test. +func (t *CommandTest) Test(c *check.C) { + b, err := t.CLI() + if t.ExpectedErr == nil { + t.assertNoError(c, err) + t.assertCLI(c, b) + t.assertLog(c, b) + } else { + t.assertError(c, err) + } + if t.Logger != nil { + t.assertLogger(c) + } +} + +// CommandSuite defines a test suite for commands. +type CommandSuite struct { + Tests []CommandTest +} + +// TestCommands runs all tests in the suite. +func (s *CommandSuite) TestCommands(c *check.C) { + for _, test := range s.Tests { + test.Test(c) + } +} + +// NewCommandSuite creates a new CommandSuite. +func NewCommandSuite(tests []CommandTest) *CommandSuite { + return &CommandSuite{Tests: tests} +} diff --git a/pkg/kopia/cli/internal/test/string_logger.go b/pkg/kopia/cli/internal/test/string_logger.go new file mode 100644 index 0000000000..1a55f46220 --- /dev/null +++ b/pkg/kopia/cli/internal/test/string_logger.go @@ -0,0 +1,58 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "io" + "regexp" + + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/log" +) + +// StringLogger implements log.Logger and stores log messages in a slice of strings. +// It is useful for testing. +type StringLogger []string + +// Print appends the message to the slice. +func (l *StringLogger) Print(msg string, fields ...field.M) { + *l = append(*l, msg) +} + +// PrintTo appends the message to the slice. +func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { + *l = append(*l, msg) +} + +// WithContext does nothing. +func (l *StringLogger) WithContext(ctx context.Context) log.Logger { + return l +} + +// WithError does nothing. +func (l *StringLogger) WithError(err error) log.Logger { + return l +} + +// MatchString returns true if any of the log messages match the pattern. +func (l *StringLogger) MatchString(pattern string) bool { + for _, line := range *l { + if found, _ := regexp.MatchString(pattern, line); found { + return true + } + } + return false +} From 8f363ed89950664f41a4739860e132aeb1cc85b0 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:05:59 -0800 Subject: [PATCH 07/54] cleanup storage tests --- .../cli/internal/flag/storage/storage_test.go | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 3341156ac4..96fe150d5a 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -169,28 +169,22 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { cmt := check.Commentf("FAIL: %v", tt.name) if tt.errMsg != "" { + c.Assert(err, check.NotNil, cmt) c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) } if tt.err == nil { c.Assert(err, check.IsNil, cmt) } else { - if errors.Cause(err) != nil { - c.Assert(errors.Cause(err), check.DeepEquals, tt.err, cmt) - } else { - c.Assert(err, check.Equals, tt.err, cmt) - } + c.Assert(errors.Cause(err), check.Equals, tt.err, cmt) } c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) } } +// MockFlagWithError is a mock flag that always returns an error type MockFlagWithError struct{} -func (f MockFlagWithError) Flag() string { - return "mock" -} - var errMock = fmt.Errorf("mock error") func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { @@ -200,8 +194,7 @@ func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { func (s *StorageSuite) TestNewStorageBuilderWithErrorFlag(c *check.C) { b, err := command.NewCommandBuilder(command.FileSystem, MockFlagWithError{}) c.Assert(b, check.IsNil) - c.Assert(err, check.NotNil) - c.Assert(err, check.DeepEquals, errMock) + c.Assert(err, check.Equals, errMock) } func (s *StorageSuite) TestStorageGetLogger(c *check.C) { @@ -213,11 +206,12 @@ func (s *StorageSuite) TestStorageGetLogger(c *check.C) { c.Assert(storage.GetLogger(), check.Equals, nopLog) } +// MockFactory is a mock storage factory type MockFactory struct{} func (f MockFactory) Create(locType rs.LocType) model.StorageBuilder { return func(s model.StorageFlag) (*safecli.Builder, error) { - return safecli.NewBuilder("mock"), nil + return safecli.NewBuilder("mockfactory"), nil } } @@ -228,4 +222,8 @@ func (s *StorageSuite) TestStorageFactory(c *check.C) { mockFactory := &MockFactory{} storage = Storage(nil, "prefix", WithFactory(mockFactory)) c.Assert(storage.Factory, check.Equals, mockFactory) + b, err := storage.Factory.Create("anything")(model.StorageFlag{}) + c.Assert(b, check.NotNil) + c.Assert(err, check.IsNil) + c.Assert(b.Build(), check.DeepEquals, []string{"mockfactory"}) } From 0eb826a093e7dd492822118d5d6a272e99eb537b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:15:21 -0800 Subject: [PATCH 08/54] Add kopia GCS storage flags --- pkg/kopia/cli/internal/command/commands.go | 1 + .../cli/internal/flag/storage/gcs/gcs.go | 34 +++++++++++ .../internal/flag/storage/gcs/gcs_flags.go | 36 +++++++++++ .../flag/storage/gcs/gcs_flags_test.go | 36 +++++++++++ .../cli/internal/flag/storage/gcs/gcs_test.go | 61 +++++++++++++++++++ .../cli/internal/flag/storage/storage.go | 5 +- .../cli/internal/flag/storage/storage_test.go | 17 ++++++ 7 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs.go create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index 17fa517524..e0136a4c71 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -3,4 +3,5 @@ package command // Repository storage sub commands. var ( FileSystem = Command{"filesystem"} + GCS = Command{"gcs"} ) diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs.go new file mode 100644 index 0000000000..076c9beda7 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs.go @@ -0,0 +1,34 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/consts" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// New returns a builder for the GCS subcommand storage. +func New(s model.StorageFlag) (*safecli.Builder, error) { + prefix := model.GenerateFullRepoPath(s.Location.Prefix(), s.RepoPathPrefix) + return command.NewCommandBuilder(command.GCS, + Bucket(s.Location.BucketName()), + CredentialsFile(consts.GoogleCloudCredsFilePath), + Prefix(prefix), + ) +} diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go new file mode 100644 index 0000000000..8800b8ce0a --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + +// +// GCS flags. +// + +// Bucket creates a new GCS bucket flag with a given bucket name. +func Bucket(bucket string) flag.Applier { + return flag.NewStringFlag("--bucket", bucket) +} + +// Prefix creates a new GCS prefix flag with a given prefix. +func Prefix(prefix string) flag.Applier { + return flag.NewStringFlag("--prefix", prefix) +} + +// CredentialsFile creates a new GCS credentials file flag with a given file path. +func CredentialsFile(filePath string) flag.Applier { + return flag.NewStringFlag("--credentials-file", filePath) +} diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go new file mode 100644 index 0000000000..4a6fadf6c6 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestStorageGCSFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty CredentialsFile should not generate a flag", + Flag: CredentialsFile(""), + }, + { + Name: "CredentialsFile with value should generate a flag with the given value", + Flag: CredentialsFile("/path/to/credentials"), + ExpectedCLI: []string{"--credentials-file=/path/to/credentials"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go new file mode 100644 index 0000000000..c2f2fe9476 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageGCS(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty GCS storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{ + "gcs", + "--credentials-file=/tmp/creds.txt", + }, + }, + { + Name: "GCS with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + rs.BucketKey: []byte("bucket"), + }, + }) + }, + ExpectedCLI: []string{ + "gcs", + "--bucket=bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=prefix/repo/path/prefix/", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go index 48431e6a97..2b0b564ac6 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -17,11 +17,13 @@ package storage import ( "sync" - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" "github.com/kanisterio/kanister/pkg/log" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/gcs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" ) @@ -54,6 +56,7 @@ func Storage(location model.Location, repoPathPrefix string, opts ...Option) mod factoryOnce.Do(func() { // Register storage builders. factory[rs.LocTypeFilestore] = fs.New + factory[rs.LocTypeGCS] = gcs.New }) // create a new storage with the given location, repo path prefix and defaults. s := model.StorageFlag{ diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 96fe150d5a..2164afc7ed 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -161,6 +161,23 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", err: cli.ErrUnsupportedStorage, }, + { + name: "GCS should generate gcs args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("gcs"), + rs.BucketKey: []byte("bucket"), + rs.PrefixKey: []byte("/path/to/prefix"), + }, + "prefixfs", + ), + expCLI: []string{ + "gcs", + "--bucket=bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, } for _, tt := range tests { From 13266243d33561e74fd9f90b61e167503f18d13a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:21:57 -0800 Subject: [PATCH 09/54] add gcs flag tests --- .../flag/storage/gcs/gcs_flags_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go index 4a6fadf6c6..45eea13c0f 100644 --- a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go @@ -24,6 +24,24 @@ import ( func TestStorageGCSFlags(t *testing.T) { check.TestingT(t) } var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Bucket should not generate a flag", + Flag: Bucket(""), + }, + { + Name: "Bucket with value should generate a flag with the given value", + Flag: Bucket("bucket"), + ExpectedCLI: []string{"--bucket=bucket"}, + }, + { + Name: "Empty Prefix should not generate a flag", + Flag: Prefix(""), + }, + { + Name: "Prefix with value should generate a flag with the given value", + Flag: Prefix("prefix"), + ExpectedCLI: []string{"--prefix=prefix"}, + }, { Name: "Empty CredentialsFile should not generate a flag", Flag: CredentialsFile(""), From c3ba3eb5567ffdd720d0746c1edbb9cddc6f4905 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:27:31 -0800 Subject: [PATCH 10/54] Add kopia azure storage flags --- pkg/kopia/cli/internal/command/commands.go | 1 + .../cli/internal/flag/storage/azure/azure.go | 31 ++++++++++ .../flag/storage/azure/azure_flags.go | 31 ++++++++++ .../flag/storage/azure/azure_flags_test.go | 45 +++++++++++++++ .../internal/flag/storage/azure/azure_test.go | 57 +++++++++++++++++++ .../cli/internal/flag/storage/storage.go | 2 + .../cli/internal/flag/storage/storage_test.go | 16 ++++++ 7 files changed, 183 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure.go create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_test.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index e0136a4c71..3108e93630 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -4,4 +4,5 @@ package command var ( FileSystem = Command{"filesystem"} GCS = Command{"gcs"} + Azure = Command{"azure"} ) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure.go b/pkg/kopia/cli/internal/flag/storage/azure/azure.go new file mode 100644 index 0000000000..0fb6e8387f --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure.go @@ -0,0 +1,31 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// New returns a builder for the Azure subcommand storage. +func New(f model.StorageFlag) (*safecli.Builder, error) { + prefix := model.GenerateFullRepoPath(f.Location.Prefix(), f.RepoPathPrefix) + return command.NewCommandBuilder(command.Azure, + Countainer(f.Location.BucketName()), + Prefix(prefix), + ) +} diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go new file mode 100644 index 0000000000..a9943e617a --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go @@ -0,0 +1,31 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + +// +// Azure flags. +// + +// Prefix creates a new Azure prefix flag with a given prefix. +func Prefix(prefix string) flag.Applier { + return flag.NewStringFlag("--prefix", prefix) +} + +// Countainer creates a new Azure container flag with a given container name. +func Countainer(name string) flag.Applier { + return flag.NewStringFlag("--container", name) +} diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go new file mode 100644 index 0000000000..6d44ed325d --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestStorageAzureFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Prefix should not generate a flag", + Flag: Prefix(""), + }, + { + Name: "Prefix with value should generate a flag with the given value", + Flag: Prefix("prefix"), + ExpectedCLI: []string{"--prefix=prefix"}, + }, + { + Name: "Empty AzureCountainer should not generate a flag", + Flag: Countainer(""), + }, + { + Name: "AzureCountainer with value should generate a flag with the given value", + Flag: Countainer("container"), + ExpectedCLI: []string{"--container=container"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go new file mode 100644 index 0000000000..3e16ac8f40 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageAzure(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty Azure storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{"azure"}, + }, + { + Name: "Azure with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + rs.BucketKey: []byte("container"), + }, + }) + }, + ExpectedCLI: []string{ + "azure", + "--container=container", + "--prefix=prefix/repo/path/prefix/", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go index 2b0b564ac6..08e36d8897 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -22,6 +22,7 @@ import ( cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/azure" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/gcs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" @@ -57,6 +58,7 @@ func Storage(location model.Location, repoPathPrefix string, opts ...Option) mod // Register storage builders. factory[rs.LocTypeFilestore] = fs.New factory[rs.LocTypeGCS] = gcs.New + factory[rs.LocTypeAzure] = azure.New }) // create a new storage with the given location, repo path prefix and defaults. s := model.StorageFlag{ diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 2164afc7ed..09031550b4 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -178,6 +178,22 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { "--prefix=/path/to/prefix/prefixfs/", }, }, + { + name: "Azure should generate azure args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("azure"), + rs.BucketKey: []byte("bucket"), + rs.PrefixKey: []byte("/path/to/prefix"), + }, + "prefixfs", + ), + expCLI: []string{ + "azure", + "--container=bucket", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, } for _, tt := range tests { From 0c7c121f4f57663766819cc06632469e12b93585 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:39:17 -0800 Subject: [PATCH 11/54] Add kopia s3 and s3 compliant storage flags --- pkg/kopia/cli/internal/command/commands.go | 1 + pkg/kopia/cli/internal/flag/storage/s3/s3.go | 61 ++++++++++ .../cli/internal/flag/storage/s3/s3_flags.go | 53 +++++++++ .../internal/flag/storage/s3/s3_flags_test.go | 81 +++++++++++++ .../cli/internal/flag/storage/s3/s3_test.go | 75 +++++++++++++ .../cli/internal/flag/storage/storage.go | 3 + .../cli/internal/flag/storage/storage_test.go | 106 ++++++++++++++++++ 7 files changed, 380 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3.go create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_test.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index 3108e93630..ec4718dbf5 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -5,4 +5,5 @@ var ( FileSystem = Command{"filesystem"} GCS = Command{"gcs"} Azure = Command{"azure"} + S3 = Command{"s3"} ) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3.go b/pkg/kopia/cli/internal/flag/storage/s3/s3.go new file mode 100644 index 0000000000..887e1e3584 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "strings" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// New returns a builder for the S3 subcommand storage. +func New(s model.StorageFlag) (*safecli.Builder, error) { + endpoint := resolveS3Endpoint(s.Location.Endpoint(), s.GetLogger()) + prefix := model.GenerateFullRepoPath(s.Location.Prefix(), s.RepoPathPrefix) + return command.NewCommandBuilder(command.S3, + Region(s.Location.Region()), + Bucket(s.Location.BucketName()), + Endpoint(endpoint), + Prefix(prefix), + DisableTLS(s.Location.IsInsecureEndpoint()), + DisableTLSVerify(s.Location.HasSkipSSLVerify()), + ) +} + +// resolveS3Endpoint removes the trailing slash and +// protocol from provided endpoint and +// returns the absolute endpoint string. +func resolveS3Endpoint(endpoint string, logger log.Logger) string { + if endpoint == "" { + return "" + } + + if strings.HasSuffix(endpoint, "/") { + logger.Print("Removing trailing slashes from the endpoint") + endpoint = strings.TrimRight(endpoint, "/") + } + + sp := strings.SplitN(endpoint, "://", 2) + if len(sp) > 1 { + logger.Print("Removing leading protocol from the endpoint") + } + + return sp[len(sp)-1] +} diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go new file mode 100644 index 0000000000..9688b81155 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// +// S3 flags. +// + +// Bucket creates a new S3 bucket flag with a given bucket name. +func Bucket(bucket string) flag.Applier { + return flag.NewStringFlag("--bucket", bucket) +} + +// Endpoint creates a new S3 endpoint flag with a given endpoint. +func Endpoint(endpoint string) flag.Applier { + return flag.NewStringFlag("--endpoint", endpoint) +} + +// Prefix creates a new S3 prefix flag with a given prefix. +func Prefix(prefix string) flag.Applier { + return flag.NewStringFlag("--prefix", prefix) +} + +// Region creates a new S3 region flag with a given region. +func Region(region string) flag.Applier { + return flag.NewStringFlag("--region", region) +} + +// DisableTLS creates a new S3 disable TLS flag. +func DisableTLS(disable bool) flag.Applier { + return flag.NewBoolFlag("--disable-tls", disable) +} + +// DisableTLSVerify creates a new S3 disable TLS verification flag. +func DisableTLSVerify(disable bool) flag.Applier { + return flag.NewBoolFlag("--disable-tls-verification", disable) +} diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go new file mode 100644 index 0000000000..f77ebafa5d --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestStorageS3Flags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Bucket should not generate a flag", + Flag: Bucket(""), + }, + { + Name: "Bucket with value should generate a flag with the given value", + Flag: Bucket("bucket"), + ExpectedCLI: []string{"--bucket=bucket"}, + }, + { + Name: "Empty Endpoint should not generate a flag", + Flag: Endpoint(""), + }, + { + Name: "Endpoint with value should generate a flag with the given value", + Flag: Endpoint("endpoint"), + ExpectedCLI: []string{"--endpoint=endpoint"}, + }, + { + Name: "Empty Prefix should not generate a flag", + Flag: Prefix(""), + }, + { + Name: "Prefix with value should generate a flag with the given value", + Flag: Prefix("prefix"), + ExpectedCLI: []string{"--prefix=prefix"}, + }, + { + Name: "Empty Region should not generate a flag", + Flag: Region(""), + }, + { + Name: "Region with value should generate a flag with the given value", + Flag: Region("region"), + ExpectedCLI: []string{"--region=region"}, + }, + { + Name: "DisableTLS(false) should not generate a flag", + Flag: DisableTLS(false), + }, + { + Name: "DisableTLS(true) should generate a flag", + Flag: DisableTLS(true), + ExpectedCLI: []string{"--disable-tls"}, + }, + { + Name: "DisableTLSVerify(false) should not generate a flag", + Flag: DisableTLSVerify(false), + }, + { + Name: "DisableTLSVerify(true) should generate a flag", + Flag: DisableTLSVerify(true), + ExpectedCLI: []string{"--disable-tls-verification"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go new file mode 100644 index 0000000000..131660c880 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go @@ -0,0 +1,75 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageS3(t *testing.T) { check.TestingT(t) } + +var logger = &test.StringLogger{} + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty S3 storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{ + "s3", + }, + }, + { + Name: "S3 with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + rs.EndpointKey: []byte("http://endpoint/path/"), + rs.RegionKey: []byte("region"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + Logger: logger, + }) + }, + ExpectedCLI: []string{ + "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repo/path/prefix/", + "--disable-tls", + "--disable-tls-verification", + }, + + Logger: logger, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go index 08e36d8897..9c21745a3e 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -26,6 +26,7 @@ import ( "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/gcs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/s3" ) // Option is a function that sets a storage option. @@ -59,6 +60,8 @@ func Storage(location model.Location, repoPathPrefix string, opts ...Option) mod factory[rs.LocTypeFilestore] = fs.New factory[rs.LocTypeGCS] = gcs.New factory[rs.LocTypeAzure] = azure.New + factory[rs.LocTypeS3] = s3.New + factory[rs.LocTypes3Compliant] = s3.New }) // create a new storage with the given location, repo path prefix and defaults. s := model.StorageFlag{ diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 09031550b4..468e65f8e8 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -194,6 +194,112 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { "--prefix=/path/to/prefix/prefixfs/", }, }, + { + name: "S3 should generate s3 args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + { + name: "S3 with no prefix should use onlu repo path prefix", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=prefixfs", + "--disable-tls", + "--disable-tls-verification", + }, + }, + { + name: "S3 with no endpoint should omit endpoint flag", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, + { + name: "S3 endpoint with trailing slashes should be trimmed", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("https://endpoint.com//////"), // slashes will be trimmed + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, + { + name: "S3 compliant should generate s3 args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3Compliant"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + "--disable-tls", + "--disable-tls-verification", + }, + }, } for _, tt := range tests { From 9850f4a450fe5a0ce37ab18189987b88d05dce00 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 12 Feb 2024 12:11:54 -0800 Subject: [PATCH 12/54] Fix Apply and test.Suit Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- pkg/kopia/cli/internal/test/flag_suite.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 66148034a1..39a357be53 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -36,7 +36,7 @@ func Apply(cli safecli.CommandAppender, flags ...Applier) error { if flag == nil { // if the flag is nil, skip it continue } - if err := flag.Apply(cli); err != nil { + if err := flag.Apply(sub); err != nil { return err } } diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index 7c41a1ea3a..7fe60d909c 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -62,7 +62,9 @@ func (t *FlagTest) assertNoError(c *check.C, err error) { // assertCLI asserts the builder's CLI output against ExpectedCLI. func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + if t.ExpectedCLI != nil { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + } } // assertLog asserts the builder's log output against ExpectedLog. @@ -74,11 +76,11 @@ func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { // Test runs the flag test. func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { err := flag.Apply(b, ft.Flag) + ft.assertCLI(c, b) if ft.ExpectedErr != nil { ft.assertError(c, err) } else { ft.assertNoError(c, err) - ft.assertCLI(c, b) ft.assertLog(c, b) } } From 246e1c1aac53b73be773197f8df1a9748a76970a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 12 Feb 2024 13:21:04 -0800 Subject: [PATCH 13/54] Remove variadic args for Common and Cache flags Signed-off-by: pavel.larkin --- .../cli/internal/flag/common/common_flags.go | 28 +++---------------- .../internal/flag/common/common_flags_test.go | 14 ++-------- 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/common/common_flags.go b/pkg/kopia/cli/internal/flag/common/common_flags.go index 6df9b1bb5c..3c3000885f 100644 --- a/pkg/kopia/cli/internal/flag/common/common_flags.go +++ b/pkg/kopia/cli/internal/flag/common/common_flags.go @@ -138,18 +138,8 @@ func (f common) Apply(cmd safecli.CommandAppender) error { } // Common creates a new common flag. -// If no arguments are provided, the default common flags are used. -// If one argument is provided, the common flags are used. -// If more than one argument is provided, ErrInvalidCommonArgs is returned. -func Common(args ...cli.CommonArgs) flag.Applier { - switch len(args) { - case 0: - return common{cli.CommonArgs{}} - case 1: - return common{args[0]} - default: - return flag.ErrorFlag(cli.ErrInvalidCommonArgs) - } +func Common(args cli.CommonArgs) flag.Applier { + return common{args} } // cache defines cache flags and implements Applier interface for the cache flags. @@ -169,18 +159,8 @@ func (f cache) Apply(cmd safecli.CommandAppender) error { } // Cache creates a new cache flag. -// If no arguments are provided, the default cache flags are used. -// If one argument is provided, the cache flags are used. -// If more than one argument is provided, ErrInvalidCacheArgs is returned. -func Cache(args ...cli.CacheArgs) flag.Applier { - switch len(args) { - case 0: - return cache{cli.CacheArgs{}} - case 1: - return cache{args[0]} - default: - return flag.ErrorFlag(cli.ErrInvalidCacheArgs) - } +func Cache(args cli.CacheArgs) flag.Applier { + return cache{args} } // JSONOutput creates a new JSON output flag. diff --git a/pkg/kopia/cli/internal/flag/common/common_flags_test.go b/pkg/kopia/cli/internal/flag/common/common_flags_test.go index 2e8ac8cfc1..a7d03f571f 100644 --- a/pkg/kopia/cli/internal/flag/common/common_flags_test.go +++ b/pkg/kopia/cli/internal/flag/common/common_flags_test.go @@ -124,14 +124,9 @@ var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ }, { Name: "Empty Common should generate a flag with default value(s)", - Flag: Common(), + Flag: Common(cli.CommonArgs{}), ExpectedCLI: []string{"--log-level=error"}, }, - { - Name: "Common with more than one cli.CommonArgs should generate an error", - Flag: Common(cli.CommonArgs{}, cli.CommonArgs{}), - ExpectedErr: cli.ErrInvalidCommonArgs, - }, { Name: "Common with values should generate multiple flags with the given values and redact password for logs", Flag: Common(cli.CommonArgs{ @@ -149,18 +144,13 @@ var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ }, { Name: "Empty FlagCacheArgs should generate multiple flags with default values", - Flag: Cache(), + Flag: Cache(cli.CacheArgs{}), ExpectedCLI: []string{ "--cache-directory=/tmp/kopia-cache", "--content-cache-size-limit-mb=0", "--metadata-cache-size-limit-mb=0", }, }, - { - Name: "Cache with more than one cli.CacheArgs should generate an error", - Flag: Cache(cli.CacheArgs{}, cli.CacheArgs{}), - ExpectedErr: cli.ErrInvalidCacheArgs, - }, { Name: "Cache with CacheArgs should generate multiple cache related flags", Flag: Cache(cli.CacheArgs{ From 24707e513c3322a79d1e61c0a24d814153b1de8d Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 16:51:58 -0800 Subject: [PATCH 14/54] pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 2 + pkg/kopia/cli/errors.go | 25 --- pkg/kopia/cli/internal/flag/bool_flag.go | 45 ------ pkg/kopia/cli/internal/flag/flag.go | 81 ---------- pkg/kopia/cli/internal/flag/flag_test.go | 171 --------------------- pkg/kopia/cli/internal/flag/string_flag.go | 78 ---------- pkg/kopia/cli/internal/test/flag_suite.go | 114 -------------- 8 files changed, 3 insertions(+), 515 deletions(-) delete mode 100644 pkg/kopia/cli/errors.go delete mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag_test.go delete mode 100644 pkg/kopia/cli/internal/flag/string_flag.go delete mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/go.mod b/go.mod index 0b81ec2854..afa1376f0e 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.3 +require github.com/kanisterio/safecli v0.0.4 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index fed0f3304c..8247482a7e 100644 --- a/go.sum +++ b/go.sum @@ -361,6 +361,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= +github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= +github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go deleted file mode 100644 index 946dd04e87..0000000000 --- a/pkg/kopia/cli/errors.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cli - -import ( - "github.com/pkg/errors" -) - -// flag errors -var ( - // ErrInvalidFlag is returned when the flag name is empty. - ErrInvalidFlag = errors.New("invalid flag") -) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go deleted file mode 100644 index 4eea665cb2..0000000000 --- a/pkg/kopia/cli/internal/flag/bool_flag.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// boolFlag defines a boolean flag with a given flag name. -// If enabled is set to true, the flag is applied; otherwise, it is not. -type boolFlag struct { - flag string - enabled bool -} - -// Apply appends the flag to the command if the flag is enabled. -func (f boolFlag) Apply(cli safecli.CommandAppender) error { - if f.enabled { - cli.AppendLoggable(f.flag) - } - return nil -} - -// NewBoolFlag creates a new bool flag with a given flag name. -// If the flag name is empty, cli.ErrInvalidFlag is returned. -func NewBoolFlag(flag string, enabled bool) Applier { - if flag == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return boolFlag{flag, enabled} -} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go deleted file mode 100644 index 39a357be53..0000000000 --- a/pkg/kopia/cli/internal/flag/flag.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" -) - -// Applier applies flags/args to the command. -type Applier interface { - // Apply applies the flags/args to the command. - Apply(cli safecli.CommandAppender) error -} - -// Apply appends multiple flags to the CLI. -// If any of the flags encounter an error during the Apply process, -// the error is returned and no changes are made to the CLI. -// If no error, the flags are appended to the CLI. -func Apply(cli safecli.CommandAppender, flags ...Applier) error { - // create a new sub builder which will be used to apply the flags - // to avoid mutating the CLI if an error is encountered. - sub := safecli.NewBuilder() - for _, flag := range flags { - if flag == nil { // if the flag is nil, skip it - continue - } - if err := flag.Apply(sub); err != nil { - return err - } - } - cli.Append(sub) - return nil -} - -// flags defines a collection of Flags. -type flags []Applier - -// Apply applies the flags to the CLI. -func (flags flags) Apply(cli safecli.CommandAppender) error { - return Apply(cli, flags...) -} - -// NewFlags creates a new collection of flags. -func NewFlags(fs ...Applier) Applier { - return flags(fs) -} - -// simpleFlag is a simple implementation of the Applier interface. -type simpleFlag struct { - err error -} - -// Apply does nothing except return an error if one is set. -func (f simpleFlag) Apply(safecli.CommandAppender) error { - return f.err -} - -// EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met -// but Applier interface is required. -func EmptyFlag() Applier { - return simpleFlag{} -} - -// ErrorFlag creates a new flag that returns an error when applied. -// It is useful for creating a flag validation if a condition is not met. -func ErrorFlag(err error) Applier { - return simpleFlag{err} -} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go deleted file mode 100644 index c20a46b450..0000000000 --- a/pkg/kopia/cli/internal/flag/flag_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag_test - -import ( - "errors" - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -var ( - ErrFlag = errors.New("flag error") -) - -// MockFlagApplier is a mock implementation of the FlagApplier interface. -type MockFlagApplier struct { - flagName string - applyErr error -} - -func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { - cli.AppendLoggable(m.flagName) - return m.applyErr -} - -func TestApply(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ - { - Name: "Apply with no flags should generate only the command", - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with nil flags should generate only the command", - Flag: flag.NewFlags(nil, nil), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with flags should generate the command and flags", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "--flag1", applyErr: nil}, - &MockFlagApplier{flagName: "--flag2", applyErr: nil}, - ), - ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, - }, - { - Name: "Apply with one error flag should not modify the command and return the error", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "flag1", applyErr: nil}, - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "NewBoolFlag", - Flag: flag.NewFlags( - flag.NewBoolFlag("--flag1", true), - flag.NewBoolFlag("--flag2", false), - ), - ExpectedCLI: []string{"cmd", "--flag1"}, - }, - { - Name: "NewBoolFlag with empty flag name should return an error", - Flag: flag.NewFlags( - flag.NewBoolFlag("", true), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringFlag", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("--flag2", ""), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1"}, - }, - { - Name: "NewStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewRedactedStringFlag", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", ""), - flag.NewRedactedStringFlag("", "value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, - ExpectedLog: "cmd --flag1=<****> <****>", - }, - { - Name: "NewRedactedStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringValue", - Flag: flag.NewFlags( - flag.NewStringArgument("value1"), - ), - ExpectedCLI: []string{"cmd", "value1"}, - }, - { - Name: "NewStringValue with empty value should return an error", - Flag: flag.NewFlags( - flag.NewStringArgument(""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewFlags should generate multiple flags", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", "value2"), - flag.NewStringArgument("value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, - ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", - }, - { - Name: "NewFlags should generate no flags if one of them returns an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "EmptyFlag should not generate any flags", - Flag: flag.EmptyFlag(), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "ErrorFlag should return an error", - Flag: flag.ErrorFlag(ErrFlag), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, -}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go deleted file mode 100644 index a94c29722c..0000000000 --- a/pkg/kopia/cli/internal/flag/string_flag.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// stringFlag defines a string flag with a given flag name and value. -// If the value is empty, the flag is not applied. -type stringFlag struct { - flag string // flag name - value string // flag value - redacted bool // output the value as redacted -} - -// appenderFunc is a function that appends strings to a command. -type appenderFunc func(...string) *safecli.Builder - -// Apply appends the flag to the command if the value is not empty. -// If the value is redacted, it is appended as redacted. -func (f stringFlag) Apply(cli safecli.CommandAppender) error { - if f.value == "" { - return nil - } - appendValue, appendFlagValue := f.selectAppenderFuncs(cli) - if f.flag == "" { - appendValue(f.value) - } else { - appendFlagValue(f.flag, f.value) - } - return nil -} - -// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. -func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { - if f.redacted { - return cli.AppendRedacted, cli.AppendRedactedKV - } - return cli.AppendLoggable, cli.AppendLoggableKV -} - -// newStringFlag creates a new string flag with a given flag name and value. -func newStringFlag(flag, val string, redacted bool) Applier { - if flag == "" && val == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return stringFlag{flag: flag, value: val, redacted: redacted} -} - -// NewStringFlag creates a new string flag with a given flag name and value. -func NewStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, false) -} - -// NewRedactedStringFlag creates a new string flag with a given flag name and value. -func NewRedactedStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, true) -} - -// NewStringArgument creates a new string argument with a given value. -func NewStringArgument(val string) Applier { - return newStringFlag("", val, false) -} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go deleted file mode 100644 index 7fe60d909c..0000000000 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ /dev/null @@ -1,114 +0,0 @@ -package test - -import ( - "strings" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// FlagTest defines a single test for a flag. -type FlagTest struct { - // Name of the test. (required) - Name string - - // Flag to test. (required) - Flag flag.Applier - - // Expected CLI arguments. (optional) - ExpectedCLI []string - - // Expected log output. (optional) - // if empty, it will be set to ExpectedCLI joined with space. - // if empty and ExpectedCLI is empty, it will be ignored. - ExpectedLog string - - // Expected error. (optional) - // If nil, no error is expected and - // ExpectedCLI and ExpectedLog are checked. - ExpectedErr error -} - -// CheckCommentString implements check.CommentInterface -func (t *FlagTest) CheckCommentString() string { - return t.Name -} - -// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. -func (t *FlagTest) setDefaultExpectedLog() { - if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") - } -} - -// assertError checks the error against ExpectedErr. -func (t *FlagTest) assertError(c *check.C, err error) { - if actualErr := errors.Cause(err); actualErr != nil { - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) - } else { - c.Assert(err, check.Equals, t.ExpectedErr, t) - } -} - -// assertNoError makes sure there is no error. -func (t *FlagTest) assertNoError(c *check.C, err error) { - c.Assert(err, check.IsNil, t) -} - -// assertCLI asserts the builder's CLI output against ExpectedCLI. -func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - if t.ExpectedCLI != nil { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) - } -} - -// assertLog asserts the builder's log output against ExpectedLog. -func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { - t.setDefaultExpectedLog() - c.Check(b.String(), check.Equals, t.ExpectedLog, t) -} - -// Test runs the flag test. -func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { - err := flag.Apply(b, ft.Flag) - ft.assertCLI(c, b) - if ft.ExpectedErr != nil { - ft.assertError(c, err) - } else { - ft.assertNoError(c, err) - ft.assertLog(c, b) - } -} - -// FlagSuite defines a test suite for flags. -type FlagSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Tests []FlagTest // Tests to run. -} - -// TestFlags runs all tests in the flag suite. -func (s *FlagSuite) TestFlags(c *check.C) { - for _, test := range s.Tests { - b := newBuilder(s.Cmd) - test.Test(c, b) - } -} - -// NewFlagSuite creates a new FlagSuite. -func NewFlagSuite(tests []FlagTest) *FlagSuite { - return &FlagSuite{Tests: tests} -} - -// newBuilder creates a new safecli.Builder with the given command. -func newBuilder(cmd string) *safecli.Builder { - builder := safecli.NewBuilder() - if cmd != "" { - builder.AppendLoggable(cmd) - } - return builder -} From 550d124c4d159e0009b2c6bca5b05c54e8368c13 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:40:54 -0800 Subject: [PATCH 15/54] Add pkg/kopia/cli package Signed-off-by: pavel.larkin --- pkg/kopia/cli/doc.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/kopia/cli/doc.go diff --git a/pkg/kopia/cli/doc.go b/pkg/kopia/cli/doc.go new file mode 100644 index 0000000000..6f0681452f --- /dev/null +++ b/pkg/kopia/cli/doc.go @@ -0,0 +1,21 @@ +package cli + +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + _ "github.com/kanisterio/safecli" +) + +// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. From fc918a0846cd6f4594da3536d0cd8982e84f2161 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:41:22 -0800 Subject: [PATCH 16/54] go mod tidy Signed-off-by: pavel.larkin --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 233bc702f1..9e093b449a 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= -github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= From 9fdf94bdfc30503fcc98e7163da50e5360f6839b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 21:37:45 -0800 Subject: [PATCH 17/54] Add Kopia storage helpers Signed-off-by: pavel.larkin --- .../internal/flag/storage/model/factory.go | 56 ------------- .../flag/storage/model/factory_test.go | 81 ------------------- .../flag/storage/model/storage_flag.go | 61 -------------- .../flag/storage/model/storage_flag_test.go | 69 ---------------- .../{flag/storage/model => }/location.go | 2 +- .../{flag/storage/model => }/location_test.go | 9 ++- .../internal/{flag/storage/model => }/path.go | 2 +- .../{flag/storage/model => }/path_test.go | 5 +- 8 files changed, 10 insertions(+), 275 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go rename pkg/kopia/cli/internal/{flag/storage/model => }/location.go (99%) rename pkg/kopia/cli/internal/{flag/storage/model => }/location_test.go (93%) rename pkg/kopia/cli/internal/{flag/storage/model => }/path.go (98%) rename pkg/kopia/cli/internal/{flag/storage/model => }/path_test.go (88%) diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory.go b/pkg/kopia/cli/internal/flag/storage/model/factory.go deleted file mode 100644 index 8a77fbf7f6..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/factory.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// StorageBuilder defines a function that creates -// a safecli.Builder for the storage sub command. -type StorageBuilder func(StorageFlag) (*safecli.Builder, error) - -// StorageBuilderFactory defines a factory interface -// for creating a StorageBuilder by type. -type StorageBuilderFactory interface { - Create(rs.LocType) StorageBuilder -} - -// BuildersFactory defines a map of StorageBuilder by LocType. -type BuildersFactory map[rs.LocType]StorageBuilder - -// Create returns a StorageBuilder by LocType and -// implements the StorageBuilderFactory interface. -func (sb BuildersFactory) Create(locType rs.LocType) StorageBuilder { - if b, found := sb[locType]; found { - return b - } - return sb.unsupportedStorageType(locType) -} - -// unsupportedStorageType returns an error for an unsupported location type. -func (sb BuildersFactory) unsupportedStorageType(locType rs.LocType) StorageBuilder { - return func(StorageFlag) (*safecli.Builder, error) { - return nil, errors.Wrap(cli.ErrUnsupportedStorage, fmt.Sprintf("unsupported location type: '%v'", locType)) - } -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory_test.go b/pkg/kopia/cli/internal/flag/storage/model/factory_test.go deleted file mode 100644 index e0bee04d65..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/factory_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" -) - -func TestFactory(t *testing.T) { check.TestingT(t) } - -type FactorySuite struct{} - -var ( - _ = check.Suite(&FactorySuite{}) - ltBlue = rs.LocType("blue") - ltRed = rs.LocType("red") - ltUnknown = rs.LocType("unknown") -) - -type mockBuilder struct { - cmd string - err error -} - -func (m *mockBuilder) New() StorageBuilder { - return func(sf StorageFlag) (*safecli.Builder, error) { - if m.err != nil { - return nil, m.err - } - b := safecli.NewBuilder() - b.AppendLoggable(m.cmd) - return b, nil - } -} - -func mockFactory() StorageBuilderFactory { - factory := make(BuildersFactory) - - blue := mockBuilder{cmd: "blue"} - red := mockBuilder{cmd: "red"} - - factory[ltBlue] = blue.New() - factory[ltRed] = red.New() - return factory -} - -func (s *FactorySuite) TestBuildersFactory(c *check.C) { - factory := mockFactory() - - b, err := factory.Create(ltBlue)(StorageFlag{}) - c.Assert(err, check.IsNil) - c.Check(b, check.NotNil) - c.Check(b.Build(), check.DeepEquals, []string{"blue"}) - - b, err = factory.Create(ltRed)(StorageFlag{}) - c.Assert(err, check.IsNil) - c.Check(b, check.NotNil) - c.Check(b.Build(), check.DeepEquals, []string{"red"}) - - b, err = factory.Create(ltUnknown)(StorageFlag{}) - c.Assert(err, check.ErrorMatches, ".*unsupported location type: 'unknown'.*") - c.Check(b, check.IsNil) -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go deleted file mode 100644 index 89ac32cb6f..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - log "github.com/kanisterio/kanister/pkg/log" -) - -var ( - // ErrInvalidFactory is returned when the factory is nil. - ErrInvalidFactory = errors.New("factory cannot be nil") -) - -// StorageFlag is a set of flags that are used to create a StorageFlag sub command. -type StorageFlag struct { - Location Location - RepoPathPrefix string - - Factory StorageBuilderFactory - Logger log.Logger -} - -// GetLogger returns the logger. -// If the logger is nil, it returns a NopLogger. -func (s StorageFlag) GetLogger() log.Logger { - if s.Logger == nil { - s.Logger = &cmdlog.NopLogger{} - } - return s.Logger -} - -// Apply applies the storage flags to the command. -func (s StorageFlag) Apply(cli safecli.CommandAppender) error { - if s.Factory == nil { - return ErrInvalidFactory - } - storageBuilder := s.Factory.Create(s.Location.Type()) - storageCLI, err := storageBuilder(s) - if err != nil { - return errors.Wrap(err, "failed to apply storage args") - } - cli.Append(storageCLI) - return nil -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go deleted file mode 100644 index dd211eef4a..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" -) - -func TestStorageFlag(t *testing.T) { check.TestingT(t) } - -type StorageFlagSuite struct{} - -var _ = check.Suite(&StorageFlagSuite{}) - -func (s *StorageFlagSuite) TestGetLogger(c *check.C) { - sf := StorageFlag{} - c.Check(sf.GetLogger(), check.NotNil) - sf.Logger = nil - c.Check(sf.GetLogger(), check.NotNil) -} - -func (s *StorageFlagSuite) TestApplyNoFactory(c *check.C) { - sf := StorageFlag{} - err := sf.Apply(nil) - c.Check(err, check.Equals, ErrInvalidFactory) -} - -func (s *StorageFlagSuite) TestApply(c *check.C) { - sf := StorageFlag{ - Location: Location{ - rs.TypeKey: []byte("blue"), - }, - Factory: mockFactory(), - } - b := safecli.NewBuilder() - err := sf.Apply(b) - c.Check(err, check.IsNil) - c.Check(b.Build(), check.DeepEquals, []string{"blue"}) -} - -func (s *StorageFlagSuite) TestApplyUnknowType(c *check.C) { - sf := StorageFlag{ - Location: Location{ - rs.TypeKey: []byte("unknow"), - }, - Factory: mockFactory(), - } - b := safecli.NewBuilder() - err := sf.Apply(b) - c.Check(err, check.ErrorMatches, ".*failed to apply storage args.*") -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/location.go b/pkg/kopia/cli/internal/location.go similarity index 99% rename from pkg/kopia/cli/internal/flag/storage/model/location.go rename to pkg/kopia/cli/internal/location.go index 3a9a4e4fc9..8b56714487 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/location.go +++ b/pkg/kopia/cli/internal/location.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal import ( "strconv" diff --git a/pkg/kopia/cli/internal/flag/storage/model/location_test.go b/pkg/kopia/cli/internal/location_test.go similarity index 93% rename from pkg/kopia/cli/internal/flag/storage/model/location_test.go rename to pkg/kopia/cli/internal/location_test.go index 1051bce642..f74ca6be71 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/location_test.go +++ b/pkg/kopia/cli/internal/location_test.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal_test import ( "testing" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" "gopkg.in/check.v1" ) @@ -40,17 +41,17 @@ func (s *LocationSuite) TestLocation(c *check.C) { tests := []struct { name string - location Location + location internal.Location expected expected }{ { name: "Test with no fields", - location: Location{}, + location: internal.Location{}, expected: expected{}, }, { name: "Test with all fields", - location: Location{ + location: internal.Location{ rs.TypeKey: []byte("Type1"), rs.RegionKey: []byte("Region1"), rs.BucketKey: []byte("Bucket1"), diff --git a/pkg/kopia/cli/internal/flag/storage/model/path.go b/pkg/kopia/cli/internal/path.go similarity index 98% rename from pkg/kopia/cli/internal/flag/storage/model/path.go rename to pkg/kopia/cli/internal/path.go index 5e18da2cb8..9085ebca4c 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/path.go +++ b/pkg/kopia/cli/internal/path.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal import ( "path" diff --git a/pkg/kopia/cli/internal/flag/storage/model/path_test.go b/pkg/kopia/cli/internal/path_test.go similarity index 88% rename from pkg/kopia/cli/internal/flag/storage/model/path_test.go rename to pkg/kopia/cli/internal/path_test.go index 2fd1762362..4ddfeabed1 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/path_test.go +++ b/pkg/kopia/cli/internal/path_test.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal_test import ( "testing" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "gopkg.in/check.v1" ) @@ -44,7 +45,7 @@ func (s *PathSuite) TestGenerateFullRepoPath(c *check.C) { }, } for _, test := range tests { - got := GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) + got := internal.GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) c.Check(got, check.Equals, test.expected) } } From 57adfc2f41db0efb90c51079a538197c30618f41 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:24:23 -0800 Subject: [PATCH 18/54] Implement Kopia storage Filesystem opts Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 + pkg/kopia/cli/internal/command/command.go | 45 ---- .../cli/internal/command/command_test.go | 97 -------- pkg/kopia/cli/internal/command/commands.go | 6 - pkg/kopia/cli/internal/flag/storage/fs/fs.go | 39 --- .../cli/internal/flag/storage/fs/fs_flags.go | 26 -- .../cli/internal/flag/storage/fs/fs_test.go | 58 ----- .../cli/internal/flag/storage/storage.go | 70 ------ .../cli/internal/flag/storage/storage_test.go | 229 ------------------ pkg/kopia/cli/internal/test/command_suite.go | 134 ---------- pkg/kopia/cli/internal/test/string_logger.go | 58 ----- pkg/kopia/cli/repository/storage/fs/fs.go | 31 +++ .../cli/repository/storage/fs/fs_opts.go | 19 ++ .../storage/fs/fs_opts_test.go} | 21 +- .../cli/repository/storage/fs/fs_test.go | 39 +++ 15 files changed, 103 insertions(+), 771 deletions(-) delete mode 100644 pkg/kopia/cli/internal/command/command.go delete mode 100644 pkg/kopia/cli/internal/command/command_test.go delete mode 100644 pkg/kopia/cli/internal/command/commands.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/storage.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/storage_test.go delete mode 100644 pkg/kopia/cli/internal/test/command_suite.go delete mode 100644 pkg/kopia/cli/internal/test/string_logger.go create mode 100644 pkg/kopia/cli/repository/storage/fs/fs.go create mode 100644 pkg/kopia/cli/repository/storage/fs/fs_opts.go rename pkg/kopia/cli/{internal/flag/storage/fs/fs_flags_test.go => repository/storage/fs/fs_opts_test.go} (54%) create mode 100644 pkg/kopia/cli/repository/storage/fs/fs_test.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 79658eb3a5..97a71e4dce 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -30,4 +30,6 @@ var ( var ( // ErrUnsupportedStorage is returned when the storage is not supported. ErrUnsupportedStorage = errors.New("unsupported storage") + // ErrInvalidRepoPath is returned when the repoPath is empty. + ErrInvalidRepoPath = errors.New("repository path cannot be empty") ) diff --git a/pkg/kopia/cli/internal/command/command.go b/pkg/kopia/cli/internal/command/command.go deleted file mode 100644 index 04acbaecd4..0000000000 --- a/pkg/kopia/cli/internal/command/command.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/kanisterio/safecli" - - clierrors "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// Command is a CLI command/subcommand. -type Command struct { - name string -} - -// Apply applies the command to the CLI. -func (c Command) Apply(cli safecli.CommandAppender) error { - if len(c.name) == 0 { - return clierrors.ErrInvalidCommand - } - cli.AppendLoggable(c.name) - return nil -} - -// NewCommandBuilder returns a new safecli.Builder for the storage sub command. -func NewCommandBuilder(cmd flag.Applier, flags ...flag.Applier) (*safecli.Builder, error) { - b := safecli.NewBuilder() - if err := flag.Apply(b, append([]flag.Applier{cmd}, flags...)...); err != nil { - return nil, err - } - return b, nil -} diff --git a/pkg/kopia/cli/internal/command/command_test.go b/pkg/kopia/cli/internal/command/command_test.go deleted file mode 100644 index f4ae074d37..0000000000 --- a/pkg/kopia/cli/internal/command/command_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "errors" - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/safecli" -) - -func TestCommand(t *testing.T) { check.TestingT(t) } - -type CommandSuite struct{} - -var _ = check.Suite(&CommandSuite{}) - -var ( - errInvalidCommand = errors.New("invalid command") - errInvalidFlag = errors.New("invalid flag") -) - -type mockCommandAndFlag struct { - flagName string - err error -} - -func (m *mockCommandAndFlag) Apply(cli safecli.CommandAppender) error { - if m.err == nil { - cli.AppendLoggable(m.flagName) - } - return m.err -} - -func (s *CommandSuite) TestCommand(c *check.C) { - b := safecli.NewBuilder() - cmd := Command{"cmd"} - err := cmd.Apply(b) - c.Assert(err, check.IsNil) - c.Check(b.Build(), check.DeepEquals, []string{"cmd"}) -} - -func (s *CommandSuite) TestEmptyCommand(c *check.C) { - b := safecli.NewBuilder() - cmd := Command{} - err := cmd.Apply(b) - c.Assert(err, check.Equals, cli.ErrInvalidCommand) -} - -func (s *CommandSuite) TestNewCommandBuilderWithFailedCommand(c *check.C) { - // test if command is invalid - b, err := NewCommandBuilder( - &mockCommandAndFlag{err: errInvalidCommand}, - ) - c.Assert(b, check.IsNil) - c.Assert(err, check.Equals, errInvalidCommand) -} - -func (s *CommandSuite) TestNewCommandBuilderWithFailedFlag(c *check.C) { - // test if flag is invalid - b, err := NewCommandBuilder( - &mockCommandAndFlag{flagName: "cmd"}, - &mockCommandAndFlag{err: errInvalidFlag}, - ) - c.Assert(b, check.IsNil) - c.Assert(err, check.Equals, errInvalidFlag) -} - -func (s *CommandSuite) TestNewCommandBuilder(c *check.C) { - // test if command and flag are valid - b, err := NewCommandBuilder( - &mockCommandAndFlag{flagName: "cmd"}, - &mockCommandAndFlag{flagName: "--flag1"}, - &mockCommandAndFlag{flagName: "--flag2"}, - ) - c.Assert(err, check.IsNil) - c.Check(b.Build(), check.DeepEquals, []string{ - "cmd", - "--flag1", - "--flag2", - }) -} diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go deleted file mode 100644 index 17fa517524..0000000000 --- a/pkg/kopia/cli/internal/command/commands.go +++ /dev/null @@ -1,6 +0,0 @@ -package command - -// Repository storage sub commands. -var ( - FileSystem = Command{"filesystem"} -) diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs.go b/pkg/kopia/cli/internal/flag/storage/fs/fs.go deleted file mode 100644 index 20a8b0fac7..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" -) - -const ( - // DefaultFSMountPath is the default mount path for the filesystem subcommand storage. - DefaultFSMountPath = "/mnt/data" -) - -// New returns a builder for the filesystem subcommand storage. -func New(f model.StorageFlag) (*safecli.Builder, error) { - path := generateFileSystemMountPath(f.Location.Prefix(), f.RepoPathPrefix) - return command.NewCommandBuilder(command.FileSystem, - Path(path), - ) -} - -func generateFileSystemMountPath(locPrefix, repoPathPrefix string) string { - return DefaultFSMountPath + "/" + model.GenerateFullRepoPath(locPrefix, repoPathPrefix) -} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go deleted file mode 100644 index 67c252e1e4..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs - -import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - -// -// Filestore flags. -// - -// Path creates a new path flag with a given path. -func Path(path string) flag.Applier { - return flag.NewStringFlag("--path", path) -} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go deleted file mode 100644 index 13d992abdb..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -func TestStorageFS(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "Empty FS storage flag should generate subcommand with default flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{}) - }, - ExpectedCLI: []string{ - "filesystem", - "--path=/mnt/data/", - }, - }, - { - Name: "FS with values should generate subcommand with specific flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{ - RepoPathPrefix: "repo/path/prefix", - Location: model.Location{ - rs.PrefixKey: []byte("prefix"), - }, - }) - }, - ExpectedCLI: []string{ - "filesystem", - "--path=/mnt/data/prefix/repo/path/prefix/", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go deleted file mode 100644 index 48431e6a97..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "sync" - - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - "github.com/kanisterio/kanister/pkg/log" - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" -) - -// Option is a function that sets a storage option. -type Option func(*model.StorageFlag) - -// WithLogger sets the logger for the storage. -func WithLogger(logger log.Logger) Option { - return func(s *model.StorageFlag) { - s.Logger = logger - } -} - -// WithFactory sets the storage args builder factory for the storage. -func WithFactory(factory model.StorageBuilderFactory) Option { - return func(s *model.StorageFlag) { - s.Factory = factory - } -} - -var ( - // factoryOnce is used to initialize the factory once. - factoryOnce sync.Once - // factory creates a new StorageBuilder by LocType. - factory = model.BuildersFactory{} -) - -// Storage creates a new storage with the given location, repo path prefix and options. -func Storage(location model.Location, repoPathPrefix string, opts ...Option) model.StorageFlag { - factoryOnce.Do(func() { - // Register storage builders. - factory[rs.LocTypeFilestore] = fs.New - }) - // create a new storage with the given location, repo path prefix and defaults. - s := model.StorageFlag{ - Location: location, - RepoPathPrefix: repoPathPrefix, - Logger: &cmdlog.NopLogger{}, - Factory: &factory, - } - // apply storage options. - for _, opt := range opts { - opt(&s) - } - return s -} diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go deleted file mode 100644 index 96fe150d5a..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "fmt" - "testing" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" -) - -func TestStorageFlags(t *testing.T) { check.TestingT(t) } - -type StorageSuite struct{} - -var _ = check.Suite(&StorageSuite{}) - -func (s *StorageSuite) TestLocationMethods(c *check.C) { - type expected struct { - Type rs.LocType - Region string - BucketName string - Endpoint string - Prefix string - IsInsecure bool - HasSkipSSLVerify bool - } - - tests := []struct { - name string - location model.Location - expected expected - }{ - { - name: "Test1", - location: model.Location{ - rs.TypeKey: []byte("Type1"), - rs.RegionKey: []byte("Region1"), - rs.BucketKey: []byte("Bucket1"), - rs.EndpointKey: []byte("http://Endpoint1"), - rs.PrefixKey: []byte("Prefix1"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - expected: expected{ - Type: "Type1", - Region: "Region1", - BucketName: "Bucket1", - Endpoint: "http://Endpoint1", - Prefix: "Prefix1", - IsInsecure: true, - HasSkipSSLVerify: true, - }, - }, - { - name: "Test2", - location: model.Location{ - rs.TypeKey: []byte("Type2"), - rs.RegionKey: []byte("Region2"), - rs.BucketKey: []byte("Bucket2"), - rs.EndpointKey: []byte("https://Endpoint2"), - rs.PrefixKey: []byte("Prefix2"), - rs.SkipSSLVerifyKey: []byte("false"), - }, - expected: expected{ - Type: "Type2", - Region: "Region2", - BucketName: "Bucket2", - Endpoint: "https://Endpoint2", - Prefix: "Prefix2", - IsInsecure: false, - HasSkipSSLVerify: false, - }, - }, - } - - for _, tt := range tests { - c.Assert(tt.location.Type(), check.Equals, tt.expected.Type) - c.Assert(tt.location.Region(), check.Equals, tt.expected.Region) - c.Assert(tt.location.BucketName(), check.Equals, tt.expected.BucketName) - c.Assert(tt.location.Endpoint(), check.Equals, tt.expected.Endpoint) - c.Assert(tt.location.Prefix(), check.Equals, tt.expected.Prefix) - c.Assert(tt.location.IsInsecureEndpoint(), check.Equals, tt.expected.IsInsecure) - c.Assert(tt.location.HasSkipSSLVerify(), check.Equals, tt.expected.HasSkipSSLVerify) - } -} - -func (s *StorageSuite) TestStorageFlag(c *check.C) { - tests := []struct { - name string - storage flag.Applier - expCLI []string - err error - errMsg string - }{ - { - name: "Empty Storage should generate an error", - storage: Storage(nil, ""), - err: cli.ErrUnsupportedStorage, - }, - { - name: "Filesystem without prefix and with repo path should generate repo path", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte(""), - }, - "dir1/subdir/", - ), - expCLI: []string{ - "filesystem", - fmt.Sprintf("--path=%s/dir1/subdir/", fs.DefaultFSMountPath), - }, - }, - { - name: "Filesystem with prefix and repo path should generate merged prefix and repo path", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte("test-prefix"), - }, - "dir1/subdir/", - ), - expCLI: []string{ - "filesystem", - fmt.Sprintf("--path=%s/test-prefix/dir1/subdir/", fs.DefaultFSMountPath), - }, - }, - { - name: "Unsupported storage type should generate an error", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("ftp"), - }, - "prefixfs", - ), - errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", - err: cli.ErrUnsupportedStorage, - }, - } - - for _, tt := range tests { - b := safecli.NewBuilder() - err := tt.storage.Apply(b) - - cmt := check.Commentf("FAIL: %v", tt.name) - if tt.errMsg != "" { - c.Assert(err, check.NotNil, cmt) - c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) - } - - if tt.err == nil { - c.Assert(err, check.IsNil, cmt) - } else { - c.Assert(errors.Cause(err), check.Equals, tt.err, cmt) - } - c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) - } -} - -// MockFlagWithError is a mock flag that always returns an error -type MockFlagWithError struct{} - -var errMock = fmt.Errorf("mock error") - -func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { - return errMock -} - -func (s *StorageSuite) TestNewStorageBuilderWithErrorFlag(c *check.C) { - b, err := command.NewCommandBuilder(command.FileSystem, MockFlagWithError{}) - c.Assert(b, check.IsNil) - c.Assert(err, check.Equals, errMock) -} - -func (s *StorageSuite) TestStorageGetLogger(c *check.C) { - storage := Storage(nil, "prefix") - c.Assert(storage.GetLogger(), check.NotNil) - - nopLog := &cmdlog.NopLogger{} - storage = Storage(nil, "prefix", WithLogger(nopLog)) - c.Assert(storage.GetLogger(), check.Equals, nopLog) -} - -// MockFactory is a mock storage factory -type MockFactory struct{} - -func (f MockFactory) Create(locType rs.LocType) model.StorageBuilder { - return func(s model.StorageFlag) (*safecli.Builder, error) { - return safecli.NewBuilder("mockfactory"), nil - } -} - -func (s *StorageSuite) TestStorageFactory(c *check.C) { - storage := Storage(nil, "prefix") - c.Assert(storage.GetLogger(), check.NotNil) - - mockFactory := &MockFactory{} - storage = Storage(nil, "prefix", WithFactory(mockFactory)) - c.Assert(storage.Factory, check.Equals, mockFactory) - b, err := storage.Factory.Create("anything")(model.StorageFlag{}) - c.Assert(b, check.NotNil) - c.Assert(err, check.IsNil) - c.Assert(b.Build(), check.DeepEquals, []string{"mockfactory"}) -} diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go deleted file mode 100644 index 480c1fcba5..0000000000 --- a/pkg/kopia/cli/internal/test/command_suite.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "fmt" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/log" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// CommonArgs is a set of common arguments for the tests. -var CommonArgs = cli.CommonArgs{ - RepoPassword: "encr-key", - ConfigFilePath: "path/kopia.config", - LogDirectory: "cache/log", -} - -// CommandTest defines a single test for a command. -type CommandTest struct { - // Name of the test. (required) - Name string - - // CLI to test. (required) - CLI func() (safecli.CommandBuilder, error) - - // Expected CLI arguments. (optional) - ExpectedCLI []string - - // Expected log output. (optional) - // if empty, it will be derived from ExpectedCLI by redacting sensitive information. - // if empty and ExpectedCLI is empty, it will be ignored. - ExpectedLog string - - // Expected error. (optional) - // If nil, no error is expected and - // ExpectedCLI and ExpectedLog are checked. - ExpectedErr error - - // LoggerRegex is a list of regular expressions to match against the log output. (optional) - Logger log.Logger - LoggerRegex []string -} - -// CheckCommentString implements check.CommentInterface -func (t *CommandTest) CheckCommentString() string { - return t.Name -} - -func (t *CommandTest) setDefaultExpectedLog() { - if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = RedactCLI(t.ExpectedCLI) - } -} - -func (t *CommandTest) assertError(c *check.C, err error) { - actualErr := errors.Cause(err) - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) -} - -func (t *CommandTest) assertNoError(c *check.C, err error) { - c.Assert(err, check.IsNil, t) -} - -func (t *CommandTest) assertCLI(c *check.C, b safecli.CommandBuilder) { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) -} - -func (t *CommandTest) assertLog(c *check.C, b safecli.CommandBuilder) { - t.setDefaultExpectedLog() - c.Check(fmt.Sprint(b), check.Equals, t.ExpectedLog, t) -} - -func (t *CommandTest) assertLogger(c *check.C) { - log, ok := t.Logger.(*StringLogger) - if !ok { - c.Fatalf("t.Logger is not a StringLogger") - } - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.Name, log, t.LoggerRegex) - for _, regex := range t.LoggerRegex { - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } -} - -// Test runs the command test. -func (t *CommandTest) Test(c *check.C) { - b, err := t.CLI() - if t.ExpectedErr == nil { - t.assertNoError(c, err) - t.assertCLI(c, b) - t.assertLog(c, b) - } else { - t.assertError(c, err) - } - if t.Logger != nil { - t.assertLogger(c) - } -} - -// CommandSuite defines a test suite for commands. -type CommandSuite struct { - Tests []CommandTest -} - -// TestCommands runs all tests in the suite. -func (s *CommandSuite) TestCommands(c *check.C) { - for _, test := range s.Tests { - test.Test(c) - } -} - -// NewCommandSuite creates a new CommandSuite. -func NewCommandSuite(tests []CommandTest) *CommandSuite { - return &CommandSuite{Tests: tests} -} diff --git a/pkg/kopia/cli/internal/test/string_logger.go b/pkg/kopia/cli/internal/test/string_logger.go deleted file mode 100644 index 1a55f46220..0000000000 --- a/pkg/kopia/cli/internal/test/string_logger.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "context" - "io" - "regexp" - - "github.com/kanisterio/kanister/pkg/field" - "github.com/kanisterio/kanister/pkg/log" -) - -// StringLogger implements log.Logger and stores log messages in a slice of strings. -// It is useful for testing. -type StringLogger []string - -// Print appends the message to the slice. -func (l *StringLogger) Print(msg string, fields ...field.M) { - *l = append(*l, msg) -} - -// PrintTo appends the message to the slice. -func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { - *l = append(*l, msg) -} - -// WithContext does nothing. -func (l *StringLogger) WithContext(ctx context.Context) log.Logger { - return l -} - -// WithError does nothing. -func (l *StringLogger) WithError(err error) log.Logger { - return l -} - -// MatchString returns true if any of the log messages match the pattern. -func (l *StringLogger) MatchString(pattern string) bool { - for _, line := range *l { - if found, _ := regexp.MatchString(pattern, line); found { - return true - } - } - return false -} diff --git a/pkg/kopia/cli/repository/storage/fs/fs.go b/pkg/kopia/cli/repository/storage/fs/fs.go new file mode 100644 index 0000000000..d93cb339dd --- /dev/null +++ b/pkg/kopia/cli/repository/storage/fs/fs.go @@ -0,0 +1,31 @@ +package fs + +import ( + "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/log" +) + +const ( + defaultFSMountPath = "/mnt/data" +) + +// New creates a new subcommand for the filesystem storage. +func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { + path, err := generateFileSystemMountPath(location.Prefix(), repoPathPrefix) + if err != nil { + return command.NewErrorArgument(err) + } + return command.NewArguments(subcmdFilesystem, optRepoPath(path)) +} + +// generateFileSystemMountPath generates the mount path for the filesystem storage. +func generateFileSystemMountPath(locPrefix, repoPrefix string) (string, error) { + fullRepoPath := internal.GenerateFullRepoPath(locPrefix, repoPrefix) + if fullRepoPath == "" { + return "", cli.ErrInvalidRepoPath + } + return defaultFSMountPath + "/" + fullRepoPath, nil +} diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts.go b/pkg/kopia/cli/repository/storage/fs/fs_opts.go new file mode 100644 index 0000000000..953b471057 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts.go @@ -0,0 +1,19 @@ +package fs + +import ( + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli/command" +) + +var ( + subcmdFilesystem = command.NewArgument("filesystem") +) + +// optRepoPath creates a new path option with a given path. +// If the path is empty, it returns an error. +func optRepoPath(path string) command.Applier { + if path == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewOptionWithArgument("--path", path) +} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go similarity index 54% rename from pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go rename to pkg/kopia/cli/repository/storage/fs/fs_opts_test.go index c0631f0a75..8bacac053b 100644 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go @@ -17,20 +17,23 @@ package fs import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" ) -func TestFilestoreFlags(t *testing.T) { check.TestingT(t) } +func TestFilesystemOptions(t *testing.T) { check.TestingT(t) } -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "Empty Path should not generate a flag", - Flag: Path(""), + Name: "optRepoPath", + Argument: command.NewArguments(optRepoPath("/path/to/repo")), + ExpectedCLI: []string{"cmd", "--path=/path/to/repo"}, }, { - Name: "Path with value should generate a flag with the given value", - Flag: Path("/path/to/file"), - ExpectedCLI: []string{"--path=/path/to/file"}, + Name: "Invalid RepoPath", + Argument: command.NewArguments(optRepoPath("")), + ExpectedErr: cli.ErrInvalidRepoPath, }, -})) +}}) diff --git a/pkg/kopia/cli/repository/storage/fs/fs_test.go b/pkg/kopia/cli/repository/storage/fs/fs_test.go new file mode 100644 index 0000000000..91bc9c183f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/fs/fs_test.go @@ -0,0 +1,39 @@ +package fs + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewFilesystem(t *testing.T) { check.TestingT(t) } + +func newFilesystem(prefix, repoPath string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewFilesystem", + Argument: newFilesystem("prefix", "repoPath"), + ExpectedCLI: []string{"cmd", "filesystem", "--path=/mnt/data/prefix/repoPath/"}, + }, + { + Name: "NewFilesystem with empty repoPath", + Argument: newFilesystem("prefix", ""), + ExpectedCLI: []string{"cmd", "filesystem", "--path=/mnt/data/prefix/"}, + }, + { + Name: "NewFilesystem with empty local prefix and repo prefix should return error", + Argument: newFilesystem("", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From 18d8dd6804ee70753f7e997a0c167e6e3ca0ae74 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:26:31 -0800 Subject: [PATCH 19/54] Add (c) headers Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/fs/fs.go | 14 ++++++++++++++ pkg/kopia/cli/repository/storage/fs/fs_opts.go | 14 ++++++++++++++ pkg/kopia/cli/repository/storage/fs/fs_test.go | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/pkg/kopia/cli/repository/storage/fs/fs.go b/pkg/kopia/cli/repository/storage/fs/fs.go index d93cb339dd..5683c1ab85 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs.go +++ b/pkg/kopia/cli/repository/storage/fs/fs.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fs import ( diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts.go b/pkg/kopia/cli/repository/storage/fs/fs_opts.go index 953b471057..4968c33045 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_opts.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fs import ( diff --git a/pkg/kopia/cli/repository/storage/fs/fs_test.go b/pkg/kopia/cli/repository/storage/fs/fs_test.go index 91bc9c183f..9f65dcdbe5 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_test.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fs import ( From e1bf966696aa7d9b42a8ae8432eaeebf0f844fd5 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:27:18 -0800 Subject: [PATCH 20/54] Remove unused error Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 97a71e4dce..93afdca4e9 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -22,8 +22,6 @@ import ( var ( // ErrInvalidID is returned when the ID is empty. ErrInvalidID = errors.New("invalid ID") - // ErrInvalidCommand is returned when the command is empty. - ErrInvalidCommand = errors.New("invalid command") ) // storage errors From e51868b7d21001f4716a4b00fc6483070b928ad6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:29:25 -0800 Subject: [PATCH 21/54] Reorganize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/fs/fs_opts.go | 3 ++- pkg/kopia/cli/repository/storage/fs/fs_opts_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts.go b/pkg/kopia/cli/repository/storage/fs/fs_opts.go index 4968c33045..23b3ba7a5c 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_opts.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts.go @@ -15,8 +15,9 @@ package fs import ( - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) var ( diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go index 8bacac053b..c84d90c138 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go @@ -17,10 +17,11 @@ package fs import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) func TestFilesystemOptions(t *testing.T) { check.TestingT(t) } From ff01936e31e71439d9952a436faeca8da13736db Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:54:20 -0800 Subject: [PATCH 22/54] Add Kopia GCS storage opts Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs.go | 37 +++++++++++++ .../cli/repository/storage/gcs/gcs_opts.go | 38 +++++++++++++ .../repository/storage/gcs/gcs_opts_test.go | 43 +++++++++++++++ .../cli/repository/storage/gcs/gcs_test.go | 53 +++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs.go create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs_opts.go create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs_test.go diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs.go b/pkg/kopia/cli/repository/storage/gcs/gcs.go new file mode 100644 index 0000000000..884f67ff99 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs.go @@ -0,0 +1,37 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/consts" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/log" +) + +// New creates a new subcommand for the GCS storage. +func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewArguments(subcmdGCS, + optBucket(location.BucketName()), + optCredentialsFile(consts.GoogleCloudCredsFilePath), + optPrefix(prefix), + ) +} diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go new file mode 100644 index 0000000000..1ddc62702d --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go @@ -0,0 +1,38 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "github.com/kanisterio/safecli/command" +) + +var ( + subcmdGCS = command.NewArgument("gcs") +) + +// optBucket creates a new bucket option with a given name. +func optBucket(name string) command.Applier { + return command.NewOptionWithArgument("--bucket", name) +} + +// optPrefix creates a new prefix option with a given prefix. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) +} + +// optCredentialsFile creates a new GCS credentials file option with a given file path. +func optCredentialsFile(path string) command.Applier { + return command.NewOptionWithArgument("--credentials-file", path) +} diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go new file mode 100644 index 0000000000..65fff33f5c --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -0,0 +1,43 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" +) + +func TestGCSOptions(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optBucket", + Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix"}, + }, + { + Name: "optCredentialsFile", + Argument: command.NewArguments(optCredentialsFile("/tmp/file.creds"), optCredentialsFile("")), + ExpectedCLI: []string{"cmd", "--credentials-file=/tmp/file.creds"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go new file mode 100644 index 0000000000..181e4427cb --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewGCS(t *testing.T) { check.TestingT(t) } + +func newGCS(prefix, repoPath string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewGCS", + Argument: newGCS("prefix", "repoPath"), + ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, + }, + { + Name: "NewGCS with empty repoPath", + Argument: newGCS("prefix", ""), + ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, + }, + { + Name: "NewGCS with empty local prefix and repo prefix should return error", + Argument: newGCS("", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From c3cd83cf9e8ff992485ec05a196264e9ba96bf3d Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:55:42 -0800 Subject: [PATCH 23/54] Reorganize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index 65fff33f5c..a0737a94cd 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -17,9 +17,10 @@ package gcs import ( "testing" + "gopkg.in/check.v1" + "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" - "gopkg.in/check.v1" ) func TestGCSOptions(t *testing.T) { check.TestingT(t) } From 78c7092af6694e839f0582dd3781d7e6971e3ca7 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 00:10:15 -0800 Subject: [PATCH 24/54] Add Kopia Azure storage opts Signed-off-by: pavel.larkin --- .../flag/storage/azure/azure_flags_test.go | 45 --------------- .../internal/flag/storage/azure/azure_test.go | 57 ------------------- .../storage/azure/azure.go | 22 ++++--- .../storage/azure/azure_opts.go} | 22 +++---- .../storage/azure/azure_opts_test.go | 39 +++++++++++++ .../repository/storage/azure/azure_test.go | 54 ++++++++++++++++++ 6 files changed, 118 insertions(+), 121 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_test.go rename pkg/kopia/cli/{internal/flag => repository}/storage/azure/azure.go (51%) rename pkg/kopia/cli/{internal/flag/storage/azure/azure_flags.go => repository/storage/azure/azure_opts.go} (57%) create mode 100644 pkg/kopia/cli/repository/storage/azure/azure_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/azure/azure_test.go diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go deleted file mode 100644 index 6d44ed325d..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package azure - -import ( - "testing" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" - "gopkg.in/check.v1" -) - -func TestStorageAzureFlags(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ - { - Name: "Empty Prefix should not generate a flag", - Flag: Prefix(""), - }, - { - Name: "Prefix with value should generate a flag with the given value", - Flag: Prefix("prefix"), - ExpectedCLI: []string{"--prefix=prefix"}, - }, - { - Name: "Empty AzureCountainer should not generate a flag", - Flag: Countainer(""), - }, - { - Name: "AzureCountainer with value should generate a flag with the given value", - Flag: Countainer("container"), - ExpectedCLI: []string{"--container=container"}, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go deleted file mode 100644 index 3e16ac8f40..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package azure - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -func TestStorageAzure(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "Empty Azure storage flag should generate subcommand with default flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{}) - }, - ExpectedCLI: []string{"azure"}, - }, - { - Name: "Azure with values should generate subcommand with specific flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{ - RepoPathPrefix: "repo/path/prefix", - Location: model.Location{ - rs.PrefixKey: []byte("prefix"), - rs.BucketKey: []byte("container"), - }, - }) - }, - ExpectedCLI: []string{ - "azure", - "--container=container", - "--prefix=prefix/repo/path/prefix/", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure.go b/pkg/kopia/cli/repository/storage/azure/azure.go similarity index 51% rename from pkg/kopia/cli/internal/flag/storage/azure/azure.go rename to pkg/kopia/cli/repository/storage/azure/azure.go index 0fb6e8387f..a00233d604 100644 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure.go +++ b/pkg/kopia/cli/repository/storage/azure/azure.go @@ -15,17 +15,21 @@ package azure import ( - "github.com/kanisterio/safecli" + "github.com/kanisterio/safecli/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/log" ) -// New returns a builder for the Azure subcommand storage. -func New(f model.StorageFlag) (*safecli.Builder, error) { - prefix := model.GenerateFullRepoPath(f.Location.Prefix(), f.RepoPathPrefix) - return command.NewCommandBuilder(command.Azure, - Countainer(f.Location.BucketName()), - Prefix(prefix), +// New creates a new subcommand for the Azure storage. +func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewArguments(subcmdAzure, + optContainer(location.BucketName()), + optPrefix(prefix), ) } diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go b/pkg/kopia/cli/repository/storage/azure/azure_opts.go similarity index 57% rename from pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go rename to pkg/kopia/cli/repository/storage/azure/azure_opts.go index a9943e617a..5618d090eb 100644 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts.go @@ -14,18 +14,20 @@ package azure -import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +import ( + "github.com/kanisterio/safecli/command" +) -// -// Azure flags. -// +var ( + subcmdAzure = command.NewArgument("azure") +) -// Prefix creates a new Azure prefix flag with a given prefix. -func Prefix(prefix string) flag.Applier { - return flag.NewStringFlag("--prefix", prefix) +// optPrefix creates a new prefix option with a given prefix. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) } -// Countainer creates a new Azure container flag with a given container name. -func Countainer(name string) flag.Applier { - return flag.NewStringFlag("--container", name) +// optContainer creates a new container option with a given container name. +func optContainer(name string) command.Applier { + return command.NewOptionWithArgument("--container", name) } diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go new file mode 100644 index 0000000000..f85a3e58da --- /dev/null +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go @@ -0,0 +1,39 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" +) + +func TestAzureOptions(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optContainer", + Argument: command.NewArguments(optContainer("containername"), optContainer("")), + ExpectedCLI: []string{"cmd", "--container=containername"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_test.go b/pkg/kopia/cli/repository/storage/azure/azure_test.go new file mode 100644 index 0000000000..942db5e698 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/azure/azure_test.go @@ -0,0 +1,54 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewAzure(t *testing.T) { check.TestingT(t) } + +func newAzure(prefix, repoPath string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + "bucket": []byte("bucket"), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewAzure", + Argument: newAzure("prefix", "repoPath"), + ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/repoPath/"}, + }, + { + Name: "NewAzure with empty repoPath", + Argument: newAzure("prefix", ""), + ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/"}, + }, + { + Name: "NewAzure with empty local prefix and repo prefix should return error", + Argument: newAzure("", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From b2f956acb2dae4be5f7473d6583c337579e73bc1 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 00:12:09 -0800 Subject: [PATCH 25/54] Fix gcs test Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index 181e4427cb..7a522e31f3 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -30,6 +30,7 @@ func TestNewGCS(t *testing.T) { check.TestingT(t) } func newGCS(prefix, repoPath string) command.Applier { l := internal.Location{ "prefix": []byte(prefix), + "bucket": []byte("bucket"), } return New(l, repoPath, nil) } @@ -38,12 +39,12 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewGCS", Argument: newGCS("prefix", "repoPath"), - ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, + ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, }, { Name: "NewGCS with empty repoPath", Argument: newGCS("prefix", ""), - ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, + ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, }, { Name: "NewGCS with empty local prefix and repo prefix should return error", From b75596634b348d3c3013a6dcef341f62a543b16a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 00:38:20 -0800 Subject: [PATCH 26/54] Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin --- .../cli/internal/flag/storage/s3/s3_flags.go | 53 ------------ .../internal/flag/storage/s3/s3_flags_test.go | 81 ------------------- .../cli/internal/flag/storage/s3/s3_test.go | 75 ----------------- .../flag => repository}/storage/s3/s3.go | 36 +++++---- .../cli/repository/storage/s3/s3_opts.go | 53 ++++++++++++ .../cli/repository/storage/s3/s3_opts_test.go | 59 ++++++++++++++ .../cli/repository/storage/s3/s3_test.go | 80 ++++++++++++++++++ 7 files changed, 213 insertions(+), 224 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_test.go rename pkg/kopia/cli/{internal/flag => repository}/storage/s3/s3.go (58%) create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_opts.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_test.go diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go deleted file mode 100644 index 9688b81155..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package s3 - -import ( - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// -// S3 flags. -// - -// Bucket creates a new S3 bucket flag with a given bucket name. -func Bucket(bucket string) flag.Applier { - return flag.NewStringFlag("--bucket", bucket) -} - -// Endpoint creates a new S3 endpoint flag with a given endpoint. -func Endpoint(endpoint string) flag.Applier { - return flag.NewStringFlag("--endpoint", endpoint) -} - -// Prefix creates a new S3 prefix flag with a given prefix. -func Prefix(prefix string) flag.Applier { - return flag.NewStringFlag("--prefix", prefix) -} - -// Region creates a new S3 region flag with a given region. -func Region(region string) flag.Applier { - return flag.NewStringFlag("--region", region) -} - -// DisableTLS creates a new S3 disable TLS flag. -func DisableTLS(disable bool) flag.Applier { - return flag.NewBoolFlag("--disable-tls", disable) -} - -// DisableTLSVerify creates a new S3 disable TLS verification flag. -func DisableTLSVerify(disable bool) flag.Applier { - return flag.NewBoolFlag("--disable-tls-verification", disable) -} diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go deleted file mode 100644 index f77ebafa5d..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package s3 - -import ( - "testing" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" - "gopkg.in/check.v1" -) - -func TestStorageS3Flags(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ - { - Name: "Empty Bucket should not generate a flag", - Flag: Bucket(""), - }, - { - Name: "Bucket with value should generate a flag with the given value", - Flag: Bucket("bucket"), - ExpectedCLI: []string{"--bucket=bucket"}, - }, - { - Name: "Empty Endpoint should not generate a flag", - Flag: Endpoint(""), - }, - { - Name: "Endpoint with value should generate a flag with the given value", - Flag: Endpoint("endpoint"), - ExpectedCLI: []string{"--endpoint=endpoint"}, - }, - { - Name: "Empty Prefix should not generate a flag", - Flag: Prefix(""), - }, - { - Name: "Prefix with value should generate a flag with the given value", - Flag: Prefix("prefix"), - ExpectedCLI: []string{"--prefix=prefix"}, - }, - { - Name: "Empty Region should not generate a flag", - Flag: Region(""), - }, - { - Name: "Region with value should generate a flag with the given value", - Flag: Region("region"), - ExpectedCLI: []string{"--region=region"}, - }, - { - Name: "DisableTLS(false) should not generate a flag", - Flag: DisableTLS(false), - }, - { - Name: "DisableTLS(true) should generate a flag", - Flag: DisableTLS(true), - ExpectedCLI: []string{"--disable-tls"}, - }, - { - Name: "DisableTLSVerify(false) should not generate a flag", - Flag: DisableTLSVerify(false), - }, - { - Name: "DisableTLSVerify(true) should generate a flag", - Flag: DisableTLSVerify(true), - ExpectedCLI: []string{"--disable-tls-verification"}, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go deleted file mode 100644 index 131660c880..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package s3 - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -func TestStorageS3(t *testing.T) { check.TestingT(t) } - -var logger = &test.StringLogger{} - -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "Empty S3 storage flag should generate subcommand with default flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{}) - }, - ExpectedCLI: []string{ - "s3", - }, - }, - { - Name: "S3 with values should generate subcommand with specific flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{ - RepoPathPrefix: "repo/path/prefix", - Location: model.Location{ - rs.PrefixKey: []byte("prefix"), - rs.EndpointKey: []byte("http://endpoint/path/"), - rs.RegionKey: []byte("region"), - rs.BucketKey: []byte("bucket"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - Logger: logger, - }) - }, - ExpectedCLI: []string{ - "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repo/path/prefix/", - "--disable-tls", - "--disable-tls-verification", - }, - - Logger: logger, - LoggerRegex: []string{ - "Removing leading", - "Removing trailing", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3.go b/pkg/kopia/cli/repository/storage/s3/s3.go similarity index 58% rename from pkg/kopia/cli/internal/flag/storage/s3/s3.go rename to pkg/kopia/cli/repository/storage/s3/s3.go index 887e1e3584..610e03cc1e 100644 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3.go +++ b/pkg/kopia/cli/repository/storage/s3/s3.go @@ -17,25 +17,31 @@ package s3 import ( "strings" - "github.com/kanisterio/safecli" + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" "github.com/kanisterio/kanister/pkg/log" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" ) -// New returns a builder for the S3 subcommand storage. -func New(s model.StorageFlag) (*safecli.Builder, error) { - endpoint := resolveS3Endpoint(s.Location.Endpoint(), s.GetLogger()) - prefix := model.GenerateFullRepoPath(s.Location.Prefix(), s.RepoPathPrefix) - return command.NewCommandBuilder(command.S3, - Region(s.Location.Region()), - Bucket(s.Location.BucketName()), - Endpoint(endpoint), - Prefix(prefix), - DisableTLS(s.Location.IsInsecureEndpoint()), - DisableTLSVerify(s.Location.HasSkipSSLVerify()), +// New creates a new subcommand for the S3 storage. +func New(location internal.Location, repoPathPrefix string, logger log.Logger) command.Applier { + if logger == nil { + logger = intlog.NopLogger{} + } + endpoint := resolveS3Endpoint(location.Endpoint(), logger) + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewArguments(subcmdS3, + optRegion(location.Region()), + optBucket(location.BucketName()), + optEndpoint(endpoint), + optPrefix(prefix), + optDisableTLS(location.IsInsecureEndpoint()), + optDisableTLSVerify(location.HasSkipSSLVerify()), ) } diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go new file mode 100644 index 0000000000..980537f24f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "github.com/kanisterio/safecli/command" +) + +var ( + subcmdS3 = command.NewArgument("s3") +) + +// optBucket creates a new bucket option with a given name. +func optBucket(name string) command.Applier { + return command.NewOptionWithArgument("--bucket", name) +} + +// optEndpoint creates a new endpoint option with a given endpoint. +func optEndpoint(endpoint string) command.Applier { + return command.NewOptionWithArgument("--endpoint", endpoint) +} + +// optPrefix creates a new prefix option with a given prefix. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) +} + +// optRegion creates a new region option with a given region. +func optRegion(region string) command.Applier { + return command.NewOptionWithArgument("--region", region) +} + +// optDisableTLS creates a new disable TLS option with a given value. +func optDisableTLS(disable bool) command.Applier { + return command.NewOption("--disable-tls", disable) +} + +// optDisableTLSVerify creates a new disable TLS verification option with a given value. +func optDisableTLSVerify(disable bool) command.Applier { + return command.NewOption("--disable-tls-verification", disable) +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go new file mode 100644 index 0000000000..44108bbf32 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -0,0 +1,59 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" +) + +func TestS3Options(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optBucket", + Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, + }, + { + Name: "optEndpoint", + Argument: command.NewArguments(optEndpoint("endpoint"), optEndpoint("")), + ExpectedCLI: []string{"cmd", "--endpoint=endpoint"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix"}, + }, + { + Name: "optRegion", + Argument: command.NewArguments(optRegion("region"), optRegion("")), + ExpectedCLI: []string{"cmd", "--region=region"}, + }, + { + Name: "optDisableTLS", + Argument: command.NewArguments(optDisableTLS(true), optDisableTLS(false)), + ExpectedCLI: []string{"cmd", "--disable-tls"}, + }, + { + Name: "optDisableTLSVerify", + Argument: command.NewArguments(optDisableTLSVerify(true), optDisableTLSVerify(false)), + ExpectedCLI: []string{"cmd", "--disable-tls-verification"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go new file mode 100644 index 0000000000..8e566a514f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewS3(t *testing.T) { check.TestingT(t) } + +func newS3(prefix, repoPath, endpoint string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + "endpoint": []byte(endpoint), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewS3", + Argument: newS3("prefix", "repoPath", "http://endpoint/path/"), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + { + Name: "NewS3 with empty repoPath and https endpoint", + Argument: newS3("prefix", "", "https://endpoint/path/"), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + { + Name: "NewS3 with empty repoPath and endpoint", + Argument: newS3("prefix", "", ""), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + { + Name: "NewS3 with empty local prefix and repo prefix should return error", + Argument: newS3("", "", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From 0764e2839f75554ffe424a3beb5987e340d9c1fa Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 01:49:25 -0800 Subject: [PATCH 27/54] Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/log/log.go | 29 +++ .../cli/repository/storage/s3/s3_test.go | 183 ++++++++++++++---- 2 files changed, 175 insertions(+), 37 deletions(-) diff --git a/pkg/kopia/cli/internal/log/log.go b/pkg/kopia/cli/internal/log/log.go index 4d2b29ffe3..e3d2ea2544 100644 --- a/pkg/kopia/cli/internal/log/log.go +++ b/pkg/kopia/cli/internal/log/log.go @@ -17,6 +17,7 @@ package storage import ( "context" "io" + "regexp" "github.com/kanisterio/kanister/pkg/field" "github.com/kanisterio/kanister/pkg/log" @@ -43,3 +44,31 @@ func (NopLogger) WithContext(ctx context.Context) log.Logger { func (NopLogger) WithError(err error) log.Logger { return &NopLogger{} } + +// StringLogger is a logger that stores log messages in a slice of strings. +type StringLogger []string + +func (l *StringLogger) Print(msg string, fields ...field.M) { + *l = append(*l, msg) +} + +func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { + *l = append(*l, msg) +} + +func (l *StringLogger) WithContext(ctx context.Context) log.Logger { + return l +} + +func (l *StringLogger) WithError(err error) log.Logger { + return l +} + +func (l *StringLogger) MatchString(pattern string) bool { + for _, line := range *l { + if found, _ := regexp.MatchString(pattern, line); found { + return true + } + } + return false +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 8e566a514f..2f67b61fb4 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -17,64 +17,173 @@ package s3 import ( "testing" - "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" ) func TestNewS3(t *testing.T) { check.TestingT(t) } -func newS3(prefix, repoPath, endpoint string) command.Applier { - l := internal.Location{ - "prefix": []byte(prefix), - "endpoint": []byte(endpoint), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), +// ArgTest extends test.ArgumentTest to include logger tests. +type ArgTest struct { + test test.ArgumentTest + + location internal.Location // location is the location to use for the test. + repoPath string // repoPath is the repository path to use for the test. + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgTest) Test(c *check.C, cmd string) { + t.test.Argument = New(t.location, t.repoPath, t.Logger) + t.test.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgTest) assertLog(c *check.C) { + if t.Logger == nil { + return + } + log := t.Logger.(*intlog.StringLogger) + if len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + } else { + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } + + } +} + +// ArgSuite defines a suite of tests for a single ArgTest. +type ArgSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) } - return New(l, repoPath, nil) } -var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ +var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ { - Name: "NewS3", - Argument: newS3("prefix", "repoPath", "http://endpoint/path/"), - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repoPath/", - "--disable-tls", - "--disable-tls-verification", + test: test.ArgumentTest{ + Name: "NewS3", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte("http://endpoint/path/"), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "repoPath", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", }, }, { - Name: "NewS3 with empty repoPath and https endpoint", - Argument: newS3("prefix", "", "https://endpoint/path/"), - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/", - "--disable-tls-verification", + test: test.ArgumentTest{ + Name: "NewS3 w/o logger should not panic", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte("http://endpoint/path/"), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), }, + repoPath: "repoPath", }, { - Name: "NewS3 with empty repoPath and endpoint", - Argument: newS3("prefix", "", ""), - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--prefix=prefix/", - "--disable-tls-verification", + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath and https endpoint", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte("https://endpoint/path/"), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", }, }, { - Name: "NewS3 with empty local prefix and repo prefix should return error", - Argument: newS3("", "", ""), - ExpectedErr: cli.ErrInvalidRepoPath, + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath and endpoint", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte(""), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath, prefix and endpoint", + ExpectedErr: cli.ErrInvalidRepoPath, + }, + location: internal.Location{ + "prefix": []byte(""), + "endpoint": []byte(""), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected }, }}) From 03607618b67820073ca87f0ecdeab841f0a85bc4 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 01:53:54 -0800 Subject: [PATCH 28/54] Cleanup tests Signed-off-by: pavel.larkin --- .../cli/repository/storage/s3/s3_test.go | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 2f67b61fb4..0d8cd65d04 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -45,21 +45,30 @@ func (t *ArgTest) Test(c *check.C, cmd string) { t.assertLog(c) } +func (t *ArgTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + // assertLog checks the log output against the expected regexs. func (t *ArgTest) assertLog(c *check.C) { if t.Logger == nil { return } - log := t.Logger.(*intlog.StringLogger) - if len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" { + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) - } else { - for _, regex := range t.LoggerRegex { - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } + return + } + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) } } From 2c6cb6e1b3dadf3a80e3355f9900ba3eddb23771 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 26 Feb 2024 15:38:03 -0800 Subject: [PATCH 29/54] Convert common flags from vars to funcs Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/opts/opts.go | 19 ++++++++++++++----- pkg/kopia/cli/internal/opts/opts_test.go | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pkg/kopia/cli/internal/opts/opts.go b/pkg/kopia/cli/internal/opts/opts.go index bee0ece9d1..fb53f5d679 100644 --- a/pkg/kopia/cli/internal/opts/opts.go +++ b/pkg/kopia/cli/internal/opts/opts.go @@ -16,11 +16,20 @@ package opts import "github.com/kanisterio/safecli/command" -var ( - All = command.NewOption("--all", true) - Delta = command.NewOption("--delta", true) - ShowIdentical = command.NewOption("--show-identical", true) -) +// All creates a new all option. +func All(enabled bool) command.Applier { + return command.NewOption("--all", enabled) +} + +// Delta creates a new delta option. +func Delta(enabled bool) command.Applier { + return command.NewOption("--delta", enabled) +} + +// ShowIdentical creates a new show identical option. +func ShowIdentical(enabled bool) command.Applier { + return command.NewOption("--show-identical", enabled) +} // ReadOnly creates a new read only option. func ReadOnly(enabled bool) command.Applier { diff --git a/pkg/kopia/cli/internal/opts/opts_test.go b/pkg/kopia/cli/internal/opts/opts_test.go index 8ea81c5e25..f7aa7178ee 100644 --- a/pkg/kopia/cli/internal/opts/opts_test.go +++ b/pkg/kopia/cli/internal/opts/opts_test.go @@ -26,6 +26,21 @@ import ( func TestOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "All", + Argument: command.NewArguments(opts.All(true), opts.All(false)), + ExpectedCLI: []string{"cmd", "--all"}, + }, + { + Name: "Delta", + Argument: command.NewArguments(opts.Delta(true), opts.Delta(false)), + ExpectedCLI: []string{"cmd", "--delta"}, + }, + { + Name: "ShowIdentical", + Argument: command.NewArguments(opts.ShowIdentical(true), opts.ShowIdentical(false)), + ExpectedCLI: []string{"cmd", "--show-identical"}, + }, { Name: "Readonly", Argument: command.NewArguments(opts.ReadOnly(true), opts.ReadOnly(false)), From 1a3ee2df4c0422ab5c87e742c5c804d2fae495dd Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:08:02 -0800 Subject: [PATCH 30/54] Add safecli dependency --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index 629fe7c11c..b00d7b6b8a 100644 --- a/go.mod +++ b/go.mod @@ -216,6 +216,8 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) +require github.com/kanisterio/safecli v0.0.3 + require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect diff --git a/go.sum b/go.sum index 8e946fc5ff..69a0e01e4b 100644 --- a/go.sum +++ b/go.sum @@ -359,6 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= +github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From 386e7e942d477e80d19d6a33081c883c88142b0e Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:09:47 -0800 Subject: [PATCH 31/54] add new flag implementations based on the safecli package for the Kopia CLI --- pkg/kopia/cli/errors.go | 25 +++ pkg/kopia/cli/internal/flag/bool_flag.go | 45 ++++++ pkg/kopia/cli/internal/flag/flag.go | 81 ++++++++++ pkg/kopia/cli/internal/flag/flag_test.go | 171 +++++++++++++++++++++ pkg/kopia/cli/internal/flag/string_flag.go | 78 ++++++++++ pkg/kopia/cli/internal/test/flag_suite.go | 112 ++++++++++++++ 6 files changed, 512 insertions(+) create mode 100644 pkg/kopia/cli/errors.go create mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag_test.go create mode 100644 pkg/kopia/cli/internal/flag/string_flag.go create mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go new file mode 100644 index 0000000000..946dd04e87 --- /dev/null +++ b/pkg/kopia/cli/errors.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "github.com/pkg/errors" +) + +// flag errors +var ( + // ErrInvalidFlag is returned when the flag name is empty. + ErrInvalidFlag = errors.New("invalid flag") +) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go new file mode 100644 index 0000000000..4eea665cb2 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/bool_flag.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// boolFlag defines a boolean flag with a given flag name. +// If enabled is set to true, the flag is applied; otherwise, it is not. +type boolFlag struct { + flag string + enabled bool +} + +// Apply appends the flag to the command if the flag is enabled. +func (f boolFlag) Apply(cli safecli.CommandAppender) error { + if f.enabled { + cli.AppendLoggable(f.flag) + } + return nil +} + +// NewBoolFlag creates a new bool flag with a given flag name. +// If the flag name is empty, cli.ErrInvalidFlag is returned. +func NewBoolFlag(flag string, enabled bool) Applier { + if flag == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return boolFlag{flag, enabled} +} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go new file mode 100644 index 0000000000..898ae31e16 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" +) + +// Applier applies flags/args to the command. +type Applier interface { + // Apply applies the flags/args to the command. + Apply(cli safecli.CommandAppender) error +} + +// Apply appends multiple flags to the CLI. +// If any of the flags encounter an error during the Apply process, +// the error is returned and no changes are made to the CLI. +// If no error, the flags are appended to the CLI. +func Apply(cli safecli.CommandAppender, flags ...Applier) error { + // create a new sub builder which will be used to apply the flags + // to avoid mutating the CLI if an error is encountered. + sub := safecli.NewBuilder() + for _, flag := range flags { + if flag == nil { // if the flag is nil, skip it + continue + } + if err := flag.Apply(cli); err != nil { + return err + } + } + cli.Append(sub) + return nil +} + +// flags defines a collection of Flags. +type flags []Applier + +// Apply applies the flags to the CLI. +func (flags flags) Apply(cli safecli.CommandAppender) error { + return Apply(cli, flags...) +} + +// NewFlags creates a new collection of flags. +func NewFlags(fs ...Applier) Applier { + return flags(fs) +} + +// simpleFlag is a simple implementation of the Applier interface. +type simpleFlag struct { + err error +} + +// Apply does nothing except return an error if one is set. +func (f simpleFlag) Apply(safecli.CommandAppender) error { + return f.err +} + +// EmptyFlag creates a new flag that does nothing. +// It is useful for creating a no-op flag when a condition is not met +// but Applier interface is required. +func EmptyFlag() Applier { + return simpleFlag{} +} + +// ErrorFlag creates a new flag that returns an error when applied. +// It is useful for creating a flag validation if a condition is not met. +func ErrorFlag(err error) Applier { + return simpleFlag{err} +} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go new file mode 100644 index 0000000000..c20a46b450 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag_test.go @@ -0,0 +1,171 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_test + +import ( + "errors" + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +var ( + ErrFlag = errors.New("flag error") +) + +// MockFlagApplier is a mock implementation of the FlagApplier interface. +type MockFlagApplier struct { + flagName string + applyErr error +} + +func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { + cli.AppendLoggable(m.flagName) + return m.applyErr +} + +func TestApply(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ + { + Name: "Apply with no flags should generate only the command", + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with nil flags should generate only the command", + Flag: flag.NewFlags(nil, nil), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with flags should generate the command and flags", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "--flag1", applyErr: nil}, + &MockFlagApplier{flagName: "--flag2", applyErr: nil}, + ), + ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, + }, + { + Name: "Apply with one error flag should not modify the command and return the error", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "flag1", applyErr: nil}, + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "NewBoolFlag", + Flag: flag.NewFlags( + flag.NewBoolFlag("--flag1", true), + flag.NewBoolFlag("--flag2", false), + ), + ExpectedCLI: []string{"cmd", "--flag1"}, + }, + { + Name: "NewBoolFlag with empty flag name should return an error", + Flag: flag.NewFlags( + flag.NewBoolFlag("", true), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringFlag", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("--flag2", ""), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1"}, + }, + { + Name: "NewStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewRedactedStringFlag", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", ""), + flag.NewRedactedStringFlag("", "value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, + ExpectedLog: "cmd --flag1=<****> <****>", + }, + { + Name: "NewRedactedStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringValue", + Flag: flag.NewFlags( + flag.NewStringArgument("value1"), + ), + ExpectedCLI: []string{"cmd", "value1"}, + }, + { + Name: "NewStringValue with empty value should return an error", + Flag: flag.NewFlags( + flag.NewStringArgument(""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewFlags should generate multiple flags", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", "value2"), + flag.NewStringArgument("value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, + ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", + }, + { + Name: "NewFlags should generate no flags if one of them returns an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "EmptyFlag should not generate any flags", + Flag: flag.EmptyFlag(), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "ErrorFlag should return an error", + Flag: flag.ErrorFlag(ErrFlag), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, +}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go new file mode 100644 index 0000000000..a94c29722c --- /dev/null +++ b/pkg/kopia/cli/internal/flag/string_flag.go @@ -0,0 +1,78 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// stringFlag defines a string flag with a given flag name and value. +// If the value is empty, the flag is not applied. +type stringFlag struct { + flag string // flag name + value string // flag value + redacted bool // output the value as redacted +} + +// appenderFunc is a function that appends strings to a command. +type appenderFunc func(...string) *safecli.Builder + +// Apply appends the flag to the command if the value is not empty. +// If the value is redacted, it is appended as redacted. +func (f stringFlag) Apply(cli safecli.CommandAppender) error { + if f.value == "" { + return nil + } + appendValue, appendFlagValue := f.selectAppenderFuncs(cli) + if f.flag == "" { + appendValue(f.value) + } else { + appendFlagValue(f.flag, f.value) + } + return nil +} + +// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. +func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { + if f.redacted { + return cli.AppendRedacted, cli.AppendRedactedKV + } + return cli.AppendLoggable, cli.AppendLoggableKV +} + +// newStringFlag creates a new string flag with a given flag name and value. +func newStringFlag(flag, val string, redacted bool) Applier { + if flag == "" && val == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return stringFlag{flag: flag, value: val, redacted: redacted} +} + +// NewStringFlag creates a new string flag with a given flag name and value. +func NewStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, false) +} + +// NewRedactedStringFlag creates a new string flag with a given flag name and value. +func NewRedactedStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, true) +} + +// NewStringArgument creates a new string argument with a given value. +func NewStringArgument(val string) Applier { + return newStringFlag("", val, false) +} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go new file mode 100644 index 0000000000..7c41a1ea3a --- /dev/null +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -0,0 +1,112 @@ +package test + +import ( + "strings" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// FlagTest defines a single test for a flag. +type FlagTest struct { + // Name of the test. (required) + Name string + + // Flag to test. (required) + Flag flag.Applier + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be set to ExpectedCLI joined with space. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error +} + +// CheckCommentString implements check.CommentInterface +func (t *FlagTest) CheckCommentString() string { + return t.Name +} + +// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. +func (t *FlagTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") + } +} + +// assertError checks the error against ExpectedErr. +func (t *FlagTest) assertError(c *check.C, err error) { + if actualErr := errors.Cause(err); actualErr != nil { + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) + } else { + c.Assert(err, check.Equals, t.ExpectedErr, t) + } +} + +// assertNoError makes sure there is no error. +func (t *FlagTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +// assertCLI asserts the builder's CLI output against ExpectedCLI. +func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) +} + +// assertLog asserts the builder's log output against ExpectedLog. +func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { + t.setDefaultExpectedLog() + c.Check(b.String(), check.Equals, t.ExpectedLog, t) +} + +// Test runs the flag test. +func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { + err := flag.Apply(b, ft.Flag) + if ft.ExpectedErr != nil { + ft.assertError(c, err) + } else { + ft.assertNoError(c, err) + ft.assertCLI(c, b) + ft.assertLog(c, b) + } +} + +// FlagSuite defines a test suite for flags. +type FlagSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Tests []FlagTest // Tests to run. +} + +// TestFlags runs all tests in the flag suite. +func (s *FlagSuite) TestFlags(c *check.C) { + for _, test := range s.Tests { + b := newBuilder(s.Cmd) + test.Test(c, b) + } +} + +// NewFlagSuite creates a new FlagSuite. +func NewFlagSuite(tests []FlagTest) *FlagSuite { + return &FlagSuite{Tests: tests} +} + +// newBuilder creates a new safecli.Builder with the given command. +func newBuilder(cmd string) *safecli.Builder { + builder := safecli.NewBuilder() + if cmd != "" { + builder.AppendLoggable(cmd) + } + return builder +} From b815f633208dd54aab0963b69d2298263190d4a6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 3 Feb 2024 16:03:14 -0800 Subject: [PATCH 32/54] apply go fmt Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 898ae31e16..66148034a1 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -68,7 +68,7 @@ func (f simpleFlag) Apply(safecli.CommandAppender) error { } // EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met +// It is useful for creating a no-op flag when a condition is not met // but Applier interface is required. func EmptyFlag() Applier { return simpleFlag{} From bcafabb2ba3c8c1cc71cb31c8417728cdd3e4610 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 12 Feb 2024 12:11:54 -0800 Subject: [PATCH 33/54] Fix Apply and test.Suit Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- pkg/kopia/cli/internal/test/flag_suite.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 66148034a1..39a357be53 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -36,7 +36,7 @@ func Apply(cli safecli.CommandAppender, flags ...Applier) error { if flag == nil { // if the flag is nil, skip it continue } - if err := flag.Apply(cli); err != nil { + if err := flag.Apply(sub); err != nil { return err } } diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index 7c41a1ea3a..7fe60d909c 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -62,7 +62,9 @@ func (t *FlagTest) assertNoError(c *check.C, err error) { // assertCLI asserts the builder's CLI output against ExpectedCLI. func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + if t.ExpectedCLI != nil { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + } } // assertLog asserts the builder's log output against ExpectedLog. @@ -74,11 +76,11 @@ func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { // Test runs the flag test. func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { err := flag.Apply(b, ft.Flag) + ft.assertCLI(c, b) if ft.ExpectedErr != nil { ft.assertError(c, err) } else { ft.assertNoError(c, err) - ft.assertCLI(c, b) ft.assertLog(c, b) } } From 6d61bb4c3e5a533f98bcea127655c7b5eba6e677 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 16:51:58 -0800 Subject: [PATCH 34/54] pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 2 + pkg/kopia/cli/errors.go | 25 --- pkg/kopia/cli/internal/flag/bool_flag.go | 45 ------ pkg/kopia/cli/internal/flag/flag.go | 81 ---------- pkg/kopia/cli/internal/flag/flag_test.go | 171 --------------------- pkg/kopia/cli/internal/flag/string_flag.go | 78 ---------- pkg/kopia/cli/internal/test/flag_suite.go | 114 -------------- 8 files changed, 3 insertions(+), 515 deletions(-) delete mode 100644 pkg/kopia/cli/errors.go delete mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag_test.go delete mode 100644 pkg/kopia/cli/internal/flag/string_flag.go delete mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/go.mod b/go.mod index b00d7b6b8a..fb6a6abd8d 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.3 +require github.com/kanisterio/safecli v0.0.4 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index 69a0e01e4b..5ebbafea17 100644 --- a/go.sum +++ b/go.sum @@ -361,6 +361,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= +github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= +github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go deleted file mode 100644 index 946dd04e87..0000000000 --- a/pkg/kopia/cli/errors.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cli - -import ( - "github.com/pkg/errors" -) - -// flag errors -var ( - // ErrInvalidFlag is returned when the flag name is empty. - ErrInvalidFlag = errors.New("invalid flag") -) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go deleted file mode 100644 index 4eea665cb2..0000000000 --- a/pkg/kopia/cli/internal/flag/bool_flag.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// boolFlag defines a boolean flag with a given flag name. -// If enabled is set to true, the flag is applied; otherwise, it is not. -type boolFlag struct { - flag string - enabled bool -} - -// Apply appends the flag to the command if the flag is enabled. -func (f boolFlag) Apply(cli safecli.CommandAppender) error { - if f.enabled { - cli.AppendLoggable(f.flag) - } - return nil -} - -// NewBoolFlag creates a new bool flag with a given flag name. -// If the flag name is empty, cli.ErrInvalidFlag is returned. -func NewBoolFlag(flag string, enabled bool) Applier { - if flag == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return boolFlag{flag, enabled} -} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go deleted file mode 100644 index 39a357be53..0000000000 --- a/pkg/kopia/cli/internal/flag/flag.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" -) - -// Applier applies flags/args to the command. -type Applier interface { - // Apply applies the flags/args to the command. - Apply(cli safecli.CommandAppender) error -} - -// Apply appends multiple flags to the CLI. -// If any of the flags encounter an error during the Apply process, -// the error is returned and no changes are made to the CLI. -// If no error, the flags are appended to the CLI. -func Apply(cli safecli.CommandAppender, flags ...Applier) error { - // create a new sub builder which will be used to apply the flags - // to avoid mutating the CLI if an error is encountered. - sub := safecli.NewBuilder() - for _, flag := range flags { - if flag == nil { // if the flag is nil, skip it - continue - } - if err := flag.Apply(sub); err != nil { - return err - } - } - cli.Append(sub) - return nil -} - -// flags defines a collection of Flags. -type flags []Applier - -// Apply applies the flags to the CLI. -func (flags flags) Apply(cli safecli.CommandAppender) error { - return Apply(cli, flags...) -} - -// NewFlags creates a new collection of flags. -func NewFlags(fs ...Applier) Applier { - return flags(fs) -} - -// simpleFlag is a simple implementation of the Applier interface. -type simpleFlag struct { - err error -} - -// Apply does nothing except return an error if one is set. -func (f simpleFlag) Apply(safecli.CommandAppender) error { - return f.err -} - -// EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met -// but Applier interface is required. -func EmptyFlag() Applier { - return simpleFlag{} -} - -// ErrorFlag creates a new flag that returns an error when applied. -// It is useful for creating a flag validation if a condition is not met. -func ErrorFlag(err error) Applier { - return simpleFlag{err} -} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go deleted file mode 100644 index c20a46b450..0000000000 --- a/pkg/kopia/cli/internal/flag/flag_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag_test - -import ( - "errors" - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -var ( - ErrFlag = errors.New("flag error") -) - -// MockFlagApplier is a mock implementation of the FlagApplier interface. -type MockFlagApplier struct { - flagName string - applyErr error -} - -func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { - cli.AppendLoggable(m.flagName) - return m.applyErr -} - -func TestApply(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ - { - Name: "Apply with no flags should generate only the command", - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with nil flags should generate only the command", - Flag: flag.NewFlags(nil, nil), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with flags should generate the command and flags", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "--flag1", applyErr: nil}, - &MockFlagApplier{flagName: "--flag2", applyErr: nil}, - ), - ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, - }, - { - Name: "Apply with one error flag should not modify the command and return the error", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "flag1", applyErr: nil}, - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "NewBoolFlag", - Flag: flag.NewFlags( - flag.NewBoolFlag("--flag1", true), - flag.NewBoolFlag("--flag2", false), - ), - ExpectedCLI: []string{"cmd", "--flag1"}, - }, - { - Name: "NewBoolFlag with empty flag name should return an error", - Flag: flag.NewFlags( - flag.NewBoolFlag("", true), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringFlag", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("--flag2", ""), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1"}, - }, - { - Name: "NewStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewRedactedStringFlag", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", ""), - flag.NewRedactedStringFlag("", "value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, - ExpectedLog: "cmd --flag1=<****> <****>", - }, - { - Name: "NewRedactedStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringValue", - Flag: flag.NewFlags( - flag.NewStringArgument("value1"), - ), - ExpectedCLI: []string{"cmd", "value1"}, - }, - { - Name: "NewStringValue with empty value should return an error", - Flag: flag.NewFlags( - flag.NewStringArgument(""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewFlags should generate multiple flags", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", "value2"), - flag.NewStringArgument("value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, - ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", - }, - { - Name: "NewFlags should generate no flags if one of them returns an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "EmptyFlag should not generate any flags", - Flag: flag.EmptyFlag(), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "ErrorFlag should return an error", - Flag: flag.ErrorFlag(ErrFlag), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, -}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go deleted file mode 100644 index a94c29722c..0000000000 --- a/pkg/kopia/cli/internal/flag/string_flag.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// stringFlag defines a string flag with a given flag name and value. -// If the value is empty, the flag is not applied. -type stringFlag struct { - flag string // flag name - value string // flag value - redacted bool // output the value as redacted -} - -// appenderFunc is a function that appends strings to a command. -type appenderFunc func(...string) *safecli.Builder - -// Apply appends the flag to the command if the value is not empty. -// If the value is redacted, it is appended as redacted. -func (f stringFlag) Apply(cli safecli.CommandAppender) error { - if f.value == "" { - return nil - } - appendValue, appendFlagValue := f.selectAppenderFuncs(cli) - if f.flag == "" { - appendValue(f.value) - } else { - appendFlagValue(f.flag, f.value) - } - return nil -} - -// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. -func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { - if f.redacted { - return cli.AppendRedacted, cli.AppendRedactedKV - } - return cli.AppendLoggable, cli.AppendLoggableKV -} - -// newStringFlag creates a new string flag with a given flag name and value. -func newStringFlag(flag, val string, redacted bool) Applier { - if flag == "" && val == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return stringFlag{flag: flag, value: val, redacted: redacted} -} - -// NewStringFlag creates a new string flag with a given flag name and value. -func NewStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, false) -} - -// NewRedactedStringFlag creates a new string flag with a given flag name and value. -func NewRedactedStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, true) -} - -// NewStringArgument creates a new string argument with a given value. -func NewStringArgument(val string) Applier { - return newStringFlag("", val, false) -} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go deleted file mode 100644 index 7fe60d909c..0000000000 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ /dev/null @@ -1,114 +0,0 @@ -package test - -import ( - "strings" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// FlagTest defines a single test for a flag. -type FlagTest struct { - // Name of the test. (required) - Name string - - // Flag to test. (required) - Flag flag.Applier - - // Expected CLI arguments. (optional) - ExpectedCLI []string - - // Expected log output. (optional) - // if empty, it will be set to ExpectedCLI joined with space. - // if empty and ExpectedCLI is empty, it will be ignored. - ExpectedLog string - - // Expected error. (optional) - // If nil, no error is expected and - // ExpectedCLI and ExpectedLog are checked. - ExpectedErr error -} - -// CheckCommentString implements check.CommentInterface -func (t *FlagTest) CheckCommentString() string { - return t.Name -} - -// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. -func (t *FlagTest) setDefaultExpectedLog() { - if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") - } -} - -// assertError checks the error against ExpectedErr. -func (t *FlagTest) assertError(c *check.C, err error) { - if actualErr := errors.Cause(err); actualErr != nil { - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) - } else { - c.Assert(err, check.Equals, t.ExpectedErr, t) - } -} - -// assertNoError makes sure there is no error. -func (t *FlagTest) assertNoError(c *check.C, err error) { - c.Assert(err, check.IsNil, t) -} - -// assertCLI asserts the builder's CLI output against ExpectedCLI. -func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - if t.ExpectedCLI != nil { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) - } -} - -// assertLog asserts the builder's log output against ExpectedLog. -func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { - t.setDefaultExpectedLog() - c.Check(b.String(), check.Equals, t.ExpectedLog, t) -} - -// Test runs the flag test. -func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { - err := flag.Apply(b, ft.Flag) - ft.assertCLI(c, b) - if ft.ExpectedErr != nil { - ft.assertError(c, err) - } else { - ft.assertNoError(c, err) - ft.assertLog(c, b) - } -} - -// FlagSuite defines a test suite for flags. -type FlagSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Tests []FlagTest // Tests to run. -} - -// TestFlags runs all tests in the flag suite. -func (s *FlagSuite) TestFlags(c *check.C) { - for _, test := range s.Tests { - b := newBuilder(s.Cmd) - test.Test(c, b) - } -} - -// NewFlagSuite creates a new FlagSuite. -func NewFlagSuite(tests []FlagTest) *FlagSuite { - return &FlagSuite{Tests: tests} -} - -// newBuilder creates a new safecli.Builder with the given command. -func newBuilder(cmd string) *safecli.Builder { - builder := safecli.NewBuilder() - if cmd != "" { - builder.AppendLoggable(cmd) - } - return builder -} From e6ddb8f6bef447dfed85029a4b6f4632c3d874d5 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:40:54 -0800 Subject: [PATCH 35/54] Add pkg/kopia/cli package Signed-off-by: pavel.larkin --- pkg/kopia/cli/doc.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/kopia/cli/doc.go diff --git a/pkg/kopia/cli/doc.go b/pkg/kopia/cli/doc.go new file mode 100644 index 0000000000..6f0681452f --- /dev/null +++ b/pkg/kopia/cli/doc.go @@ -0,0 +1,21 @@ +package cli + +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + _ "github.com/kanisterio/safecli" +) + +// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. From 4e7ffd787fc799d5869d2cbd083bd2b8f591ee52 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:41:22 -0800 Subject: [PATCH 36/54] go mod tidy Signed-off-by: pavel.larkin --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 5ebbafea17..67c3138454 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= -github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= From f16aea76d2e21eab94df31c1a689eeba7d57e375 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 27 Feb 2024 13:33:47 -0800 Subject: [PATCH 37/54] Update safecli to v0.0.5 Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb6a6abd8d..95b851aa8f 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.4 +require github.com/kanisterio/safecli v0.0.5 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index 67c3138454..26b154c2a6 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= -github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= +github.com/kanisterio/safecli v0.0.5 h1:1B9JkmmE4YYCIj4eYMVUXJXQpbpQ+GSOrGoqacfi58Q= +github.com/kanisterio/safecli v0.0.5/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From dcd6425038f059ea148871fcd8ab1e44b754d417 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 27 Feb 2024 16:25:09 -0800 Subject: [PATCH 38/54] Update safecli to v0.0.6 Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 95b851aa8f..f655f747a9 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.5 +require github.com/kanisterio/safecli v0.0.6 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index 26b154c2a6..0d88ed3030 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.5 h1:1B9JkmmE4YYCIj4eYMVUXJXQpbpQ+GSOrGoqacfi58Q= -github.com/kanisterio/safecli v0.0.5/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= +github.com/kanisterio/safecli v0.0.6 h1:Mq99jK7A/SBiHKZalUIlsk4qaSVZJNbBWb8rjaJv/Jk= +github.com/kanisterio/safecli v0.0.6/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From b093421406784745d822f1fc8c92402803230d25 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 14:58:18 -0800 Subject: [PATCH 39/54] Fix tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/opts/common_opts.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/kopia/cli/internal/opts/common_opts.go b/pkg/kopia/cli/internal/opts/common_opts.go index 4eb8b2b101..641de33528 100644 --- a/pkg/kopia/cli/internal/opts/common_opts.go +++ b/pkg/kopia/cli/internal/opts/common_opts.go @@ -24,7 +24,11 @@ const ( ) // LogDirectory creates a new log directory option with a given directory. +// if the directory is empty, the log directory option is not set. func LogDirectory(dir string) command.Applier { + if dir == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--log-dir", dir) } @@ -38,12 +42,20 @@ func LogLevel(level string) command.Applier { } // ConfigFilePath creates a new config file path option with a given path. +// If the path is empty, the config file path option is not set. func ConfigFilePath(path string) command.Applier { + if path == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--config-file", path) } // RepoPassword creates a new repository password option with a given password. +// If the password is empty, the repository password option is not set. func RepoPassword(password string) command.Applier { + if password == "" { + return command.NewNoopArgument() + } return command.NewOptionWithRedactedArgument("--password", password) } From de3a52bf38e787bf52470b0d81f4043929eb86e3 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:03:11 -0800 Subject: [PATCH 40/54] Add Location.IsPointInTypeSupported Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/location.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/kopia/cli/internal/location.go b/pkg/kopia/cli/internal/location.go index 8b56714487..1d01cfbae6 100644 --- a/pkg/kopia/cli/internal/location.go +++ b/pkg/kopia/cli/internal/location.go @@ -59,3 +59,14 @@ func (l Location) HasSkipSSLVerify() bool { v, _ := strconv.ParseBool(string(l[rs.SkipSSLVerifyKey])) return v } + +// IsPointInTimeSupported returns true if the location supports point-in-time recovery. +// Currently, only S3 and Azure support point-in-time recovery. +func (l Location) IsPointInTypeSupported() bool { + switch l.Type() { + case rs.LocTypeAzure, rs.LocTypeS3, rs.LocTypes3Compliant: + return true + default: + return false + } +} From 722b27915574ed350edf3ccf4436051bfd02bdf4 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:12:55 -0800 Subject: [PATCH 41/54] Add tests for Location.IsPointInTypeSupported Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/location_test.go | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/kopia/cli/internal/location_test.go b/pkg/kopia/cli/internal/location_test.go index f74ca6be71..20d8e979ac 100644 --- a/pkg/kopia/cli/internal/location_test.go +++ b/pkg/kopia/cli/internal/location_test.go @@ -37,6 +37,7 @@ func (s *LocationSuite) TestLocation(c *check.C) { Prefix string IsInsecure bool HasSkipSSLVerify bool + IsPITSupported bool } tests := []struct { @@ -69,6 +70,56 @@ func (s *LocationSuite) TestLocation(c *check.C) { HasSkipSSLVerify: true, }, }, + { + name: "Test PIT Support for S3 Compliant", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypes3Compliant), + }, + expected: expected{ + Type: "s3Compliant", + IsPITSupported: true, + }, + }, + { + name: "Test PIT Support for S3", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeS3), + }, + expected: expected{ + Type: "s3", + IsPITSupported: true, + }, + }, + { + name: "Test PIT Support for Azure", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeAzure), + }, + expected: expected{ + Type: "azure", + IsPITSupported: true, + }, + }, + { + name: "Test No PIT Support for GCS", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeGCS), + }, + expected: expected{ + Type: "gcs", + IsPITSupported: false, + }, + }, + { + name: "Test No PIT Support for FS", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeFilestore), + }, + expected: expected{ + Type: "filestore", + IsPITSupported: false, + }, + }, } for _, test := range tests { c.Check(test.location.Type(), check.Equals, test.expected.Type) @@ -78,5 +129,6 @@ func (s *LocationSuite) TestLocation(c *check.C) { c.Check(test.location.Prefix(), check.Equals, test.expected.Prefix) c.Check(test.location.IsInsecureEndpoint(), check.Equals, test.expected.IsInsecure) c.Check(test.location.HasSkipSSLVerify(), check.Equals, test.expected.HasSkipSSLVerify) + c.Check(test.location.IsPointInTypeSupported(), check.Equals, test.expected.IsPITSupported) } } From db06c4007fa45a7d047f83ddbc6fbae1056271dc Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:31:56 -0800 Subject: [PATCH 42/54] Fix s3 options Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/s3/s3_opts.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go index 980537f24f..8097a78ba4 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -28,7 +28,11 @@ func optBucket(name string) command.Applier { } // optEndpoint creates a new endpoint option with a given endpoint. +// If the endpoint is empty, the endpoint option is not set. func optEndpoint(endpoint string) command.Applier { + if endpoint == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--endpoint", endpoint) } @@ -38,7 +42,11 @@ func optPrefix(prefix string) command.Applier { } // optRegion creates a new region option with a given region. +// If the region is empty, the region option is not set. func optRegion(region string) command.Applier { + if region == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--region", region) } From 0962e3bd3066858781871d68c19530c0f5ddd022 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:32:19 -0800 Subject: [PATCH 43/54] Fix s3 options Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/s3/s3_opts.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go index 8097a78ba4..4c5b3da64c 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -23,7 +23,11 @@ var ( ) // optBucket creates a new bucket option with a given name. +// If the name is empty, the bucket option is not set. func optBucket(name string) command.Applier { + if name == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--bucket", name) } @@ -37,7 +41,11 @@ func optEndpoint(endpoint string) command.Applier { } // optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, the prefix option is not set. func optPrefix(prefix string) command.Applier { + if prefix == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--prefix", prefix) } From a0053897672e19e1fd82c08c9927195734a71408 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:01:47 -0800 Subject: [PATCH 44/54] Fix options to return errors for empty args Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 8 +++-- pkg/kopia/cli/repository/storage/gcs/gcs.go | 4 --- .../cli/repository/storage/gcs/gcs_opts.go | 13 +++++++++ .../repository/storage/gcs/gcs_opts_test.go | 29 ++++++++++++++----- .../cli/repository/storage/gcs/gcs_test.go | 2 +- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 93afdca4e9..0b0b6db69c 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -28,6 +28,10 @@ var ( var ( // ErrUnsupportedStorage is returned when the storage is not supported. ErrUnsupportedStorage = errors.New("unsupported storage") - // ErrInvalidRepoPath is returned when the repoPath is empty. - ErrInvalidRepoPath = errors.New("repository path cannot be empty") + // ErrInvalidPrefix is returned when the prefix is empty. + ErrInvalidPrefix = errors.New("prefix cannot be empty") + // ErrInvalidBucketName is returned when the bucketName is empty. + ErrInvalidBucketName = errors.New("bucket name cannot be empty") + // ErrInvalidCredentialsFile is returned when the credentials file is empty. + ErrInvalidCredentialsFile = errors.New("credentials file cannot be empty") ) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs.go b/pkg/kopia/cli/repository/storage/gcs/gcs.go index 884f67ff99..a430d3a95a 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs.go @@ -18,7 +18,6 @@ import ( "github.com/kanisterio/safecli/command" "github.com/kanisterio/kanister/pkg/consts" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/log" ) @@ -26,9 +25,6 @@ import ( // New creates a new subcommand for the GCS storage. func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidRepoPath) - } return command.NewArguments(subcmdGCS, optBucket(location.BucketName()), optCredentialsFile(consts.GoogleCloudCredsFilePath), diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go index 1ddc62702d..690ce4a03b 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go @@ -15,6 +15,7 @@ package gcs import ( + "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" ) @@ -23,16 +24,28 @@ var ( ) // optBucket creates a new bucket option with a given name. +// If the name is empty, it returns ErrInvalidBucketName. func optBucket(name string) command.Applier { + if name == "" { + return command.NewErrorArgument(cli.ErrInvalidBucketName) + } return command.NewOptionWithArgument("--bucket", name) } // optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidPrefix) + } return command.NewOptionWithArgument("--prefix", prefix) } // optCredentialsFile creates a new GCS credentials file option with a given file path. +// If the file path is empty, it returns ErrInvalidCredentialsFile. func optCredentialsFile(path string) command.Applier { + if path == "" { + return command.NewErrorArgument(cli.ErrInvalidCredentialsFile) + } return command.NewOptionWithArgument("--credentials-file", path) } diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index a0737a94cd..2d4dd99525 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -19,7 +19,7 @@ import ( "gopkg.in/check.v1" - "github.com/kanisterio/safecli/command" + "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/test" ) @@ -27,18 +27,33 @@ func TestGCSOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optBucket", - Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + Name: "optBucket with bucketname should return option", + Argument: optBucket("bucketname"), ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, }, { - Name: "optPrefix", - Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + Name: "optBucket with empty bucketname should return error", + Argument: optBucket(""), + ExpectedErr: cli.ErrInvalidBucketName, + }, + { + Name: "optPrefix with prefix should return option", + Argument: optPrefix("prefix"), ExpectedCLI: []string{"cmd", "--prefix=prefix"}, }, { - Name: "optCredentialsFile", - Argument: command.NewArguments(optCredentialsFile("/tmp/file.creds"), optCredentialsFile("")), + Name: "optPrefix with empty prefix should return error", + Argument: optPrefix(""), + ExpectedErr: cli.ErrInvalidPrefix, + }, + { + Name: "optCredentialsFile with path should return option", + Argument: optCredentialsFile("/tmp/file.creds"), ExpectedCLI: []string{"cmd", "--credentials-file=/tmp/file.creds"}, }, + { + Name: "optCredentialsFile with empty path should return error", + Argument: optCredentialsFile(""), + ExpectedErr: cli.ErrInvalidCredentialsFile, + }, }}) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index 7a522e31f3..f27bf1cbbc 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -49,6 +49,6 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewGCS with empty local prefix and repo prefix should return error", Argument: newGCS("", ""), - ExpectedErr: cli.ErrInvalidRepoPath, + ExpectedErr: cli.ErrInvalidPrefix, }, }}) From 7cabf8a8b2e7cdc203dbe24a0d745fc8b6dd285f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:04:15 -0800 Subject: [PATCH 45/54] Fix options to return errors for empty args Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 0b0b6db69c..b6eea2f9ab 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -28,6 +28,8 @@ var ( var ( // ErrUnsupportedStorage is returned when the storage is not supported. ErrUnsupportedStorage = errors.New("unsupported storage") + // ErrInvalidRepoPath is returned when the repoPath is empty. + ErrInvalidRepoPath = errors.New("repository path cannot be empty") // ErrInvalidPrefix is returned when the prefix is empty. ErrInvalidPrefix = errors.New("prefix cannot be empty") // ErrInvalidBucketName is returned when the bucketName is empty. From 8e1c1f7142e4d4ae23e565e595bc46ee4ab8a352 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:11:02 -0800 Subject: [PATCH 46/54] Fix options to return errors for empty args Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 ++ .../cli/repository/storage/azure/azure.go | 4 ---- .../repository/storage/azure/azure_opts.go | 10 +++++++++ .../storage/azure/azure_opts_test.go | 22 ++++++++++++++----- .../repository/storage/azure/azure_test.go | 2 +- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index b6eea2f9ab..483302b57f 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -36,4 +36,6 @@ var ( ErrInvalidBucketName = errors.New("bucket name cannot be empty") // ErrInvalidCredentialsFile is returned when the credentials file is empty. ErrInvalidCredentialsFile = errors.New("credentials file cannot be empty") + // ErrInvalidContainerName is returned when the containerName is empty. + ErrInvalidContainerName = errors.New("container name cannot be empty") ) diff --git a/pkg/kopia/cli/repository/storage/azure/azure.go b/pkg/kopia/cli/repository/storage/azure/azure.go index a00233d604..5929ff4e27 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure.go +++ b/pkg/kopia/cli/repository/storage/azure/azure.go @@ -17,7 +17,6 @@ package azure import ( "github.com/kanisterio/safecli/command" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/log" ) @@ -25,9 +24,6 @@ import ( // New creates a new subcommand for the Azure storage. func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidRepoPath) - } return command.NewArguments(subcmdAzure, optContainer(location.BucketName()), optPrefix(prefix), diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts.go b/pkg/kopia/cli/repository/storage/azure/azure_opts.go index 5618d090eb..bff5920b52 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts.go @@ -16,6 +16,8 @@ package azure import ( "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) var ( @@ -23,11 +25,19 @@ var ( ) // optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidPrefix) + } return command.NewOptionWithArgument("--prefix", prefix) } // optContainer creates a new container option with a given container name. +// If the name is empty, it returns ErrInvalidContainerName. func optContainer(name string) command.Applier { + if name == "" { + return command.NewErrorArgument(cli.ErrInvalidContainerName) + } return command.NewOptionWithArgument("--container", name) } diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go index f85a3e58da..cdaffb7ced 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go @@ -17,23 +17,33 @@ package azure import ( "testing" + "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" - "github.com/kanisterio/safecli/command" - "github.com/kanisterio/safecli/test" + "github.com/kanisterio/kanister/pkg/kopia/cli" ) func TestAzureOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optContainer", - Argument: command.NewArguments(optContainer("containername"), optContainer("")), + Name: "optContainer with containername should return option", + Argument: optContainer("containername"), ExpectedCLI: []string{"cmd", "--container=containername"}, }, { - Name: "optPrefix", - Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + Name: "optContainer with empty containername should return error", + Argument: optContainer(""), + ExpectedErr: cli.ErrInvalidContainerName, + }, + { + Name: "optPrefix with prefix should return option", + Argument: optPrefix("prefix"), ExpectedCLI: []string{"cmd", "--prefix=prefix"}, }, + { + Name: "optPrefix with empty prefix should return error", + Argument: optPrefix(""), + ExpectedErr: cli.ErrInvalidPrefix, + }, }}) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_test.go b/pkg/kopia/cli/repository/storage/azure/azure_test.go index 942db5e698..f2ec9095aa 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_test.go @@ -49,6 +49,6 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewAzure with empty local prefix and repo prefix should return error", Argument: newAzure("", ""), - ExpectedErr: cli.ErrInvalidRepoPath, + ExpectedErr: cli.ErrInvalidPrefix, }, }}) From 9500b329f18e4fcb106b8213d27c1aff012b212b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:45:17 -0800 Subject: [PATCH 47/54] Support empty prefix Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs_opts.go | 4 ---- pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go | 4 ++-- pkg/kopia/cli/repository/storage/gcs/gcs_test.go | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go index 690ce4a03b..d04c89480c 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go @@ -33,11 +33,7 @@ func optBucket(name string) command.Applier { } // optPrefix creates a new prefix option with a given prefix. -// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidPrefix) - } return command.NewOptionWithArgument("--prefix", prefix) } diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index 2d4dd99525..46d79a7d6e 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -42,9 +42,9 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedCLI: []string{"cmd", "--prefix=prefix"}, }, { - Name: "optPrefix with empty prefix should return error", + Name: "optPrefix with empty prefix should return option with empty string", Argument: optPrefix(""), - ExpectedErr: cli.ErrInvalidPrefix, + ExpectedCLI: []string{"cmd", "--prefix="}, }, { Name: "optCredentialsFile with path should return option", diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index f27bf1cbbc..2bc9280c8b 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -21,7 +21,6 @@ import ( "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) @@ -49,6 +48,6 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewGCS with empty local prefix and repo prefix should return error", Argument: newGCS("", ""), - ExpectedErr: cli.ErrInvalidPrefix, + ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix="}, }, }}) From 777209609104e63a95a0aa163b82e41033a46c69 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:55:25 -0800 Subject: [PATCH 48/54] Support empty prefix Signed-off-by: pavel.larkin --- .../cli/repository/storage/azure/azure_opts.go | 4 ---- .../repository/storage/azure/azure_opts_test.go | 14 +++++--------- .../cli/repository/storage/azure/azure_test.go | 17 +++++++++++------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts.go b/pkg/kopia/cli/repository/storage/azure/azure_opts.go index bff5920b52..1433388c09 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts.go @@ -25,11 +25,7 @@ var ( ) // optPrefix creates a new prefix option with a given prefix. -// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidPrefix) - } return command.NewOptionWithArgument("--prefix", prefix) } diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go index cdaffb7ced..bc46a8b7a0 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go @@ -17,6 +17,7 @@ package azure import ( "testing" + "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" @@ -27,7 +28,7 @@ func TestAzureOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optContainer with containername should return option", + Name: "optContainer", Argument: optContainer("containername"), ExpectedCLI: []string{"cmd", "--container=containername"}, }, @@ -37,13 +38,8 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedErr: cli.ErrInvalidContainerName, }, { - Name: "optPrefix with prefix should return option", - Argument: optPrefix("prefix"), - ExpectedCLI: []string{"cmd", "--prefix=prefix"}, - }, - { - Name: "optPrefix with empty prefix should return error", - Argument: optPrefix(""), - ExpectedErr: cli.ErrInvalidPrefix, + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, }, }}) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_test.go b/pkg/kopia/cli/repository/storage/azure/azure_test.go index f2ec9095aa..6f426edf46 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_test.go @@ -27,10 +27,10 @@ import ( func TestNewAzure(t *testing.T) { check.TestingT(t) } -func newAzure(prefix, repoPath string) command.Applier { +func newAzure(prefix, repoPath, bucket string) command.Applier { l := internal.Location{ "prefix": []byte(prefix), - "bucket": []byte("bucket"), + "bucket": []byte(bucket), } return New(l, repoPath, nil) } @@ -38,17 +38,22 @@ func newAzure(prefix, repoPath string) command.Applier { var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { Name: "NewAzure", - Argument: newAzure("prefix", "repoPath"), + Argument: newAzure("prefix", "repoPath", "bucket"), ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/repoPath/"}, }, { Name: "NewAzure with empty repoPath", - Argument: newAzure("prefix", ""), + Argument: newAzure("prefix", "", "bucket"), ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/"}, }, { Name: "NewAzure with empty local prefix and repo prefix should return error", - Argument: newAzure("", ""), - ExpectedErr: cli.ErrInvalidPrefix, + Argument: newAzure("", "", "bucket"), + ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix="}, + }, + { + Name: "NewAzure with empty bucket should return ErrInvalidContainerName", + Argument: newAzure("", "", ""), + ExpectedErr: cli.ErrInvalidContainerName, }, }}) From a4ddad67f0f8135fac7570a8c850e625b44a39e6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:57:21 -0800 Subject: [PATCH 49/54] Support empty prefix Signed-off-by: pavel.larkin --- .../cli/repository/storage/gcs/gcs_opts_test.go | 12 ++++-------- pkg/kopia/cli/repository/storage/gcs/gcs_test.go | 16 +++++++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index 46d79a7d6e..d45adab333 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -20,6 +20,7 @@ import ( "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" ) @@ -37,14 +38,9 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedErr: cli.ErrInvalidBucketName, }, { - Name: "optPrefix with prefix should return option", - Argument: optPrefix("prefix"), - ExpectedCLI: []string{"cmd", "--prefix=prefix"}, - }, - { - Name: "optPrefix with empty prefix should return option with empty string", - Argument: optPrefix(""), - ExpectedCLI: []string{"cmd", "--prefix="}, + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, }, { Name: "optCredentialsFile with path should return option", diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index 2bc9280c8b..09356db69d 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -21,15 +21,16 @@ import ( "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) func TestNewGCS(t *testing.T) { check.TestingT(t) } -func newGCS(prefix, repoPath string) command.Applier { +func newGCS(prefix, repoPath, bucket string) command.Applier { l := internal.Location{ "prefix": []byte(prefix), - "bucket": []byte("bucket"), + "bucket": []byte(bucket), } return New(l, repoPath, nil) } @@ -37,17 +38,22 @@ func newGCS(prefix, repoPath string) command.Applier { var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { Name: "NewGCS", - Argument: newGCS("prefix", "repoPath"), + Argument: newGCS("prefix", "repoPath", "bucket"), ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, }, { Name: "NewGCS with empty repoPath", - Argument: newGCS("prefix", ""), + Argument: newGCS("prefix", "", "bucket"), ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, }, { Name: "NewGCS with empty local prefix and repo prefix should return error", - Argument: newGCS("", ""), + Argument: newGCS("", "", "bucket"), ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix="}, }, + { + Name: "NewGCS with empty bucket should return ErrInvalidBucketName", + Argument: newGCS("", "", ""), + ExpectedErr: cli.ErrInvalidBucketName, + }, }}) From 37e1dc7903c0ec0ae620335fa286d2a871bd74ab Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 18:21:49 -0800 Subject: [PATCH 50/54] Support empty prefix Signed-off-by: pavel.larkin --- .../cli/repository/storage/s3/arg_test.go | 68 ++++++++++ pkg/kopia/cli/repository/storage/s3/s3.go | 4 - .../cli/repository/storage/s3/s3_opts.go | 9 +- .../cli/repository/storage/s3/s3_opts_test.go | 21 +-- .../cli/repository/storage/s3/s3_test.go | 121 +++++------------- 5 files changed, 112 insertions(+), 111 deletions(-) create mode 100644 pkg/kopia/cli/repository/storage/s3/arg_test.go diff --git a/pkg/kopia/cli/repository/storage/s3/arg_test.go b/pkg/kopia/cli/repository/storage/s3/arg_test.go new file mode 100644 index 0000000000..90776f5a4f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/arg_test.go @@ -0,0 +1,68 @@ +package s3 + +import ( + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" +) + +// ArgTest extends test.ArgumentTest to include logger tests. +type ArgTest struct { + test test.ArgumentTest + + location internal.Location // location is the location to use for the test. + repoPath string // repoPath is the repository path to use for the test. + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgTest) Test(c *check.C, cmd string) { + t.test.Argument = New(t.location, t.repoPath, t.Logger) + t.test.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgTest) assertLog(c *check.C) { + if t.Logger == nil { + return + } + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + return + } + + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// isEmptyLogExpected returns true if the test expects an empty log. +func (t *ArgTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + +// ArgSuite defines a suite of tests for a single ArgTest. +type ArgSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) + } +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3.go b/pkg/kopia/cli/repository/storage/s3/s3.go index 610e03cc1e..7a5489755b 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3.go +++ b/pkg/kopia/cli/repository/storage/s3/s3.go @@ -19,7 +19,6 @@ import ( "github.com/kanisterio/safecli/command" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" "github.com/kanisterio/kanister/pkg/log" @@ -32,9 +31,6 @@ func New(location internal.Location, repoPathPrefix string, logger log.Logger) c } endpoint := resolveS3Endpoint(location.Endpoint(), logger) prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidRepoPath) - } return command.NewArguments(subcmdS3, optRegion(location.Region()), optBucket(location.BucketName()), diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go index 4c5b3da64c..131c5803ad 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -16,6 +16,8 @@ package s3 import ( "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) var ( @@ -23,10 +25,10 @@ var ( ) // optBucket creates a new bucket option with a given name. -// If the name is empty, the bucket option is not set. +// If the name is empty, it returns ErrInvalidBucketName. func optBucket(name string) command.Applier { if name == "" { - return command.NewNoopArgument() + return command.NewErrorArgument(cli.ErrInvalidBucketName) } return command.NewOptionWithArgument("--bucket", name) } @@ -43,9 +45,6 @@ func optEndpoint(endpoint string) command.Applier { // optPrefix creates a new prefix option with a given prefix. // If the prefix is empty, the prefix option is not set. func optPrefix(prefix string) command.Applier { - if prefix == "" { - return command.NewNoopArgument() - } return command.NewOptionWithArgument("--prefix", prefix) } diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go index 44108bbf32..3485f852d2 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -17,20 +17,26 @@ package s3 import ( "testing" - "gopkg.in/check.v1" - "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) func TestS3Options(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optBucket", - Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + Name: "optBucket with bucketname should return option", + Argument: optBucket("bucketname"), ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, }, + { + Name: "optBucket with empty bucketname should return error", + Argument: optBucket(""), + ExpectedErr: cli.ErrInvalidBucketName, + }, { Name: "optEndpoint", Argument: command.NewArguments(optEndpoint("endpoint"), optEndpoint("")), @@ -39,12 +45,7 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "optPrefix", Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), - ExpectedCLI: []string{"cmd", "--prefix=prefix"}, - }, - { - Name: "optRegion", - Argument: command.NewArguments(optRegion("region"), optRegion("")), - ExpectedCLI: []string{"cmd", "--region=region"}, + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, }, { Name: "optDisableTLS", diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 0d8cd65d04..8f688b2309 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -15,6 +15,7 @@ package s3 import ( + "strconv" "testing" "github.com/kanisterio/safecli/test" @@ -23,65 +24,17 @@ import ( "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - "github.com/kanisterio/kanister/pkg/log" ) func TestNewS3(t *testing.T) { check.TestingT(t) } -// ArgTest extends test.ArgumentTest to include logger tests. -type ArgTest struct { - test test.ArgumentTest - - location internal.Location // location is the location to use for the test. - repoPath string // repoPath is the repository path to use for the test. - Logger log.Logger // Logger is the logger to use for the test. (optional) - LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) -} - -// Test runs the test with the given command and checks the log output. -func (t *ArgTest) Test(c *check.C, cmd string) { - t.test.Argument = New(t.location, t.repoPath, t.Logger) - t.test.Test(c, cmd) - t.assertLog(c) -} - -func (t *ArgTest) isEmptyLogExpected() bool { - return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" -} - -// assertLog checks the log output against the expected regexs. -func (t *ArgTest) assertLog(c *check.C) { - if t.Logger == nil { - return - } - - log, ok := t.Logger.(*intlog.StringLogger) - if !ok { - c.Fatalf("t.Logger is not *intlog.StringLogger") - } - if t.isEmptyLogExpected() { - cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) - c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) - return - } - - // Check each regex. - for _, regex := range t.LoggerRegex { - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } -} - -// ArgSuite defines a suite of tests for a single ArgTest. -type ArgSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Arguments []ArgTest // Tests to run. -} - -// TestArguments runs all tests in the suite. -func (s *ArgSuite) TestArguments(c *check.C) { - for _, arg := range s.Arguments { - arg.Test(c, s.Cmd) +func newLocation(prefix, endpoint, region, bucket string, skipSSLVerify bool) internal.Location { + return internal.Location{ + "prefix": []byte(prefix), + "endpoint": []byte(endpoint), + "region": []byte(region), + "bucket": []byte(bucket), + "skipSSLVerify": []byte(strconv.FormatBool(skipSSLVerify)), } } @@ -98,13 +51,7 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte("http://endpoint/path/"), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), repoPath: "repoPath", Logger: &intlog.StringLogger{}, LoggerRegex: []string{ @@ -124,13 +71,7 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte("http://endpoint/path/"), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), repoPath: "repoPath", }, { @@ -141,16 +82,9 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--bucket=bucket", "--endpoint=endpoint/path", "--prefix=prefix/", - "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte("https://endpoint/path/"), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{ @@ -168,29 +102,32 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte(""), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "", "region", "bucket", true), repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, { test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath, prefix and endpoint", - ExpectedErr: cli.ErrInvalidRepoPath, + Name: "NewS3 with empty repoPath, prefix and endpoint", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=", + "--disable-tls-verification", + }, }, - location: internal.Location{ - "prefix": []byte(""), - "endpoint": []byte(""), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), + location: newLocation("", "", "region", "bucket", true), + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", + ExpectedErr: cli.ErrInvalidBucketName, }, + location: internal.Location{}, repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected From 44c133ec05dae55112f85ae614237cc19882092d Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 13:54:31 -0800 Subject: [PATCH 51/54] Fix formatting Signed-off-by: pavel.larkin --- pkg/kopia/cli/doc.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kopia/cli/doc.go b/pkg/kopia/cli/doc.go index 6f0681452f..75425c26e4 100644 --- a/pkg/kopia/cli/doc.go +++ b/pkg/kopia/cli/doc.go @@ -1,5 +1,3 @@ -package cli - // Copyright 2024 The Kanister Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +12,10 @@ package cli // See the License for the specific language governing permissions and // limitations under the License. +// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. + +package cli + import ( _ "github.com/kanisterio/safecli" ) - -// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. From 33b1342f23779161506b26d7f65cf81732cb4b7f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 15:16:35 -0800 Subject: [PATCH 52/54] organize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/args/args.go | 3 ++- pkg/kopia/cli/internal/args/args_test.go | 5 +++-- pkg/kopia/cli/internal/opts/cache_opts.go | 3 ++- pkg/kopia/cli/internal/opts/cache_opts_test.go | 5 +++-- pkg/kopia/cli/internal/opts/common_opts.go | 3 ++- pkg/kopia/cli/internal/opts/common_opts_test.go | 5 +++-- pkg/kopia/cli/internal/opts/opts_test.go | 3 ++- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/kopia/cli/internal/args/args.go b/pkg/kopia/cli/internal/args/args.go index a6a8223b6d..cad732dc9d 100644 --- a/pkg/kopia/cli/internal/args/args.go +++ b/pkg/kopia/cli/internal/args/args.go @@ -15,8 +15,9 @@ package args import ( - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) // ID creates a new ID argument. diff --git a/pkg/kopia/cli/internal/args/args_test.go b/pkg/kopia/cli/internal/args/args_test.go index ea6fac26ff..c8362009e8 100644 --- a/pkg/kopia/cli/internal/args/args_test.go +++ b/pkg/kopia/cli/internal/args/args_test.go @@ -17,10 +17,11 @@ package args_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/args" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/args" ) func TestArgs(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/opts/cache_opts.go b/pkg/kopia/cli/internal/opts/cache_opts.go index 75fa0b8f89..fb1222b4af 100644 --- a/pkg/kopia/cli/internal/opts/cache_opts.go +++ b/pkg/kopia/cli/internal/opts/cache_opts.go @@ -17,8 +17,9 @@ package opts import ( "strconv" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" ) const ( diff --git a/pkg/kopia/cli/internal/opts/cache_opts_test.go b/pkg/kopia/cli/internal/opts/cache_opts_test.go index 40c56d8966..ee87091144 100644 --- a/pkg/kopia/cli/internal/opts/cache_opts_test.go +++ b/pkg/kopia/cli/internal/opts/cache_opts_test.go @@ -17,11 +17,12 @@ package opts_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" ) func TestCacheOptions(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/opts/common_opts.go b/pkg/kopia/cli/internal/opts/common_opts.go index 641de33528..d17f71e5c3 100644 --- a/pkg/kopia/cli/internal/opts/common_opts.go +++ b/pkg/kopia/cli/internal/opts/common_opts.go @@ -15,8 +15,9 @@ package opts import ( - "github.com/kanisterio/kanister/pkg/kopia/cli/args" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" ) const ( diff --git a/pkg/kopia/cli/internal/opts/common_opts_test.go b/pkg/kopia/cli/internal/opts/common_opts_test.go index f3f24131a4..28560a9537 100644 --- a/pkg/kopia/cli/internal/opts/common_opts_test.go +++ b/pkg/kopia/cli/internal/opts/common_opts_test.go @@ -17,11 +17,12 @@ package opts_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" ) func TestCommonOptions(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/opts/opts_test.go b/pkg/kopia/cli/internal/opts/opts_test.go index f7aa7178ee..c0e955fd85 100644 --- a/pkg/kopia/cli/internal/opts/opts_test.go +++ b/pkg/kopia/cli/internal/opts/opts_test.go @@ -17,10 +17,11 @@ package opts_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" ) func TestOptions(t *testing.T) { check.TestingT(t) } From 516a14c56b6d20ce6bbc8e1b9db40a5f65a22581 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 15:20:39 -0800 Subject: [PATCH 53/54] organize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/location_test.go | 3 ++- pkg/kopia/cli/internal/path_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/internal/location_test.go b/pkg/kopia/cli/internal/location_test.go index 20d8e979ac..d28defd5ed 100644 --- a/pkg/kopia/cli/internal/location_test.go +++ b/pkg/kopia/cli/internal/location_test.go @@ -17,9 +17,10 @@ package internal_test import ( "testing" + "gopkg.in/check.v1" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - "gopkg.in/check.v1" ) func TestLocation(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/path_test.go b/pkg/kopia/cli/internal/path_test.go index 4ddfeabed1..e52b0d2c52 100644 --- a/pkg/kopia/cli/internal/path_test.go +++ b/pkg/kopia/cli/internal/path_test.go @@ -17,8 +17,9 @@ package internal_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) func TestPath(t *testing.T) { check.TestingT(t) } From da5058df57c56cd0d7051996615878bf3a2bd7bc Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 1 Mar 2024 13:30:24 -0800 Subject: [PATCH 54/54] Fix s3 tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/test/arg_suit.go | 81 +++++++++ .../cli/repository/storage/s3/arg_test.go | 68 -------- .../cli/repository/storage/s3/s3_opts_test.go | 5 + .../cli/repository/storage/s3/s3_test.go | 157 +++++++++++------- 4 files changed, 179 insertions(+), 132 deletions(-) create mode 100644 pkg/kopia/cli/internal/test/arg_suit.go delete mode 100644 pkg/kopia/cli/repository/storage/s3/arg_test.go diff --git a/pkg/kopia/cli/internal/test/arg_suit.go b/pkg/kopia/cli/internal/test/arg_suit.go new file mode 100644 index 0000000000..21ccbf70e4 --- /dev/null +++ b/pkg/kopia/cli/internal/test/arg_suit.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" +) + +// ArgumentTest extends test.ArgumentTest to include logger tests. +type ArgumentTest struct { + test.ArgumentTest + + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgumentTest) Test(c *check.C, cmd string) { + t.ArgumentTest.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgumentTest) assertLog(c *check.C) { + if t.Logger == nil { + if len(t.LoggerRegex) > 0 { + c.Fatalf("t.Logger is nil but t.LoggerRegex is %#v", t.LoggerRegex) + } + return + } + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + return + } + + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.ArgumentTest.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// isEmptyLogExpected returns true if the test expects an empty log. +func (t *ArgumentTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + +// ArgumentSuite defines a suite of tests for a single ArgumentTest. +type ArgumentSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgumentTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgumentSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) + } +} diff --git a/pkg/kopia/cli/repository/storage/s3/arg_test.go b/pkg/kopia/cli/repository/storage/s3/arg_test.go deleted file mode 100644 index 90776f5a4f..0000000000 --- a/pkg/kopia/cli/repository/storage/s3/arg_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package s3 - -import ( - "github.com/kanisterio/safecli/test" - "gopkg.in/check.v1" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal" - intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - "github.com/kanisterio/kanister/pkg/log" -) - -// ArgTest extends test.ArgumentTest to include logger tests. -type ArgTest struct { - test test.ArgumentTest - - location internal.Location // location is the location to use for the test. - repoPath string // repoPath is the repository path to use for the test. - Logger log.Logger // Logger is the logger to use for the test. (optional) - LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) -} - -// Test runs the test with the given command and checks the log output. -func (t *ArgTest) Test(c *check.C, cmd string) { - t.test.Argument = New(t.location, t.repoPath, t.Logger) - t.test.Test(c, cmd) - t.assertLog(c) -} - -// assertLog checks the log output against the expected regexs. -func (t *ArgTest) assertLog(c *check.C) { - if t.Logger == nil { - return - } - - log, ok := t.Logger.(*intlog.StringLogger) - if !ok { - c.Fatalf("t.Logger is not *intlog.StringLogger") - } - if t.isEmptyLogExpected() { - cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) - c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) - return - } - - // Check each regex. - for _, regex := range t.LoggerRegex { - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } -} - -// isEmptyLogExpected returns true if the test expects an empty log. -func (t *ArgTest) isEmptyLogExpected() bool { - return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" -} - -// ArgSuite defines a suite of tests for a single ArgTest. -type ArgSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Arguments []ArgTest // Tests to run. -} - -// TestArguments runs all tests in the suite. -func (s *ArgSuite) TestArguments(c *check.C) { - for _, arg := range s.Arguments { - arg.Test(c, s.Cmd) - } -} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go index 3485f852d2..a903634819 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -27,6 +27,11 @@ import ( func TestS3Options(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optRegion", + Argument: command.NewArguments(optRegion("region"), optRegion("")), + ExpectedCLI: []string{"cmd", "--region=region"}, + }, { Name: "optBucket with bucketname should return option", Argument: optBucket("bucketname"), diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 8f688b2309..1bb9c77cf4 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -24,6 +24,8 @@ import ( "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + inttest "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/kanister/pkg/log" ) func TestNewS3(t *testing.T) { check.TestingT(t) } @@ -38,98 +40,125 @@ func newLocation(prefix, endpoint, region, bucket string, skipSSLVerify bool) in } } -var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ +// s3test is a test case for NewS3. +type s3test struct { + Name string + Location internal.Location + RepoPath string + ExpectedCLI []string + ExpectedErr error + Logger log.Logger + LoggerRegex []string +} + +// newS3Test creates a new test case for NewS3. +func newS3Test(s3t s3test) inttest.ArgumentTest { + return inttest.ArgumentTest{ + ArgumentTest: test.ArgumentTest{ + Name: s3t.Name, + Argument: New(s3t.Location, s3t.RepoPath, s3t.Logger), + ExpectedCLI: s3t.ExpectedCLI, + ExpectedErr: s3t.ExpectedErr, + }, + Logger: s3t.Logger, + LoggerRegex: s3t.LoggerRegex, + } +} + +// toArgTests converts a list of s3tests to a list of ArgumentTests. +func toArgTests(s3tests []s3test) []inttest.ArgumentTest { + argTests := make([]inttest.ArgumentTest, len(s3tests)) + for i, s3t := range s3tests { + argTests[i] = newS3Test(s3t) + } + return argTests +} + +var _ = check.Suite(&inttest.ArgumentSuite{Cmd: "cmd", Arguments: toArgTests([]s3test{ { - test: test.ArgumentTest{ - Name: "NewS3", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repoPath/", - "--disable-tls", - "--disable-tls-verification", - }, + Name: "NewS3", + Location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), + RepoPath: "repoPath", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", }, - location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), - repoPath: "repoPath", - Logger: &intlog.StringLogger{}, + Logger: &intlog.StringLogger{}, LoggerRegex: []string{ "Removing leading", "Removing trailing", }, }, { - test: test.ArgumentTest{ - Name: "NewS3 w/o logger should not panic", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repoPath/", - "--disable-tls", - "--disable-tls-verification", - }, + Name: "NewS3 w/o logger should not panic", + Location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), + RepoPath: "repoPath", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", }, - location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), - repoPath: "repoPath", + Logger: &intlog.StringLogger{}, }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath and https endpoint", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/", - }, + Name: "NewS3 with empty repoPath and https endpoint", + Location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", }, - location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), - repoPath: "", - Logger: &intlog.StringLogger{}, + Logger: &intlog.StringLogger{}, LoggerRegex: []string{ "Removing leading", "Removing trailing", }, }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath and endpoint", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--prefix=prefix/", - "--disable-tls-verification", - }, + Name: "NewS3 with empty repoPath and endpoint", + Location: newLocation("prefix", "", "region", "bucket", true), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", }, - location: newLocation("prefix", "", "region", "bucket", true), - repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath, prefix and endpoint", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--prefix=", - "--disable-tls-verification", - }, + Name: "NewS3 with empty repoPath, prefix and endpoint", + Location: newLocation("", "", "region", "bucket", true), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=", + "--disable-tls-verification", }, - location: newLocation("", "", "region", "bucket", true), - repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", - ExpectedErr: cli.ErrInvalidBucketName, - }, - location: internal.Location{}, - repoPath: "", + Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", + ExpectedErr: cli.ErrInvalidBucketName, Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, -}}) + { + Name: "NewS3 with empty logger should not panic", + Location: newLocation("", "https://endpoint/path/", "", "bucket", false), + ExpectedCLI: []string{"cmd", "s3", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=", + }, + }, +})})