From 02f0449a3dd3221a970d3178179e4d51092c41ac Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 16:19:04 -0800 Subject: [PATCH] Add kopia CLI repository connect command --- pkg/kopia/cli/internal/command/command.go | 8 + .../cli/internal/command/command_test.go | 11 + pkg/kopia/cli/internal/command/commands.go | 25 ++ .../flag/repository/repository_flags.go | 65 +++++ .../flag/repository/repository_flags_test.go | 107 +++++++ pkg/kopia/cli/repository/repository_create.go | 61 ++++ pkg/kopia/cli/repository/repository_test.go | 275 ++++++++++++++++++ 7 files changed, 552 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/repository/repository_flags.go create mode 100644 pkg/kopia/cli/internal/flag/repository/repository_flags_test.go create mode 100644 pkg/kopia/cli/repository/repository_create.go create mode 100644 pkg/kopia/cli/repository/repository_test.go diff --git a/pkg/kopia/cli/internal/command/command.go b/pkg/kopia/cli/internal/command/command.go index 04acbaecd4..68c7d531e3 100644 --- a/pkg/kopia/cli/internal/command/command.go +++ b/pkg/kopia/cli/internal/command/command.go @@ -17,8 +17,10 @@ package command import ( "github.com/kanisterio/safecli" + "github.com/kanisterio/kanister/pkg/kopia/cli" clierrors "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" ) // Command is a CLI command/subcommand. @@ -43,3 +45,9 @@ func NewCommandBuilder(cmd flag.Applier, flags ...flag.Applier) (*safecli.Builde } return b, nil } + +// NewKopiaCommandBuilder returns a new Kopia command builder. +func NewKopiaCommandBuilder(args cli.CommonArgs, flags ...flag.Applier) (*safecli.Builder, error) { + flags = append([]flag.Applier{flagcommon.Common(args)}, flags...) + return NewCommandBuilder(KopiaBinaryName, flags...) +} diff --git a/pkg/kopia/cli/internal/command/command_test.go b/pkg/kopia/cli/internal/command/command_test.go index f4ae074d37..70e6a34981 100644 --- a/pkg/kopia/cli/internal/command/command_test.go +++ b/pkg/kopia/cli/internal/command/command_test.go @@ -95,3 +95,14 @@ func (s *CommandSuite) TestNewCommandBuilder(c *check.C) { "--flag2", }) } + +func (s *CommandSuite) TestNewKopiaCommandBuilder(c *check.C) { + b, err := NewKopiaCommandBuilder(cli.CommonArgs{}, &mockCommandAndFlag{flagName: "--flag1"}, &mockCommandAndFlag{flagName: "--flag2"}) + c.Assert(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{ + "kopia", + "--log-level=error", + "--flag1", + "--flag2", + }) +} diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index ec4718dbf5..73052ac336 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -1,5 +1,30 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command +// KopiaBinaryName is the name of the Kopia binary. +var ( + KopiaBinaryName = Command{"kopia"} +) + +// Repository commands. +var ( + Repository = Command{"repository"} + Create = Command{"create"} +) + // Repository storage sub commands. var ( FileSystem = Command{"filesystem"} diff --git a/pkg/kopia/cli/internal/flag/repository/repository_flags.go b/pkg/kopia/cli/internal/flag/repository/repository_flags.go new file mode 100644 index 0000000000..bd586781cd --- /dev/null +++ b/pkg/kopia/cli/internal/flag/repository/repository_flags.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "time" + + "github.com/go-openapi/strfmt" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// Hostname creates a new host flag with a given hostname. +func Hostname(hostname string) flag.Applier { + return flag.NewStringFlag("--override-hostname", hostname) +} + +// Username creates a new username flag with a given username. +func Username(username string) flag.Applier { + return flag.NewStringFlag("--override-username", username) +} + +// BlobRetention creates a new blob retention flag with a given mode and period. +// If mode is empty, the flag will be a no-op. +func BlobRetention(mode string, period time.Duration) flag.Applier { + if mode == "" { + return flag.EmptyFlag() + } + return flag.NewFlags( + flag.NewStringFlag("--retention-mode", mode), + flag.NewStringFlag("--retention-period", period.String()), + ) +} + +// PIT creates a new point-in-time flag with a given point-in-time. +// If pit is zero, the flag will be a no-op. +func PIT(pit strfmt.DateTime) flag.Applier { + dt := strfmt.DateTime(pit) + if time.Time(dt).IsZero() { + return flag.EmptyFlag() + } + return flag.NewStringFlag("--point-in-time", dt.String()) +} + +// ServerURL creates a new server URL flag with a given server URL. +func ServerURL(serverURL string) flag.Applier { + return flag.NewStringFlag("--url", serverURL) +} + +// ServerCertFingerprint creates a new server certificate fingerprint flag with a given fingerprint. +func ServerCertFingerprint(fingerprint string) flag.Applier { + return flag.NewRedactedStringFlag("--server-cert-fingerprint", fingerprint) +} diff --git a/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go b/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go new file mode 100644 index 0000000000..cb50443d85 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go @@ -0,0 +1,107 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "testing" + "time" + + "github.com/go-openapi/strfmt" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestRepositoryFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Hostname should not generate a flag", + Flag: Hostname(""), + }, + { + Name: "Hostname with value should generate a flag with given value", + Flag: Hostname("hostname"), + ExpectedCLI: []string{ + "--override-hostname=hostname", + }, + }, + { + Name: "Empty Username should not generate a flag", + Flag: Username(""), + }, + { + Name: "Username with value should generate a flag with given value", + Flag: Username("username"), + ExpectedCLI: []string{ + "--override-username=username", + }, + }, + { + Name: "Empty BlobRetention should not generate a flag", + Flag: BlobRetention("", time.Duration(0)), + }, + { + Name: "BlobRetention with values should generate multiple flags with given values", + Flag: BlobRetention("mode", 24*time.Hour), + ExpectedCLI: []string{ + "--retention-mode=mode", + "--retention-period=24h0m0s", + }, + }, + { + Name: "BlobRetention with RetentionMode only should generate mode flag with zero period", + Flag: BlobRetention("mode", 0), + ExpectedCLI: []string{ + "--retention-mode=mode", + "--retention-period=0s", + }, + }, + { + Name: "Empty PIT should not generate a flag", + Flag: PIT(strfmt.DateTime{}), + }, + { + Name: "PIT with value should generate a flag with given value", + Flag: PIT(func() strfmt.DateTime { + dt, _ := strfmt.ParseDateTime("2024-01-02T03:04:05.678Z") + return dt + }()), + ExpectedCLI: []string{ + "--point-in-time=2024-01-02T03:04:05.678Z", + }, + }, + { + Name: "Empty ServerURL should not generate a flag", + Flag: ServerURL(""), + }, + { + Name: "ServerURL with value should generate a flag with given value", + Flag: ServerURL("ServerURL"), + ExpectedCLI: []string{ + "--url=ServerURL", + }, + }, + { + Name: "Empty ServerCertFingerprint should not generate a flag", + Flag: ServerCertFingerprint(""), + }, + { + Name: "ServerCertFingerprint with value should generate a flag with given value and redact fingerprint for logs", + Flag: ServerCertFingerprint("ServerCertFingerprint"), + ExpectedCLI: []string{ + "--server-cert-fingerprint=ServerCertFingerprint", + }, + }, +})) diff --git a/pkg/kopia/cli/repository/repository_create.go b/pkg/kopia/cli/repository/repository_create.go new file mode 100644 index 0000000000..90291380c2 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_create.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "time" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" + flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" + flagstorage "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage" +) + +// CreateArgs defines the arguments for the `kopia repository create` command. +type CreateArgs struct { + cli.CommonArgs + cli.CacheArgs + + Hostname string // the hostname of the repository + Username string // the username of the repository + Location map[string][]byte // the location of the repository + RepoPathPrefix string // the prefix of the repository path + RetentionMode string // retention mode for supported storage backends + RetentionPeriod time.Duration // retention period for supported storage backends + + Logger log.Logger +} + +// Create creates a new `kopia repository create ...` command. +func Create(args CreateArgs) (safecli.CommandBuilder, error) { + return command.NewKopiaCommandBuilder(args.CommonArgs, + command.Repository, command.Create, + flagcommon.NoCheckForUpdates, + flagcommon.Cache(args.CacheArgs), + flagrepo.Hostname(args.Hostname), + flagrepo.Username(args.Username), + flagrepo.BlobRetention(args.RetentionMode, args.RetentionPeriod), + flagstorage.Storage( + args.Location, + args.RepoPathPrefix, + flagstorage.WithLogger(args.Logger), // log.Debug for old output + ), + ) +} diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go new file mode 100644 index 0000000000..bd9ad2b720 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_test.go @@ -0,0 +1,275 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "testing" + "time" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestRepositoryCommands(t *testing.T) { check.TestingT(t) } + +var ( + cacheArgs = cli.CacheArgs{ + CacheDirectory: "/tmp/cache.dir", + ContentCacheSizeLimitMB: 0, + MetadataCacheSizeLimitMB: 0, + } + + retentionMode = "Locked" + retentionPeriod = 15 * time.Minute + + locFS = model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte("test-prefix"), + } + + locAzure = model.Location{ + rs.TypeKey: []byte("azure"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + } + + locGCS = model.Location{ + rs.TypeKey: []byte("gcs"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + } + + locS3 = model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("test-endpoint"), + rs.RegionKey: []byte("test-region"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + rs.SkipSSLVerifyKey: []byte("false"), + } + + locS3Compliant = model.Location{ + rs.TypeKey: []byte("s3Compliant"), + rs.EndpointKey: []byte("test-endpoint"), + rs.RegionKey: []byte("test-region"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + rs.SkipSSLVerifyKey: []byte("false"), + } +) + +// Test Repository Create command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository create with no storage", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + } + return Create(args) + }, + ExpectedErr: cli.ErrUnsupportedStorage, + }, + { + Name: "repository create with filestore location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locFS, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "filesystem", + "--path=/mnt/data/test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with azure location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locAzure, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "azure", + "--container=test-bucket", + "--prefix=test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with gcs location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locGCS, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "gcs", + "--bucket=test-bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with s3 location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locS3, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "s3", + "--region=test-region", + "--bucket=test-bucket", + "--endpoint=test-endpoint", + "--prefix=test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with s3 compliant location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locS3Compliant, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "s3", + "--region=test-region", + "--bucket=test-bucket", + "--endpoint=test-endpoint", + "--prefix=test-prefix/test-path/prefix/", + }, + }, +}))