From 2167aef84e57fb726b863cd4cc3d50ce0ff1a53c Mon Sep 17 00:00:00 2001 From: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> Date: Sun, 15 Oct 2023 12:47:10 -0700 Subject: [PATCH] Remove unused DescribeBackups function and related code (#2401) * Remove unused DescribeBackups function * Remove related tests * Remove related docs * Use a different constant --- docs/functions.rst | 51 ------- pkg/function/backup_data_stats.go | 2 +- pkg/function/data_test.go | 77 ----------- pkg/function/describe_backups.go | 223 ------------------------------ 4 files changed, 1 insertion(+), 352 deletions(-) delete mode 100644 pkg/function/describe_backups.go diff --git a/docs/functions.rst b/docs/functions.rst index 6b2cc2102f..84de34a90a 100644 --- a/docs/functions.rst +++ b/docs/functions.rst @@ -889,57 +889,6 @@ Example: mode: restore-size backupID: "{{ .ArtifactsIn.snapshot.KeyValue.backupIdentifier }}" -DescribeBackups ---------------- - -This function describes the backups for an object store location - -.. note:: - It is important that the application includes a ``kanister-tools`` - sidecar container. This sidecar is necessary to run the - tools that get the information from the object store. - -Arguments: - -.. csv-table:: - :header: "Argument", "Required", "Type", "Description" - :align: left - :widths: 5,5,5,15 - - `backupArtifactPrefix`, Yes, `string`, path to the object store location - `encryptionKey`, No, `string`, encryption key to be used for backups - -Outputs: - -.. csv-table:: - :header: "Output", "Type", "Description" - :align: left - :widths: 5,5,15 - - `fileCount`,`string`, number of files in backup object store location - `size`, `string`, size of the number of files in in backup object store location - `passwordIncorrect`, `string`, true if encryption key is incorrect - `repoDoesNotExist`, `string`, true if object store location does not exist - -Example: - -.. code-block:: yaml - :linenos: - - actions: - backupStats: - outputArtifacts: - backupStats: - keyValue: - fileCount: "{{ .Phases.DescribeBackupsFromObjectStore.Output.fileCount }}" - size: "{{ .Phases.DescribeBackupsFromObjectStore.Output.size }}" - passwordIncorrect: "{{ .Phases.DescribeBackupsFromObjectStore.Output.passwordIncorrect }}" - repoDoesNotExist: "{{ .Phases.DescribeBackupsFromObjectStore.Output.repoDoesNotExist }}" - phases: - - func: DescribeBackups - name: DescribeBackupsFromObjectStore - args: - backupArtifactPrefix: s3-bucket/path/artifactPrefix CreateRDSSnapshot ----------------- diff --git a/pkg/function/backup_data_stats.go b/pkg/function/backup_data_stats.go index 1c1b7dde67..60e5a2a14c 100644 --- a/pkg/function/backup_data_stats.go +++ b/pkg/function/backup_data_stats.go @@ -157,7 +157,7 @@ func (b *BackupDataStatsFunc) Exec(ctx context.Context, tp param.TemplateParams, if err = OptArg(args, BackupDataStatsEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { return nil, err } - podOverride, err := GetPodSpecOverride(tp, args, DescribeBackupsPodOverrideArg) + podOverride, err := GetPodSpecOverride(tp, args, CheckRepositoryPodOverrideArg) if err != nil { return nil, err } diff --git a/pkg/function/data_test.go b/pkg/function/data_test.go index 069a0898d2..63ca6f077e 100644 --- a/pkg/function/data_test.go +++ b/pkg/function/data_test.go @@ -175,26 +175,6 @@ func newBackupDataBlueprint() *crv1alpha1.Blueprint { } } -func newDescribeBackupsBlueprint() *crv1alpha1.Blueprint { - return &crv1alpha1.Blueprint{ - Actions: map[string]*crv1alpha1.BlueprintAction{ - "describeBackups": { - Kind: param.StatefulSetKind, - Phases: []crv1alpha1.BlueprintPhase{ - { - Name: "testDescribeBackups", - Func: DescribeBackupsFuncName, - Args: map[string]interface{}{ - DescribeBackupsArtifactPrefixArg: "{{ .Profile.Location.Prefix }}", - DescribeBackupsEncryptionKeyArg: "{{ .Secrets.backupKey.Data.password | toString }}", - }, - }, - }, - }, - }, - } -} - func newCheckRepositoryBlueprint() *crv1alpha1.Blueprint { return &crv1alpha1.Blueprint{ Actions: map[string]*crv1alpha1.BlueprintAction{ @@ -590,63 +570,6 @@ func (s *DataSuite) initPVCTemplateParams(c *C, pvc *v1.PersistentVolumeClaim, o tp.Profile = s.profile return tp } -func (s *DataSuite) TestDescribeBackups(c *C) { - tp, _ := s.getTemplateParamsAndPVCName(c, 1) - - // Test backup - bp := *newBackupDataBlueprint() - out := runAction(c, bp, "backup", tp) - c.Assert(out[BackupDataOutputBackupID].(string), Not(Equals), "") - c.Assert(out[BackupDataOutputBackupTag].(string), Not(Equals), "") - c.Assert(out[FunctionOutputVersion].(string), Equals, kanister.DefaultVersion) - - // Test DescribeBackups - bp2 := *newDescribeBackupsBlueprint() - out2 := runAction(c, bp2, "describeBackups", tp) - c.Assert(out2[DescribeBackupsFileCount].(string), Not(Equals), "") - c.Assert(out2[DescribeBackupsSize].(string), Not(Equals), "") - c.Assert(out2[DescribeBackupsPasswordIncorrect].(string), Not(Equals), "") - c.Assert(out2[DescribeBackupsRepoDoesNotExist].(string), Not(Equals), "") - c.Assert(out2[FunctionOutputVersion].(string), Equals, kanister.DefaultVersion) -} - -func (s *DataSuite) TestDescribeBackupsWrongPassword(c *C) { - tp, _ := s.getTemplateParamsAndPVCName(c, 1) - - // Test backup - bp := *newBackupDataBlueprint() - bp.Actions["backup"].Phases[0].Args[BackupDataBackupArtifactPrefixArg] = fmt.Sprintf("%s/%s", bp.Actions["backup"].Phases[0].Args[BackupDataBackupArtifactPrefixArg], "abcde") - bp.Actions["backup"].Phases[0].Args[BackupDataEncryptionKeyArg] = "foobar" - out := runAction(c, bp, "backup", tp) - c.Assert(out[BackupDataOutputBackupID].(string), Not(Equals), "") - c.Assert(out[BackupDataOutputBackupTag].(string), Not(Equals), "") - c.Assert(out[FunctionOutputVersion].(string), Equals, kanister.DefaultVersion) - - // Test DescribeBackups - bp2 := *newDescribeBackupsBlueprint() - bp2.Actions["describeBackups"].Phases[0].Args[DescribeBackupsArtifactPrefixArg] = fmt.Sprintf("%s/%s", bp2.Actions["describeBackups"].Phases[0].Args[DescribeBackupsArtifactPrefixArg], "abcde") - out2 := runAction(c, bp2, "describeBackups", tp) - c.Assert(out2[DescribeBackupsPasswordIncorrect].(string), Equals, "true") - c.Assert(out2[FunctionOutputVersion].(string), Equals, kanister.DefaultVersion) -} - -func (s *DataSuite) TestDescribeBackupsRepoNotAvailable(c *C) { - tp, _ := s.getTemplateParamsAndPVCName(c, 1) - - // Test backup - bp := *newBackupDataBlueprint() - out := runAction(c, bp, "backup", tp) - c.Assert(out[BackupDataOutputBackupID].(string), Not(Equals), "") - c.Assert(out[BackupDataOutputBackupTag].(string), Not(Equals), "") - c.Assert(out[FunctionOutputVersion].(string), Equals, kanister.DefaultVersion) - - // Test DescribeBackups - bp2 := *newDescribeBackupsBlueprint() - bp2.Actions["describeBackups"].Phases[0].Args[DescribeBackupsArtifactPrefixArg] = fmt.Sprintf("%s/%s", bp2.Actions["describeBackups"].Phases[0].Args[DescribeBackupsArtifactPrefixArg], c.TestName()) - out2 := runAction(c, bp2, "describeBackups", tp) - c.Assert(out2[DescribeBackupsRepoDoesNotExist].(string), Equals, "true") - c.Assert(out2[FunctionOutputVersion].(string), Equals, kanister.DefaultVersion) -} func (s *DataSuite) TestCheckRepository(c *C) { tp, _ := s.getTemplateParamsAndPVCName(c, 1) diff --git a/pkg/function/describe_backups.go b/pkg/function/describe_backups.go deleted file mode 100644 index 879610d7d6..0000000000 --- a/pkg/function/describe_backups.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2019 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package function - -import ( - "bytes" - "context" - "strings" - "time" - - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - kanister "github.com/kanisterio/kanister/pkg" - crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" - "github.com/kanisterio/kanister/pkg/consts" - "github.com/kanisterio/kanister/pkg/format" - "github.com/kanisterio/kanister/pkg/kube" - "github.com/kanisterio/kanister/pkg/param" - "github.com/kanisterio/kanister/pkg/progress" - "github.com/kanisterio/kanister/pkg/restic" -) - -const ( - // DescribeBackupsFuncName gives the function name - DescribeBackupsFuncName = "DescribeBackups" - // DescribeBackupsArtifactPrefixArg provides the path to restore backed up data - DescribeBackupsArtifactPrefixArg = "backupArtifactPrefix" - // DescribeBackupsEncryptionKeyArg provides the encryption key to be used for deletes - DescribeBackupsEncryptionKeyArg = "encryptionKey" - // DescribeBackupsPodOverrideArg contains pod specs to override default pod specs - DescribeBackupsPodOverrideArg = "podOverride" - DescribeBackupsJobPrefix = "describe-backups-" - DescribeBackupsFileCount = "fileCount" - DescribeBackupsSize = "size" - DescribeBackupsPasswordIncorrect = "passwordIncorrect" - DescribeBackupsRepoDoesNotExist = "repoUnavailable" - RawDataStatsMode = "raw-data" -) - -func init() { - _ = kanister.Register(&DescribeBackupsFunc{}) -} - -var _ kanister.Func = (*DescribeBackupsFunc)(nil) - -type DescribeBackupsFunc struct { - progressPercent string -} - -func (*DescribeBackupsFunc) Name() string { - return DescribeBackupsFuncName -} - -func describeBackups(ctx context.Context, cli kubernetes.Interface, tp param.TemplateParams, encryptionKey, targetPaths, jobPrefix string, podOverride crv1alpha1.JSONMap) (map[string]interface{}, error) { - namespace, err := kube.GetControllerNamespace() - if err != nil { - return nil, errors.Wrapf(err, "Failed to get controller namespace") - } - options := &kube.PodOptions{ - Namespace: namespace, - GenerateName: jobPrefix, - Image: consts.GetKanisterToolsImage(), - Command: []string{"sh", "-c", "tail -f /dev/null"}, - PodOverride: podOverride, - } - pr := kube.NewPodRunner(cli, options) - podFunc := describeBackupsPodFunc(cli, tp, encryptionKey, targetPaths) - return pr.Run(ctx, podFunc) -} - -func describeBackupsPodFunc( - cli kubernetes.Interface, - tp param.TemplateParams, - encryptionKey, - targetPath string, -) func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { - return func(ctx context.Context, pc kube.PodController) (map[string]interface{}, error) { - pod := pc.Pod() - - // Wait for pod to reach running state - if err := pc.WaitForPodReady(ctx); err != nil { - return nil, errors.Wrapf(err, "Failed while waiting for Pod %s to be ready", pod.Name) - } - - remover, err := MaybeWriteProfileCredentials(ctx, pc, tp.Profile) - if err != nil { - return nil, err - } - - // Parent context could already be dead, so removing file within new context - defer remover.Remove(context.Background()) //nolint:errcheck - - err = restic.CheckIfRepoIsReachable( - tp.Profile, - targetPath, - encryptionKey, - cli, pod.Namespace, - pod.Name, - pod.Spec.Containers[0].Name, - ) - switch { - case err == nil: - break - case strings.Contains(err.Error(), restic.PasswordIncorrect): - return map[string]interface{}{ - DescribeBackupsFileCount: nil, - DescribeBackupsSize: nil, - DescribeBackupsPasswordIncorrect: "true", - DescribeBackupsRepoDoesNotExist: "false", - FunctionOutputVersion: kanister.DefaultVersion, - }, - nil - - case strings.Contains(err.Error(), restic.RepoDoesNotExist): - return map[string]interface{}{ - DescribeBackupsFileCount: nil, - DescribeBackupsSize: nil, - DescribeBackupsPasswordIncorrect: "false", - DescribeBackupsRepoDoesNotExist: "true", - FunctionOutputVersion: kanister.DefaultVersion, - }, - nil - default: - return nil, err - } - - cmd, err := restic.StatsCommandByID(tp.Profile, targetPath, "" /* get all snapshot stats */, RawDataStatsMode, encryptionKey) - if err != nil { - return nil, err - } - - ex, err := pc.GetCommandExecutor() - if err != nil { - return nil, err - } - - var stdout, stderr bytes.Buffer - err = ex.Exec(ctx, cmd, nil, &stdout, &stderr) - format.LogWithCtx(ctx, pod.Name, pod.Spec.Containers[0].Name, stdout.String()) - format.LogWithCtx(ctx, pod.Name, pod.Spec.Containers[0].Name, stderr.String()) - if err != nil { - return nil, errors.Wrapf(err, "Failed to get backup stats") - } - - // Get File Count and Size from Stats - _, fc, size := restic.SnapshotStatsFromStatsLog(stdout.String()) - if fc == "" || size == "" { - return nil, errors.New("Failed to parse snapshot stats from logs") - } - return map[string]interface{}{ - DescribeBackupsFileCount: fc, - DescribeBackupsSize: size, - DescribeBackupsPasswordIncorrect: "false", - DescribeBackupsRepoDoesNotExist: "false", - FunctionOutputVersion: kanister.DefaultVersion, - }, - nil - } -} - -func (d *DescribeBackupsFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) { - // Set progress percent - d.progressPercent = progress.StartedPercent - defer func() { d.progressPercent = progress.CompletedPercent }() - - var describeBackupsArtifactPrefix, encryptionKey string - var err error - if err = Arg(args, DescribeBackupsArtifactPrefixArg, &describeBackupsArtifactPrefix); err != nil { - return nil, err - } - if err = OptArg(args, DescribeBackupsEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil { - return nil, err - } - podOverride, err := GetPodSpecOverride(tp, args, DescribeBackupsPodOverrideArg) - if err != nil { - return nil, err - } - - if err = ValidateProfile(tp.Profile); err != nil { - return nil, err - } - - describeBackupsArtifactPrefix = ResolveArtifactPrefix(describeBackupsArtifactPrefix, tp.Profile) - - cli, err := kube.NewClient() - if err != nil { - return nil, errors.Wrapf(err, "Failed to create Kubernetes client") - } - return describeBackups(ctx, cli, tp, encryptionKey, describeBackupsArtifactPrefix, DescribeBackupsJobPrefix, podOverride) -} - -func (*DescribeBackupsFunc) RequiredArgs() []string { - return []string{DescribeBackupsArtifactPrefixArg} -} - -func (*DescribeBackupsFunc) Arguments() []string { - return []string{ - DescribeBackupsArtifactPrefixArg, - DescribeBackupsEncryptionKeyArg, - } -} - -func (d *DescribeBackupsFunc) ExecutionProgress() (crv1alpha1.PhaseProgress, error) { - metav1Time := metav1.NewTime(time.Now()) - return crv1alpha1.PhaseProgress{ - ProgressPercent: d.progressPercent, - LastTransitionTime: &metav1Time, - }, nil -}