Add ability to pass estimation type, when building SnapshotCreate c…

…ommand (#3186) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Daniil Fedotov <[email protected]>
kanisterio · Oct 23, 2024 · 793523b · 793523b
1 parent 81a6355
commit 793523b
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 30 deletions.
diff --git a/pkg/kopia/command/const.go b/pkg/kopia/command/const.go
@@ -33,36 +33,39 @@ const (
 	statsSubCommand       = "stats"
 	verifySubCommand      = "verify"
 
-	allFlag                    = "--all"
-	configFileFlag             = "--config-file"
-	deleteFlag                 = "--delete"
-	deltaFlag                  = "--delta"
-	directoryIDFlag            = "--directory-id"
-	fileIDFlag                 = "--file-id"
-	fileParallelismFlag        = "--file-parallelism"
-	fileQueueLengthFlag        = "--file-queue-length"
-	filterFlag                 = "--filter"
-	globalFlag                 = "--global"
-	jsonFlag                   = "--json"
-	logDirectoryFlag           = "--log-dir"
-	logLevelFlag               = "--log-level"
-	fileLogLevelFlag           = "--file-log-level"
-	LogLevelError              = "error"
-	LogLevelInfo               = "info"
-	maxErrorsFlag              = "--max-errors"
-	parallelFlag               = "--parallel"
-	passwordFlag               = "--password"
-	progressUpdateIntervalFlag = "--progress-update-interval"
-	rawFlag                    = "--raw"
-	showIdenticalFlag          = "--show-identical"
-	sourcesFlag                = "--sources"
-	tagsFlag                   = "--tags"
-	unsafeIgnoreSourceFlag     = "--unsafe-ignore-source"
-	ownerFlag                  = "--owner"
-	sparseFlag                 = "--write-sparse-files"
-	verifyFilesPercentFlag     = "--verify-files-percent"
-	ignorePermissionsError     = "--ignore-permission-errors"
-	noIgnorePermissionsError   = "--no-ignore-permission-errors"
+	allFlag                         = "--all"
+	configFileFlag                  = "--config-file"
+	deleteFlag                      = "--delete"
+	deltaFlag                       = "--delta"
+	directoryIDFlag                 = "--directory-id"
+	fileIDFlag                      = "--file-id"
+	fileParallelismFlag             = "--file-parallelism"
+	fileQueueLengthFlag             = "--file-queue-length"
+	filterFlag                      = "--filter"
+	globalFlag                      = "--global"
+	jsonFlag                        = "--json"
+	logDirectoryFlag                = "--log-dir"
+	logLevelFlag                    = "--log-level"
+	fileLogLevelFlag                = "--file-log-level"
+	LogLevelError                   = "error"
+	LogLevelInfo                    = "info"
+	maxErrorsFlag                   = "--max-errors"
+	parallelFlag                    = "--parallel"
+	progressEstimationTypeFlag      = "--progress-estimation-type"
+	adaptiveEstimationThresholdFlag = "--adaptive-estimation-threshold"
+	progressEstimationTypeAdaptive  = "adaptive"
+	passwordFlag                    = "--password"
+	progressUpdateIntervalFlag      = "--progress-update-interval"
+	rawFlag                         = "--raw"
+	showIdenticalFlag               = "--show-identical"
+	sourcesFlag                     = "--sources"
+	tagsFlag                        = "--tags"
+	unsafeIgnoreSourceFlag          = "--unsafe-ignore-source"
+	ownerFlag                       = "--owner"
+	sparseFlag                      = "--write-sparse-files"
+	verifyFilesPercentFlag          = "--verify-files-percent"
+	ignorePermissionsError          = "--ignore-permission-errors"
+	noIgnorePermissionsError        = "--no-ignore-permission-errors"
 
 	// Server specific
 	addSubCommand             = "add"
@@ -120,6 +123,9 @@ const (
 
 // Constants for kopia defaults
 const (
+	// defaultAdaptiveEstimationThreshold is a default value of adaptive estimation threshold.
+	defaultAdaptiveEstimationThreshold = 300000
+
 	// DefaultCacheDirectory is the directory where kopia content cache is created
 	DefaultCacheDirectory = "/tmp/kopia-cache"
 

diff --git a/pkg/kopia/command/snapshot.go b/pkg/kopia/command/snapshot.go
@@ -32,6 +32,8 @@ type SnapshotCreateCommandArgs struct {
 	Tags                   []string
 	ProgressUpdateInterval time.Duration
 	Parallelism            int
+	EstimationType         string
+	EstimationThreshold    int
 }
 
 // SnapshotCreate returns the kopia command for creation of a snapshot
@@ -48,6 +50,18 @@ func SnapshotCreate(cmdArgs SnapshotCreateCommandArgs) []string {
 	args := commonArgs(cmdArgs.CommandArgs)
 	args = args.AppendLoggable(snapshotSubCommand, createSubCommand, cmdArgs.PathToBackup, jsonFlag)
 	args = args.AppendLoggableKV(parallelFlag, parallelismStr)
+
+	if cmdArgs.EstimationType != "" {
+		args = args.AppendLoggableKV(progressEstimationTypeFlag, cmdArgs.EstimationType)
+		if cmdArgs.EstimationType == progressEstimationTypeAdaptive {
+			threshold := cmdArgs.EstimationThreshold
+			if threshold == 0 {
+				threshold = defaultAdaptiveEstimationThreshold
+			}
+			thresholdStr := strconv.Itoa(threshold)
+			args = args.AppendLoggableKV(adaptiveEstimationThresholdFlag, thresholdStr)
+		}
+	}
 	args = addTags(cmdArgs.Tags, args)
 
 	// kube.Exec might timeout after 4h if there is no output from the command

diff --git a/pkg/kopia/command/snapshot_test.go b/pkg/kopia/command/snapshot_test.go
@@ -60,6 +60,43 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *check.C) {
 			},
 			expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-update-interval=2m",
 		},
+		{
+			f: func() []string {
+				args := SnapshotCreateCommandArgs{
+					CommandArgs:    commandArgs,
+					PathToBackup:   "path/to/backup",
+					Parallelism:    8,
+					EstimationType: "rough",
+				}
+				return SnapshotCreate(args)
+			},
+			expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-estimation-type=rough --progress-update-interval=1h",
+		},
+		{
+			f: func() []string {
+				args := SnapshotCreateCommandArgs{
+					CommandArgs:    commandArgs,
+					PathToBackup:   "path/to/backup",
+					Parallelism:    8,
+					EstimationType: "adaptive",
+				}
+				return SnapshotCreate(args)
+			},
+			expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-estimation-type=adaptive --adaptive-estimation-threshold=300000 --progress-update-interval=1h",
+		},
+		{
+			f: func() []string {
+				args := SnapshotCreateCommandArgs{
+					CommandArgs:         commandArgs,
+					PathToBackup:        "path/to/backup",
+					Parallelism:         8,
+					EstimationType:      "adaptive",
+					EstimationThreshold: 100,
+				}
+				return SnapshotCreate(args)
+			},
+			expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-estimation-type=adaptive --adaptive-estimation-threshold=100 --progress-update-interval=1h",
+		},
 		{
 			f: func() []string {
 				args := SnapshotExpireCommandArgs{