From 793523b6a886f326af1c4e6040898ae2a10e9390 Mon Sep 17 00:00:00 2001 From: Eugene Sumin <95425330+e-sumin@users.noreply.github.com> Date: Wed, 23 Oct 2024 22:17:58 +0200 Subject: [PATCH] Add ability to pass estimation type, when building `SnapshotCreate` command (#3186) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Daniil Fedotov --- pkg/kopia/command/const.go | 66 ++++++++++++++++-------------- pkg/kopia/command/snapshot.go | 14 +++++++ pkg/kopia/command/snapshot_test.go | 37 +++++++++++++++++ 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/pkg/kopia/command/const.go b/pkg/kopia/command/const.go index b3532bdc82..c45b61b36f 100644 --- a/pkg/kopia/command/const.go +++ b/pkg/kopia/command/const.go @@ -33,36 +33,39 @@ const ( statsSubCommand = "stats" verifySubCommand = "verify" - allFlag = "--all" - configFileFlag = "--config-file" - deleteFlag = "--delete" - deltaFlag = "--delta" - directoryIDFlag = "--directory-id" - fileIDFlag = "--file-id" - fileParallelismFlag = "--file-parallelism" - fileQueueLengthFlag = "--file-queue-length" - filterFlag = "--filter" - globalFlag = "--global" - jsonFlag = "--json" - logDirectoryFlag = "--log-dir" - logLevelFlag = "--log-level" - fileLogLevelFlag = "--file-log-level" - LogLevelError = "error" - LogLevelInfo = "info" - maxErrorsFlag = "--max-errors" - parallelFlag = "--parallel" - passwordFlag = "--password" - progressUpdateIntervalFlag = "--progress-update-interval" - rawFlag = "--raw" - showIdenticalFlag = "--show-identical" - sourcesFlag = "--sources" - tagsFlag = "--tags" - unsafeIgnoreSourceFlag = "--unsafe-ignore-source" - ownerFlag = "--owner" - sparseFlag = "--write-sparse-files" - verifyFilesPercentFlag = "--verify-files-percent" - ignorePermissionsError = "--ignore-permission-errors" - noIgnorePermissionsError = "--no-ignore-permission-errors" + allFlag = "--all" + configFileFlag = "--config-file" + deleteFlag = "--delete" + deltaFlag = "--delta" + directoryIDFlag = "--directory-id" + fileIDFlag = "--file-id" + fileParallelismFlag = "--file-parallelism" + fileQueueLengthFlag = "--file-queue-length" + filterFlag = "--filter" + globalFlag = "--global" + jsonFlag = "--json" + logDirectoryFlag = "--log-dir" + logLevelFlag = "--log-level" + fileLogLevelFlag = "--file-log-level" + LogLevelError = "error" + LogLevelInfo = "info" + maxErrorsFlag = "--max-errors" + parallelFlag = "--parallel" + progressEstimationTypeFlag = "--progress-estimation-type" + adaptiveEstimationThresholdFlag = "--adaptive-estimation-threshold" + progressEstimationTypeAdaptive = "adaptive" + passwordFlag = "--password" + progressUpdateIntervalFlag = "--progress-update-interval" + rawFlag = "--raw" + showIdenticalFlag = "--show-identical" + sourcesFlag = "--sources" + tagsFlag = "--tags" + unsafeIgnoreSourceFlag = "--unsafe-ignore-source" + ownerFlag = "--owner" + sparseFlag = "--write-sparse-files" + verifyFilesPercentFlag = "--verify-files-percent" + ignorePermissionsError = "--ignore-permission-errors" + noIgnorePermissionsError = "--no-ignore-permission-errors" // Server specific addSubCommand = "add" @@ -120,6 +123,9 @@ const ( // Constants for kopia defaults const ( + // defaultAdaptiveEstimationThreshold is a default value of adaptive estimation threshold. + defaultAdaptiveEstimationThreshold = 300000 + // DefaultCacheDirectory is the directory where kopia content cache is created DefaultCacheDirectory = "/tmp/kopia-cache" diff --git a/pkg/kopia/command/snapshot.go b/pkg/kopia/command/snapshot.go index 2bd4f15cf1..1093e0bcde 100644 --- a/pkg/kopia/command/snapshot.go +++ b/pkg/kopia/command/snapshot.go @@ -32,6 +32,8 @@ type SnapshotCreateCommandArgs struct { Tags []string ProgressUpdateInterval time.Duration Parallelism int + EstimationType string + EstimationThreshold int } // SnapshotCreate returns the kopia command for creation of a snapshot @@ -48,6 +50,18 @@ func SnapshotCreate(cmdArgs SnapshotCreateCommandArgs) []string { args := commonArgs(cmdArgs.CommandArgs) args = args.AppendLoggable(snapshotSubCommand, createSubCommand, cmdArgs.PathToBackup, jsonFlag) args = args.AppendLoggableKV(parallelFlag, parallelismStr) + + if cmdArgs.EstimationType != "" { + args = args.AppendLoggableKV(progressEstimationTypeFlag, cmdArgs.EstimationType) + if cmdArgs.EstimationType == progressEstimationTypeAdaptive { + threshold := cmdArgs.EstimationThreshold + if threshold == 0 { + threshold = defaultAdaptiveEstimationThreshold + } + thresholdStr := strconv.Itoa(threshold) + args = args.AppendLoggableKV(adaptiveEstimationThresholdFlag, thresholdStr) + } + } args = addTags(cmdArgs.Tags, args) // kube.Exec might timeout after 4h if there is no output from the command diff --git a/pkg/kopia/command/snapshot_test.go b/pkg/kopia/command/snapshot_test.go index e87e98b762..6e255a9d81 100644 --- a/pkg/kopia/command/snapshot_test.go +++ b/pkg/kopia/command/snapshot_test.go @@ -60,6 +60,43 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *check.C) { }, expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-update-interval=2m", }, + { + f: func() []string { + args := SnapshotCreateCommandArgs{ + CommandArgs: commandArgs, + PathToBackup: "path/to/backup", + Parallelism: 8, + EstimationType: "rough", + } + return SnapshotCreate(args) + }, + expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-estimation-type=rough --progress-update-interval=1h", + }, + { + f: func() []string { + args := SnapshotCreateCommandArgs{ + CommandArgs: commandArgs, + PathToBackup: "path/to/backup", + Parallelism: 8, + EstimationType: "adaptive", + } + return SnapshotCreate(args) + }, + expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-estimation-type=adaptive --adaptive-estimation-threshold=300000 --progress-update-interval=1h", + }, + { + f: func() []string { + args := SnapshotCreateCommandArgs{ + CommandArgs: commandArgs, + PathToBackup: "path/to/backup", + Parallelism: 8, + EstimationType: "adaptive", + EstimationThreshold: 100, + } + return SnapshotCreate(args) + }, + expectedLog: "kopia --log-level=info --config-file=path/kopia.config --log-dir=cache/log --password=encr-key snapshot create path/to/backup --json --parallel=8 --progress-estimation-type=adaptive --adaptive-estimation-threshold=100 --progress-update-interval=1h", + }, { f: func() []string { args := SnapshotExpireCommandArgs{