Skip to content

Commit

Permalink
pkg/kopia
Browse files Browse the repository at this point in the history
  • Loading branch information
e-sumin committed Mar 21, 2024
1 parent b18e381 commit 32661a3
Show file tree
Hide file tree
Showing 13 changed files with 108 additions and 159 deletions.
43 changes: 21 additions & 22 deletions pkg/kopia/command/parse_command_output.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,12 @@ import (
"strings"

"github.com/dustin/go-humanize"
"github.com/kanisterio/errkit"
"github.com/kanisterio/kanister/pkg/field"
"github.com/kanisterio/kanister/pkg/log"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/pkg/errors"

"github.com/kanisterio/kanister/pkg/field"
"github.com/kanisterio/kanister/pkg/log"
)

const (
Expand All @@ -47,7 +46,7 @@ const (
// SnapshotIDsFromSnapshot extracts root ID of a snapshot from the logs
func SnapshotIDsFromSnapshot(output string) (snapID, rootID string, err error) {
if output == "" {
return snapID, rootID, errors.New("Received empty output")
return snapID, rootID, errkit.New("Received empty output")
}

logs := regexp.MustCompile("[\r\n]").Split(output, -1)
Expand All @@ -61,7 +60,7 @@ func SnapshotIDsFromSnapshot(output string) (snapID, rootID string, err error) {
return snapID, rootID, nil
}
}
return snapID, rootID, errors.New("Failed to find Root ID from output")
return snapID, rootID, errkit.New("Failed to find Root ID from output")
}

// LatestSnapshotInfoFromManifestList returns snapshot ID and backup path of the latest snapshot from `manifests list` output
Expand All @@ -72,7 +71,7 @@ func LatestSnapshotInfoFromManifestList(output string) (string, string, error) {

err := json.Unmarshal([]byte(output), &manifestList)
if err != nil {
return snapID, backupPath, errors.Wrap(err, "Failed to unmarshal manifest list")
return snapID, backupPath, errkit.Wrap(err, "Failed to unmarshal manifest list")
}
for _, manifest := range manifestList {
for key, value := range manifest.Labels {
Expand All @@ -85,10 +84,10 @@ func LatestSnapshotInfoFromManifestList(output string) (string, string, error) {
}
}
if snapID == "" {
return "", "", errors.New("Failed to get latest snapshot ID from manifest list")
return "", "", errkit.New("Failed to get latest snapshot ID from manifest list")
}
if backupPath == "" {
return "", "", errors.New("Failed to get latest snapshot backup path from manifest list")
return "", "", errkit.New("Failed to get latest snapshot backup path from manifest list")
}
return snapID, backupPath, nil
}
Expand All @@ -108,15 +107,15 @@ func SnapshotInfoFromSnapshotCreateOutput(output string) (string, string, error)
if snapManifest.RootEntry != nil {
rootID = snapManifest.RootEntry.ObjectID.String()
if snapManifest.RootEntry.DirSummary != nil && snapManifest.RootEntry.DirSummary.FatalErrorCount > 0 {
return "", "", errors.New(fmt.Sprintf("Error occurred during snapshot creation. Output %s", output))
return "", "", errkit.New(fmt.Sprintf("Error occurred during snapshot creation. Output %s", output))
}
}
}
if snapID == "" {
return "", "", errors.New(fmt.Sprintf("Failed to get snapshot ID from create snapshot output %s", output))
return "", "", errkit.New(fmt.Sprintf("Failed to get snapshot ID from create snapshot output %s", output))
}
if rootID == "" {
return "", "", errors.New(fmt.Sprintf("Failed to get root ID from create snapshot output %s", output))
return "", "", errkit.New(fmt.Sprintf("Failed to get root ID from create snapshot output %s", output))
}
return snapID, rootID, nil
}
Expand All @@ -125,12 +124,12 @@ func SnapshotInfoFromSnapshotCreateOutput(output string) (string, string, error)
// is formatted as the output of a kopia snapshot list --all command.
func SnapSizeStatsFromSnapListAll(output string) (totalSizeB int64, numSnapshots int, err error) {
if output == "" {
return 0, 0, errors.New("Received empty output")
return 0, 0, errkit.New("Received empty output")
}

snapList, err := ParseSnapshotManifestList(output)
if err != nil {
return 0, 0, errors.Wrap(err, "Parsing snapshot list output as snapshot manifest list")
return 0, 0, errkit.Wrap(err, "Parsing snapshot list output as snapshot manifest list")
}

totalSizeB = sumSnapshotSizes(snapList)
Expand Down Expand Up @@ -162,7 +161,7 @@ func ParseSnapshotManifestList(output string) ([]*snapshot.Manifest, error) {
snapInfoList := []*snapshot.Manifest{}

if err := json.Unmarshal([]byte(output), &snapInfoList); err != nil {
return nil, errors.Wrap(err, "Failed to unmarshal snapshot manifest list")
return nil, errkit.Wrap(err, "Failed to unmarshal snapshot manifest list")
}

return snapInfoList, nil
Expand Down Expand Up @@ -332,7 +331,7 @@ func parseKopiaProgressLine(line string, matchOnlyFinished bool) (stats *Snapsho
// size in bytes or an error if parsing is unsuccessful.
func RepoSizeStatsFromBlobStatsRaw(blobStats string) (phySizeTotal int64, blobCount int, err error) {
if blobStats == "" {
return phySizeTotal, blobCount, errors.New("received empty blob stats string")
return phySizeTotal, blobCount, errkit.New("received empty blob stats string")
}

sizePattern := regexp.MustCompile(repoTotalSizeFromBlobStatsRegEx)
Expand Down Expand Up @@ -362,21 +361,21 @@ func RepoSizeStatsFromBlobStatsRaw(blobStats string) (phySizeTotal int64, blobCo
}

if countStr == "" {
return phySizeTotal, blobCount, errors.New("could not find count field in the blob stats")
return phySizeTotal, blobCount, errkit.New("could not find count field in the blob stats")
}

if sizeStr == "" {
return phySizeTotal, blobCount, errors.New("could not find size field in the blob stats")
return phySizeTotal, blobCount, errkit.New("could not find size field in the blob stats")
}

countVal, err := strconv.Atoi(countStr)
if err != nil {
return phySizeTotal, blobCount, errors.Wrap(err, fmt.Sprintf("unable to convert parsed count value %s", countStr))
return phySizeTotal, blobCount, errkit.Wrap(err, fmt.Sprintf("unable to convert parsed count value %s", countStr))
}

sizeValBytes, err := strconv.Atoi(sizeStr)
if err != nil {
return phySizeTotal, blobCount, errors.Wrap(err, fmt.Sprintf("unable to convert parsed size value %s", countStr))
return phySizeTotal, blobCount, errkit.Wrap(err, fmt.Sprintf("unable to convert parsed size value %s", countStr))
}

return int64(sizeValBytes), countVal, nil
Expand Down Expand Up @@ -411,7 +410,7 @@ func ErrorsFromOutput(output string) []error {
clean := ANSIEscapeCode.ReplaceAllString(l, "") // Strip all ANSI escape codes from line
match := kopiaErrorPattern.FindAllStringSubmatch(clean, 1)
if len(match) > 0 {
err = append(err, errors.New(match[0][1]))
err = append(err, errkit.New(match[0][1]))
}
}

Expand All @@ -423,7 +422,7 @@ func ParsePolicyShow(output string) (policy.Policy, error) {
policy := policy.Policy{}

if err := json.Unmarshal([]byte(output), &policy); err != nil {
return policy, errors.Wrap(err, "Failed to unmarshal snapshot manifest list")
return policy, errkit.Wrap(err, "Failed to unmarshal snapshot manifest list")
}

return policy, nil
Expand Down
7 changes: 3 additions & 4 deletions pkg/kopia/command/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ import (
"time"

"github.com/go-openapi/strfmt"
"github.com/pkg/errors"

"github.com/kanisterio/errkit"
"github.com/kanisterio/kanister/pkg/kopia/command/storage"
)

Expand Down Expand Up @@ -67,7 +66,7 @@ func RepositoryConnectCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
RepoPathPrefix: cmdArgs.RepoPathPrefix,
})
if err != nil {
return nil, errors.Wrap(err, "Failed to generate storage args")
return nil, errkit.Wrap(err, "Failed to generate storage args")
}

if !time.Time(cmdArgs.PITFlag).IsZero() {
Expand Down Expand Up @@ -103,7 +102,7 @@ func RepositoryCreateCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
RepoPathPrefix: cmdArgs.RepoPathPrefix,
})
if err != nil {
return nil, errors.Wrap(err, "Failed to generate storage args")
return nil, errkit.Wrap(err, "Failed to generate storage args")
}

return stringSliceCommand(args.Combine(bsArgs)), nil
Expand Down
6 changes: 3 additions & 3 deletions pkg/kopia/command/storage/secret_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ import (
"context"
"time"

"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"

"github.com/kanisterio/errkit"
crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
"github.com/kanisterio/kanister/pkg/aws"
"github.com/kanisterio/kanister/pkg/blockstorage/azure"
Expand Down Expand Up @@ -64,7 +64,7 @@ func locationType(m map[string][]byte) repositoryserver.LocType {
// list of EnvVar based on secret type
func GenerateEnvSpecFromCredentialSecret(s *corev1.Secret, assumeRoleDurationS3 time.Duration) ([]corev1.EnvVar, error) {
if s == nil {
return nil, errors.New("Secret cannot be nil")
return nil, errkit.New("Secret cannot be nil")
}
secType := string(s.Type)
switch secType {
Expand Down Expand Up @@ -111,7 +111,7 @@ func getEnvSpecForAzureCredentialSecret(s *corev1.Secret) ([]corev1.EnvVar, erro
if storageEnv != "" {
env, err := azure.EnvironmentFromName(storageEnv)
if err != nil {
return nil, errors.Wrapf(err, "Failed to get azure environment from name: %s", storageEnv)
return nil, errkit.Wrap(err, "Failed to get azure environment from name", "storageEnv", storageEnv)
}
blobDomain := "blob." + env.StorageEndpointSuffix
// TODO : Check how we can set this env to use value from secret
Expand Down
5 changes: 2 additions & 3 deletions pkg/kopia/command/storage/storage_args.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@
package storage

import (
"fmt"

"github.com/kanisterio/errkit"
"github.com/kanisterio/kanister/pkg/logsafe"
"github.com/kanisterio/kanister/pkg/secrets/repositoryserver"
)
Expand Down Expand Up @@ -47,6 +46,6 @@ func KopiaStorageArgs(params *StorageCommandParams) (logsafe.Cmd, error) {
case repositoryserver.LocTypeAzure:
return azureArgs(params.Location, params.RepoPathPrefix), nil
default:
return nil, fmt.Errorf("unsupported type for the location: %s", LocType)
return nil, errkit.New("unsupported type for the location", "locationType", LocType)
}
}
49 changes: 2 additions & 47 deletions pkg/kopia/errors/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,62 +15,17 @@
package errors

import (
"bytes"
"regexp"
"strings"

"github.com/pkg/errors"
"github.com/kanisterio/errkit"
)

type errorList []error

var _ error = errorList{}

func (e errorList) String() string {
sep := ""
var buf bytes.Buffer
buf.WriteRune('[')
for _, err := range e {
buf.WriteString(sep)
sep = ","
buf.WriteRune('"')
buf.WriteString(err.Error())
buf.WriteRune('"')
}
buf.WriteRune(']')
return buf.String()
}

func (e errorList) Error() string {
return e.String()
}

// Append creates a new combined error from err1, err2. If either error is nil,
// then the other error is returned.
func Append(err1, err2 error) error {
if err1 == nil {
return err2
}
if err2 == nil {
return err1
}
el1, ok1 := err1.(errorList)
el2, ok2 := err2.(errorList)
switch {
case ok1 && ok2:
return append(el1, el2...)
case ok1:
return append(el1, err2)
case ok2:
return append(el2, err1)
}
return errorList{err1, err2}
}

// FirstMatching returns the first error that matches the predicate in a
// causal dependency err->Cause()->Cause() ....
func FirstMatching(err error, predicate func(error) bool) error {
for ; err != nil; err = errors.Unwrap(err) {
for ; err != nil; err = errkit.Unwrap(err) {
if predicate(err) {
return err
}
Expand Down
5 changes: 2 additions & 3 deletions pkg/kopia/maintenance/get_maintenance_owner.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"

"github.com/kanisterio/errkit"
kopiacli "github.com/kopia/kopia/cli"
"github.com/kopia/kopia/repo/manifest"
"github.com/pkg/errors"

"github.com/kanisterio/kanister/pkg/format"
"github.com/kanisterio/kanister/pkg/kopia/command"
Expand Down Expand Up @@ -76,7 +75,7 @@ func GetMaintenanceOwnerForConnectedRepository(
func parseOwner(output []byte) (string, error) {
maintInfo := kopiacli.MaintenanceInfo{}
if err := json.Unmarshal(output, &maintInfo); err != nil {
return "", errors.New(fmt.Sprintf("failed to unmarshal maintenance info output: %v", err))
return "", errkit.Wrap(err, "failed to unmarshal maintenance info output")
}
return maintInfo.Owner, nil
}
19 changes: 9 additions & 10 deletions pkg/kopia/repository/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,12 @@ import (
"time"

"github.com/jpillora/backoff"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/content"
"github.com/pkg/errors"

"github.com/kanisterio/errkit"
"github.com/kanisterio/kanister/pkg/kopia"
"github.com/kanisterio/kanister/pkg/log"
"github.com/kanisterio/kanister/pkg/poll"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/content"
)

const (
Expand All @@ -50,7 +49,7 @@ func ConnectToAPIServer(
// Extra fingerprint from the TLS Certificate secret
fingerprint, err := kopia.ExtractFingerprintFromCertificate(tlsCert)
if err != nil {
return errors.Wrap(err, "Failed to extract fingerprint from Kopia API Server Certificate Secret")
return errkit.Wrap(err, "Failed to extract fingerprint from Kopia API Server Certificate Secret")
}

serverInfo := &repo.APIServerInfo{
Expand Down Expand Up @@ -92,7 +91,7 @@ func ConnectToAPIServer(
}
return true, nil
})
return errors.Wrap(err, "Failed connecting to the Kopia API Server")
return errkit.Wrap(err, "Failed connecting to the Kopia API Server")
}

// Open connects to the kopia repository based on the config stored in the config file
Expand All @@ -101,24 +100,24 @@ func ConnectToAPIServer(
func Open(ctx context.Context, configFile, password, purpose string) (repo.RepositoryWriter, error) {
repoConfig := repositoryConfigFileName(configFile)
if _, err := os.Stat(repoConfig); os.IsNotExist(err) {
return nil, errors.New("Failed find kopia configuration file")
return nil, errkit.New("Failed find kopia configuration file")
}

r, err := repo.Open(ctx, repoConfig, password, &repo.Options{})
if os.IsNotExist(err) {
return nil, errors.New("Failed to find kopia repository, use `kopia repository create` or kopia repository connect` if already created")
return nil, errkit.New("Failed to find kopia repository, use `kopia repository create` or kopia repository connect` if already created")
}

if err != nil {
return nil, errors.Wrap(err, "Failed to open kopia repository")
return nil, errkit.Wrap(err, "Failed to open kopia repository")
}

_, rw, err := r.NewWriter(ctx, repo.WriteSessionOptions{
Purpose: purpose,
OnUpload: func(i int64) {},
})

return rw, errors.Wrap(err, "Failed to open kopia repository writer")
return rw, errkit.Wrap(err, "Failed to open kopia repository writer")
}

func repositoryConfigFileName(configFile string) string {
Expand Down
Loading

0 comments on commit 32661a3

Please sign in to comment.