diff --git a/commands/cmd_pop.go b/commands/cmd_pop.go new file mode 100644 index 00000000..3d3466cf --- /dev/null +++ b/commands/cmd_pop.go @@ -0,0 +1,105 @@ +package commands + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/DanEngelbrecht/golongtail/longtailutils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func pop( + numWorkerCount int, + stackOffset uint32, + retainPermissions bool, + enableFileMapping bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) { + const fname = "get" + log := logrus.WithFields(logrus.Fields{ + "fname": fname, + "stackOffset": stackOffset, + "numWorkerCount": numWorkerCount, + "retainPermissions": retainPermissions, + "enableFileMapping": enableFileMapping, + }) + log.Debug(fname) + + storeStats := []longtailutils.StoreStat{} + timeStats := []longtailutils.TimeStat{} + + setupStartTime := time.Now() + + logFile := ".longtail/log" + targetFolderPath := "." + excludeFilterRegEx := "^\\.longtail(/|$)" + blobStoreURI := ".longtail/store" + + history := "" + _, err := os.Stat(logFile) + if err == nil { + historyBytes, err := ioutil.ReadFile(logFile) + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + history = string(historyBytes) + } else if !os.IsNotExist(err) { + return storeStats, timeStats, errors.Wrap(err, fname) + } + + historyLines := make([]string, 0) + if history != "" { + historyLines = strings.Split(string(history), "\n") + } + + if len(historyLines) < int(stackOffset+1) { + err = fmt.Errorf("stack offset out of bounds for `%s`, log contains `%d` entries, need `%d` entries", logFile, len(historyLines), int(stackOffset+1)) + return storeStats, timeStats, errors.Wrap(err, fname) + } + + sourceFilePath := historyLines[len(historyLines)-int(stackOffset+1)] + versionLocalStoreIndexPath := sourceFilePath[:len(sourceFilePath)-3] + "lsi" + + setupTime := time.Since(setupStartTime) + timeStats = append(timeStats, longtailutils.TimeStat{"Setup", setupTime}) + + downSyncStoreStats, downSyncTimeStats, err := downsync( + numWorkerCount, + blobStoreURI, + sourceFilePath, + targetFolderPath, + "", + "", + retainPermissions, + false, + versionLocalStoreIndexPath, + "", + excludeFilterRegEx, + true, + false, + enableFileMapping) + + storeStats = append(storeStats, downSyncStoreStats...) + timeStats = append(timeStats, downSyncTimeStats...) + + return storeStats, timeStats, errors.Wrap(err, fname) +} + +type PopCmd struct { + StackOffset uint32 `name:"offset" help:"Offset into log, zero equals top of stack (latest push)" default:"0"` + RetainPermissionsOption + EnableFileMappingOption +} + +func (r *PopCmd) Run(ctx *Context) error { + storeStats, timeStats, err := pop( + ctx.NumWorkerCount, + r.StackOffset, + r.RetainPermissions, + r.EnableFileMapping) + ctx.StoreStats = append(ctx.StoreStats, storeStats...) + ctx.TimeStats = append(ctx.TimeStats, timeStats...) + return err +} diff --git a/commands/cmd_push.go b/commands/cmd_push.go new file mode 100644 index 00000000..fdcd72ef --- /dev/null +++ b/commands/cmd_push.go @@ -0,0 +1,149 @@ +package commands + +import ( + "crypto/sha256" + "fmt" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/DanEngelbrecht/golongtail/longtailutils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func push( + numWorkerCount int, + targetChunkSize uint32, + targetBlockSize uint32, + maxChunksPerBlock uint32, + compressionAlgorithm string, + hashAlgorithm string, + minBlockUsagePercent uint32, + enableFileMapping bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) { + const fname = "get" + log := logrus.WithFields(logrus.Fields{ + "fname": fname, + "numWorkerCount": numWorkerCount, + "targetChunkSize": targetChunkSize, + "targetBlockSize": targetBlockSize, + "maxChunksPerBlock": maxChunksPerBlock, + "compressionAlgorithm": compressionAlgorithm, + "hashAlgorithm": hashAlgorithm, + "minBlockUsagePercent": minBlockUsagePercent, + "enableFileMapping": enableFileMapping, + }) + log.Debug(fname) + + setupStartTime := time.Now() + + storeStats := []longtailutils.StoreStat{} + timeStats := []longtailutils.TimeStat{} + + logFile := ".longtail/log" + excludeFilterRegEx := "^\\.longtail(/|$)" + sourceFolderPath := "" + targetFilePath := ".longtail/tmp.lvi" + blobStoreURI := ".longtail/store" + versionLocalStoreIndexPath := ".longtail/tmp.lsi" + err := os.MkdirAll(".longtail/store/versions", 0777) + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + + setupTime := time.Since(setupStartTime) + timeStats = append(timeStats, longtailutils.TimeStat{"Setup", setupTime}) + + upSyncStoreStats, upSyncTimeStats, err := upsync( + numWorkerCount, + blobStoreURI, + sourceFolderPath, + "", + targetFilePath, + targetChunkSize, + targetBlockSize, + maxChunksPerBlock, + compressionAlgorithm, + hashAlgorithm, + "", + excludeFilterRegEx, + minBlockUsagePercent, + versionLocalStoreIndexPath, + enableFileMapping) + + versionFile, err := os.Open(targetFilePath) + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + versionFileData, err := ioutil.ReadAll(versionFile) + versionFile.Close() + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + + sum := sha256.Sum256(versionFileData) + + hashedVersionFilePath := fmt.Sprintf(".longtail/store/versions/%x.lvi", sum) + err = os.Rename(targetFilePath, hashedVersionFilePath) + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + hashedVersionLocalStoreIndexPath := fmt.Sprintf(".longtail/store/versions/%x.lsi", sum) + err = os.Rename(versionLocalStoreIndexPath, hashedVersionLocalStoreIndexPath) + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + + history := "" + _, err = os.Stat(logFile) + if err == nil { + historyBytes, err := ioutil.ReadFile(logFile) + if err != nil { + return storeStats, timeStats, errors.Wrap(err, fname) + } + history = string(historyBytes) + } else if !os.IsNotExist(err) { + return storeStats, timeStats, errors.Wrap(err, fname) + } + + historyLines := make([]string, 0) + if history != "" { + historyLines = strings.Split(string(history), "\n") + } + + historyLines = append(historyLines, hashedVersionFilePath) + history = strings.Join(historyLines, "\n") + + ioutil.WriteFile(logFile, []byte(history), 0666) + + storeStats = append(storeStats, upSyncStoreStats...) + timeStats = append(timeStats, upSyncTimeStats...) + + return storeStats, timeStats, errors.Wrap(err, fname) +} + +type PushCmd struct { + TargetChunkSizeOption + MaxChunksPerBlockOption + TargetBlockSizeOption + MinBlockUsagePercentOption + CompressionOption + HashingOption + EnableFileMappingOption +} + +func (r *PushCmd) Run(ctx *Context) error { + storeStats, timeStats, err := push( + ctx.NumWorkerCount, + r.TargetChunkSize, + r.TargetBlockSize, + r.MaxChunksPerBlock, + r.Compression, + r.Hashing, + r.MinBlockUsagePercent, + r.EnableFileMapping) + ctx.StoreStats = append(ctx.StoreStats, storeStats...) + ctx.TimeStats = append(ctx.TimeStats, timeStats...) + return err +} diff --git a/commands/commands.go b/commands/commands.go index eec340c1..199b7b60 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -29,4 +29,6 @@ var Cli struct { Pack PackCmd `cmd:"" name:"pack" help:"Pack a source to an archive"` Unpack UnpackCmd `cmd:"" name:"unpack" help:"Unpack an archive"` Put PutCmd `cmd:"" name:"put" help:"Upload a folder"` + Push PushCmd `cmd:"" name:"push" help:"Makes a snapshot of the current folder and store in .longtail folder"` + Pop PopCmd `cmd:"" name:"pop" help:"Restores the most current snapshot to the current folder from store in .longtail folder"` }