feature/snapshot

commands/cmd_pop.go

@@ -0,0 +1,105 @@
+package commands
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/DanEngelbrecht/golongtail/longtailutils"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func pop(
+	numWorkerCount int,
+	stackOffset uint32,
+	retainPermissions bool,
+	enableFileMapping bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
+	const fname = "get"
+	log := logrus.WithFields(logrus.Fields{
+		"fname":             fname,
+		"stackOffset":       stackOffset,
+		"numWorkerCount":    numWorkerCount,
+		"retainPermissions": retainPermissions,
+		"enableFileMapping": enableFileMapping,
+	})
+	log.Debug(fname)
+
+	storeStats := []longtailutils.StoreStat{}
+	timeStats := []longtailutils.TimeStat{}
+
+	setupStartTime := time.Now()
+
+	logFile := ".longtail/log"
+	targetFolderPath := "."
+	excludeFilterRegEx := "^\\.longtail(/|$)"
+	blobStoreURI := ".longtail/store"
+
+	history := ""
+	_, err := os.Stat(logFile)
+	if err == nil {
+		historyBytes, err := ioutil.ReadFile(logFile)
+		if err != nil {
+			return storeStats, timeStats, errors.Wrap(err, fname)
+		}
+		history = string(historyBytes)
+	} else if !os.IsNotExist(err) {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+
+	historyLines := make([]string, 0)
+	if history != "" {
+		historyLines = strings.Split(string(history), "\n")
+	}
+
+	if len(historyLines) < int(stackOffset+1) {
+		err = fmt.Errorf("stack offset out of bounds for `%s`, log contains `%d` entries, need `%d` entries", logFile, len(historyLines), int(stackOffset+1))
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+
+	sourceFilePath := historyLines[len(historyLines)-int(stackOffset+1)]
+	versionLocalStoreIndexPath := sourceFilePath[:len(sourceFilePath)-3] + "lsi"
+
+	setupTime := time.Since(setupStartTime)
+	timeStats = append(timeStats, longtailutils.TimeStat{"Setup", setupTime})
+
+	downSyncStoreStats, downSyncTimeStats, err := downsync(
+		numWorkerCount,
+		blobStoreURI,
+		sourceFilePath,
+		targetFolderPath,
+		"",
+		"",
+		retainPermissions,
+		false,
+		versionLocalStoreIndexPath,
+		"",
+		excludeFilterRegEx,
+		true,
+		false,
+		enableFileMapping)
+
+	storeStats = append(storeStats, downSyncStoreStats...)
+	timeStats = append(timeStats, downSyncTimeStats...)
+
+	return storeStats, timeStats, errors.Wrap(err, fname)
+}
+
+type PopCmd struct {
+	StackOffset uint32 `name:"offset" help:"Offset into log, zero equals top of stack (latest push)" default:"0"`
+	RetainPermissionsOption
+	EnableFileMappingOption
+}
+
+func (r *PopCmd) Run(ctx *Context) error {
+	storeStats, timeStats, err := pop(
+		ctx.NumWorkerCount,
+		r.StackOffset,
+		r.RetainPermissions,
+		r.EnableFileMapping)
+	ctx.StoreStats = append(ctx.StoreStats, storeStats...)
+	ctx.TimeStats = append(ctx.TimeStats, timeStats...)
+	return err
+}

commands/cmd_push.go

@@ -0,0 +1,149 @@
+package commands
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/DanEngelbrecht/golongtail/longtailutils"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func push(
+	numWorkerCount int,
+	targetChunkSize uint32,
+	targetBlockSize uint32,
+	maxChunksPerBlock uint32,
+	compressionAlgorithm string,
+	hashAlgorithm string,
+	minBlockUsagePercent uint32,
+	enableFileMapping bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
+	const fname = "get"
+	log := logrus.WithFields(logrus.Fields{
+		"fname":                fname,
+		"numWorkerCount":       numWorkerCount,
+		"targetChunkSize":      targetChunkSize,
+		"targetBlockSize":      targetBlockSize,
+		"maxChunksPerBlock":    maxChunksPerBlock,
+		"compressionAlgorithm": compressionAlgorithm,
+		"hashAlgorithm":        hashAlgorithm,
+		"minBlockUsagePercent": minBlockUsagePercent,
+		"enableFileMapping":    enableFileMapping,
+	})
+	log.Debug(fname)
+
+	setupStartTime := time.Now()
+
+	storeStats := []longtailutils.StoreStat{}
+	timeStats := []longtailutils.TimeStat{}
+
+	logFile := ".longtail/log"
+	excludeFilterRegEx := "^\\.longtail(/|$)"
+	sourceFolderPath := ""
+	targetFilePath := ".longtail/tmp.lvi"
+	blobStoreURI := ".longtail/store"
+	versionLocalStoreIndexPath := ".longtail/tmp.lsi"
+	err := os.MkdirAll(".longtail/store/versions", 0777)
+	if err != nil {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+
+	setupTime := time.Since(setupStartTime)
+	timeStats = append(timeStats, longtailutils.TimeStat{"Setup", setupTime})
+
+	upSyncStoreStats, upSyncTimeStats, err := upsync(
+		numWorkerCount,
+		blobStoreURI,
+		sourceFolderPath,
+		"",
+		targetFilePath,
+		targetChunkSize,
+		targetBlockSize,
+		maxChunksPerBlock,
+		compressionAlgorithm,
+		hashAlgorithm,
+		"",
+		excludeFilterRegEx,
+		minBlockUsagePercent,
+		versionLocalStoreIndexPath,
+		enableFileMapping)
+
+	versionFile, err := os.Open(targetFilePath)
+	if err != nil {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+	versionFileData, err := ioutil.ReadAll(versionFile)
+	versionFile.Close()
+	if err != nil {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+
+	sum := sha256.Sum256(versionFileData)
+
+	hashedVersionFilePath := fmt.Sprintf(".longtail/store/versions/%x.lvi", sum)
+	err = os.Rename(targetFilePath, hashedVersionFilePath)
+	if err != nil {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+	hashedVersionLocalStoreIndexPath := fmt.Sprintf(".longtail/store/versions/%x.lsi", sum)
+	err = os.Rename(versionLocalStoreIndexPath, hashedVersionLocalStoreIndexPath)
+	if err != nil {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+
+	history := ""
+	_, err = os.Stat(logFile)
+	if err == nil {
+		historyBytes, err := ioutil.ReadFile(logFile)
+		if err != nil {
+			return storeStats, timeStats, errors.Wrap(err, fname)
+		}
+		history = string(historyBytes)
+	} else if !os.IsNotExist(err) {
+		return storeStats, timeStats, errors.Wrap(err, fname)
+	}
+
+	historyLines := make([]string, 0)
+	if history != "" {
+		historyLines = strings.Split(string(history), "\n")
+	}
+
+	historyLines = append(historyLines, hashedVersionFilePath)
+	history = strings.Join(historyLines, "\n")
+
+	ioutil.WriteFile(logFile, []byte(history), 0666)
+
+	storeStats = append(storeStats, upSyncStoreStats...)
+	timeStats = append(timeStats, upSyncTimeStats...)
+
+	return storeStats, timeStats, errors.Wrap(err, fname)
+}
+
+type PushCmd struct {
+	TargetChunkSizeOption
+	MaxChunksPerBlockOption
+	TargetBlockSizeOption
+	MinBlockUsagePercentOption
+	CompressionOption
+	HashingOption
+	EnableFileMappingOption
+}
+
+func (r *PushCmd) Run(ctx *Context) error {
+	storeStats, timeStats, err := push(
+		ctx.NumWorkerCount,
+		r.TargetChunkSize,
+		r.TargetBlockSize,
+		r.MaxChunksPerBlock,
+		r.Compression,
+		r.Hashing,
+		r.MinBlockUsagePercent,
+		r.EnableFileMapping)
+	ctx.StoreStats = append(ctx.StoreStats, storeStats...)
+	ctx.TimeStats = append(ctx.TimeStats, timeStats...)
+	return err
+}

commands/commands.go

@@ -29,4 +29,6 @@ var Cli struct {
 	Pack                    PackCmd                    `cmd:"" name:"pack" help:"Pack a source to an archive"`
 	Unpack                  UnpackCmd                  `cmd:"" name:"unpack" help:"Unpack an archive"`
 	Put                     PutCmd                     `cmd:"" name:"put" help:"Upload a folder"`
+	Push                    PushCmd                    `cmd:"" name:"push" help:"Makes a snapshot of the current folder and store in .longtail folder"`
+	Pop                     PopCmd                     `cmd:"" name:"pop" help:"Restores the most current snapshot to the current folder from store in .longtail folder"`
 }