Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature/snapshot #206

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
push command
DanEngelbrecht committed May 29, 2022
commit 4cf0a2349ead2c30ef477e0011d91592d804594d
Binary file not shown.
125 changes: 125 additions & 0 deletions commands/cmd_push.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package commands

import (
"crypto/sha256"
"fmt"
"io/ioutil"
"os"
"time"

"github.com/DanEngelbrecht/golongtail/longtailutils"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

func push(
numWorkerCount int,
targetChunkSize uint32,
targetBlockSize uint32,
maxChunksPerBlock uint32,
compressionAlgorithm string,
hashAlgorithm string,
minBlockUsagePercent uint32,
enableFileMapping bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
const fname = "get"
log := logrus.WithFields(logrus.Fields{
"fname": fname,
"numWorkerCount": numWorkerCount,
"targetChunkSize": targetChunkSize,
"targetBlockSize": targetBlockSize,
"maxChunksPerBlock": maxChunksPerBlock,
"compressionAlgorithm": compressionAlgorithm,
"hashAlgorithm": hashAlgorithm,
"minBlockUsagePercent": minBlockUsagePercent,
"enableFileMapping": enableFileMapping,
})
log.Debug(fname)

setupStartTime := time.Now()

storeStats := []longtailutils.StoreStat{}
timeStats := []longtailutils.TimeStat{}

excludeFilterRegEx := ".longtail/*."
sourceFolderPath := ""
targetFilePath := ".longtail/tmp.lvi"
blobStoreURI := ".longtail/store"
versionLocalStoreIndexPath := ".longtail/tmp.lsi"
err := os.MkdirAll(".longtail/store/versions", 0777)
if err != nil {
return storeStats, timeStats, errors.Wrap(err, fname)
}

setupTime := time.Since(setupStartTime)
timeStats = append(timeStats, longtailutils.TimeStat{"Setup", setupTime})

upSyncStoreStats, upSyncTimeStats, err := upsync(
numWorkerCount,
blobStoreURI,
sourceFolderPath,
"",
targetFilePath,
targetChunkSize,
targetBlockSize,
maxChunksPerBlock,
compressionAlgorithm,
hashAlgorithm,
"",
excludeFilterRegEx,
minBlockUsagePercent,
versionLocalStoreIndexPath,
enableFileMapping)

versionFile, err := os.Open(targetFilePath)
if err != nil {
return storeStats, timeStats, errors.Wrap(err, fname)
}
versionFileData, err := ioutil.ReadAll(versionFile)
versionFile.Close()
if err != nil {
return storeStats, timeStats, errors.Wrap(err, fname)
}

sum := sha256.Sum256(versionFileData)

hashedVersionFilePath := fmt.Sprintf(".longtail/store/versions/%x.lvi", sum)
err = os.Rename(targetFilePath, hashedVersionFilePath)
if err != nil {
return storeStats, timeStats, errors.Wrap(err, fname)
}
hashedVersionLocalStoreIndexPath := fmt.Sprintf(".longtail/store/versions/%x.lsi", sum)
err = os.Rename(versionLocalStoreIndexPath, hashedVersionLocalStoreIndexPath)
if err != nil {
return storeStats, timeStats, errors.Wrap(err, fname)
}

storeStats = append(storeStats, upSyncStoreStats...)
timeStats = append(timeStats, upSyncTimeStats...)

return storeStats, timeStats, errors.Wrap(err, fname)
}

type PushCmd struct {
TargetChunkSizeOption
MaxChunksPerBlockOption
TargetBlockSizeOption
MinBlockUsagePercentOption
CompressionOption
HashingOption
EnableFileMappingOption
}

func (r *PushCmd) Run(ctx *Context) error {
storeStats, timeStats, err := push(
ctx.NumWorkerCount,
r.TargetChunkSize,
r.TargetBlockSize,
r.MaxChunksPerBlock,
r.Compression,
r.Hashing,
r.MinBlockUsagePercent,
r.EnableFileMapping)
ctx.StoreStats = append(ctx.StoreStats, storeStats...)
ctx.TimeStats = append(ctx.TimeStats, timeStats...)
return err
}
2 changes: 2 additions & 0 deletions commands/commands.go
Original file line number Diff line number Diff line change
@@ -29,4 +29,6 @@ var Cli struct {
Pack PackCmd `cmd:"" name:"pack" help:"Pack a source to an archive"`
Unpack UnpackCmd `cmd:"" name:"unpack" help:"Unpack an archive"`
Put PutCmd `cmd:"" name:"put" help:"Upload a folder"`
Push PushCmd `cmd:"" name:"push" help:"Makes a snapshot of the current folder and store in .longtail folder"`
// Pop PopCmd `cmd:"" name:"pop" help:"Restores the most current snapshot to the current folder from store in .longtail folder"`
}