Feature/store target index (#198)

* update longtail to  v0.2.18
* Add --cache-target-index option (default on)
Linting fixes

.github/workflows/create-release.yml

@@ -125,8 +125,8 @@ jobs:
         release_name: Release ${{ github.ref }}
         body: |
           Changes in this Release
-          - **UPDATED** Updated longtail to v0.2.16 - fixes handling of UNC paths
-          - **FIX** Minor logging tweaks
+          - **ADDED** Added `--cache-target-index` option to downsync/unpack that automatically caches target folder state. Default on, turn off with `--no-cache-target-index`
+          - **UPDATED** Updated longtail to v0.2.18
         draft: false
         prerelease: false
     - name: Download Linux artifacts

commands/cmd_clonestore.go

@@ -92,10 +92,10 @@ func downloadFromZip(targetPath string, sourceFileZipPath string) error {
 		"sourceFileZipPath": sourceFileZipPath,
 	})
 	if sourceFileZipPath == "" {
-		err := fmt.Errorf("Skipping, no zip file available for `%s`", sourceFileZipPath)
+		err := fmt.Errorf("skipping, no zip file available for `%s`", sourceFileZipPath)
 		return errors.Wrap(err, fname)
 	}
-	log.Infof("Falling back to reading ZIP source from `%s`", sourceFileZipPath)
+	log.Infof("falling back to reading ZIP source from `%s`", sourceFileZipPath)
 	zipBytes, err := longtailutils.ReadFromURI(sourceFileZipPath)
 	if err != nil {
 		return errors.Wrap(err, fname)
@@ -128,7 +128,7 @@ func downloadFromZip(targetPath string, sourceFileZipPath string) error {
 		defer func() {
 			if err := rc.Close(); err != nil {
 				err = errors.Wrap(err, fname)
-				log.WithError(err).Errorf("Failed to close zip file")
+				log.WithError(err).Errorf("failed to close zip file")
 			}
 		}()
 
@@ -137,7 +137,7 @@ func downloadFromZip(targetPath string, sourceFileZipPath string) error {
 
 		// Check for ZipSlip (Directory traversal)
 		if !strings.HasPrefix(path, filepath.Clean(targetPath)+string(os.PathSeparator)) {
-			err := fmt.Errorf("Illegal file path: `%s`", path)
+			err := fmt.Errorf("illegal file path: `%s`", path)
 			return errors.Wrap(err, fname)
 		}
 
@@ -158,7 +158,7 @@ func downloadFromZip(targetPath string, sourceFileZipPath string) error {
 			defer func() {
 				if err := f.Close(); err != nil {
 					err = errors.Wrap(err, fname)
-					log.WithError(err).Errorf("Failed to close target file")
+					log.WithError(err).Errorf("failed to close target file")
 				}
 			}()
 
@@ -217,7 +217,7 @@ func updateCurrentVersionFromLongtail(
 		localVersionIndex = cloneVersionIndex(targetPathVersionIndex)
 		hash, err = hashRegistry.GetHashAPI(hashIdentifier)
 		if err != nil {
-			err = errors.Wrap(err, fmt.Sprintf("Unsupported hash identifier `%d`", hashIdentifier))
+			err = errors.Wrap(err, fmt.Sprintf("unsupported hash identifier `%d`", hashIdentifier))
 			return localVersionIndex, longtaillib.Longtail_HashAPI{}, errors.Wrap(err, fname)
 		}
 	} else {
@@ -238,7 +238,7 @@ func updateCurrentVersionFromLongtail(
 
 		localVersionIndex, hash, _, err = targetIndexReader.Get()
 		if err != nil {
-			err := errors.Wrap(err, "Failed scanning target path")
+			err := errors.Wrap(err, "failed scanning target path")
 			return longtaillib.Longtail_VersionIndex{}, hash, errors.Wrap(err, fname)
 		}
 	}
@@ -248,7 +248,7 @@ func updateCurrentVersionFromLongtail(
 		localVersionIndex,
 		sourceVersionIndex)
 	if err != nil {
-		err = errors.Wrap(err, fmt.Sprintf("Failed to create version diff. `%s` -> `%s`", targetPath, sourceFilePath))
+		err = errors.Wrap(err, fmt.Sprintf("failed to create version diff. `%s` -> `%s`", targetPath, sourceFilePath))
 		return localVersionIndex, hash, errors.Wrap(err, fname)
 	}
 	defer versionDiff.Dispose()
@@ -257,7 +257,7 @@ func updateCurrentVersionFromLongtail(
 		sourceVersionIndex,
 		versionDiff)
 	if err != nil {
-		err = errors.Wrap(err, fmt.Sprintf("Failed to get required chunk hashes. `%s` -> `%s`", targetPath, sourceFilePath))
+		err = errors.Wrap(err, fmt.Sprintf("failed to get required chunk hashes. `%s` -> `%s`", targetPath, sourceFilePath))
 		return localVersionIndex, hash, errors.Wrap(err, fname)
 	}
 
@@ -313,7 +313,7 @@ func updateCurrentVersionFromLongtail(
 
 	localVersionIndex, hash, _, err = targetIndexReader.Get()
 	if err != nil {
-		err := errors.Wrap(err, "Failed scanning target path")
+		err := errors.Wrap(err, "failed scanning target path")
 		return longtaillib.Longtail_VersionIndex{}, hash, errors.Wrap(err, fname)
 	}
 	return localVersionIndex, hash, nil
@@ -388,7 +388,7 @@ func cloneOneVersion(
 		targetBlockSize,
 		maxChunksPerBlock)
 	if err != nil {
-		err = errors.Wrap(err, fmt.Sprintf("Failed creating missing content store index for `%s`", targetPath))
+		err = errors.Wrap(err, fmt.Sprintf("failed creating missing content store index for `%s`", targetPath))
 		return targetVersionIndex, errors.Wrap(err, fname)
 	}
 	defer versionMissingStoreIndex.Dispose()
@@ -406,7 +406,7 @@ func cloneOneVersion(
 			longtailutils.NormalizePath(targetPath))
 		writeContentProgress.Dispose()
 		if err != nil {
-			err = errors.Wrap(err, fmt.Sprintf("Failed writing content from `%s`", targetPath))
+			err = errors.Wrap(err, fmt.Sprintf("failed writing content from `%s`", targetPath))
 			return targetVersionIndex, errors.Wrap(err, fname)
 		}
 	}
@@ -434,13 +434,13 @@ func cloneOneVersion(
 		versionLocalStoreIndexPath := strings.Replace(targetFilePath, ".lvi", ".lsi", -1) // TODO: This should use a file with path names instead of this rename hack!
 		versionLocalStoreIndex, err := longtaillib.MergeStoreIndex(newExistingStoreIndex, versionMissingStoreIndex)
 		if err != nil {
-			err = errors.Wrap(err, fmt.Sprintf("Failed merging store index for `%s`", versionLocalStoreIndexPath))
+			err = errors.Wrap(err, fmt.Sprintf("failed merging store index for `%s`", versionLocalStoreIndexPath))
 			return targetVersionIndex, errors.Wrap(err, fname)
 		}
 		versionLocalStoreIndexBuffer, err := longtaillib.WriteStoreIndexToBuffer(versionLocalStoreIndex)
 		versionLocalStoreIndex.Dispose()
 		if err != nil {
-			err = errors.Wrap(err, fmt.Sprintf("Failed serializing store index for `%s`", versionLocalStoreIndexPath))
+			err = errors.Wrap(err, fmt.Sprintf("failed serializing store index for `%s`", versionLocalStoreIndexPath))
 			return targetVersionIndex, errors.Wrap(err, fname)
 		}
 		err = longtailutils.WriteToURI(versionLocalStoreIndexPath, versionLocalStoreIndexBuffer)

commands/cmd_downsync.go

@@ -24,7 +24,8 @@ func downsync(
 	versionLocalStoreIndexPath string,
 	includeFilterRegEx string,
 	excludeFilterRegEx string,
-	scanTarget bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
+	scanTarget bool,
+	cacheTargetIndex bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
 	const fname = "downsync"
 	log := logrus.WithFields(logrus.Fields{
 		"fname":                      fname,
@@ -40,6 +41,7 @@ func downsync(
 		"includeFilterRegEx":         includeFilterRegEx,
 		"excludeFilterRegEx":         excludeFilterRegEx,
 		"scanTarget":                 scanTarget,
+		"cacheTargetIndex":           cacheTargetIndex,
 	})
 	log.Debug(fname)
 
@@ -63,7 +65,7 @@ func downsync(
 		sourceNameSplit := strings.Split(sourceName, ".")
 		resolvedTargetFolderPath = sourceNameSplit[0]
 		if resolvedTargetFolderPath == "" {
-			err = fmt.Errorf("Unable to resolve target path using `%s` as base", sourceFilePath)
+			err = fmt.Errorf("unable to resolve target path using `%s` as base", sourceFilePath)
 			return storeStats, timeStats, errors.Wrap(err, fname)
 		}
 	} else {
@@ -73,6 +75,18 @@ func downsync(
 	fs := longtaillib.CreateFSStorageAPI()
 	defer fs.Dispose()
 
+	if targetIndexPath != "" {
+		cacheTargetIndex = false
+	}
+
+	cacheTargetIndexPath := resolvedTargetFolderPath + "/.longtail.index.cache.lvi"
+
+	if cacheTargetIndex {
+		if longtaillib.FileExists(fs, cacheTargetIndexPath) {
+			targetIndexPath = cacheTargetIndexPath
+		}
+	}
+
 	targetFolderScanner := longtailutils.AsyncFolderScanner{}
 	if scanTarget && targetIndexPath == "" {
 		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs)
@@ -191,6 +205,13 @@ func downsync(
 	getExistingContentTime := time.Since(getExistingContentStartTime)
 	timeStats = append(timeStats, longtailutils.TimeStat{"Get content index", getExistingContentTime})
 
+	if cacheTargetIndex && longtaillib.FileExists(fs, cacheTargetIndexPath) {
+		err = longtaillib.DeleteFile(fs, cacheTargetIndexPath)
+		if err != nil {
+			return storeStats, timeStats, errors.Wrap(err, fname)
+		}
+	}
+
 	changeVersionStartTime := time.Now()
 	changeVersionProgress := longtailutils.CreateProgress("Updating version", 2)
 	defer changeVersionProgress.Dispose()
@@ -334,6 +355,13 @@ func downsync(
 		timeStats = append(timeStats, longtailutils.TimeStat{"Validate", validateTime})
 	}
 
+	if cacheTargetIndex {
+		err = longtaillib.WriteVersionIndex(fs, sourceVersionIndex, cacheTargetIndexPath)
+		if err != nil {
+			return storeStats, timeStats, errors.Wrap(err, fname)
+		}
+	}
+
 	return storeStats, timeStats, nil
 }
 
@@ -349,6 +377,7 @@ type DownsyncCmd struct {
 	TargetPathIncludeRegExOption
 	TargetPathExcludeRegExOption
 	ScanTargetOption
+	CacheTargetIndexOption
 }
 
 func (r *DownsyncCmd) Run(ctx *Context) error {
@@ -364,7 +393,8 @@ func (r *DownsyncCmd) Run(ctx *Context) error {
 		r.VersionLocalStoreIndexPath,
 		r.IncludeFilterRegEx,
 		r.ExcludeFilterRegEx,
-		r.ScanTarget)
+		r.ScanTarget,
+		r.CacheTargetIndex)
 	ctx.StoreStats = append(ctx.StoreStats, storeStats...)
 	ctx.TimeStats = append(ctx.TimeStats, timeStats...)
 	return err

commands/cmd_get.go

@@ -21,7 +21,8 @@ func get(
 	validate bool,
 	includeFilterRegEx string,
 	excludeFilterRegEx string,
-	scanTarget bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
+	scanTarget bool,
+	cacheTargetIndex bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
 	const fname = "get"
 	log := logrus.WithFields(logrus.Fields{
 		"fname":              fname,
@@ -35,6 +36,7 @@ func get(
 		"includeFilterRegEx": includeFilterRegEx,
 		"excludeFilterRegEx": excludeFilterRegEx,
 		"scanTarget":         scanTarget,
+		"cacheTargetIndex":   cacheTargetIndex,
 	})
 	log.Debug(fname)
 
@@ -57,7 +59,7 @@ func get(
 
 	blobStoreURI := v.GetString("storage-uri")
 	if blobStoreURI == "" {
-		err = fmt.Errorf("Missing storage-uri in get-config `%s`", getConfigPath)
+		err = fmt.Errorf("missing storage-uri in get-config `%s`", getConfigPath)
 		return storeStats, timeStats, errors.Wrap(err, fname)
 	}
 	sourceFilePath := v.GetString("source-path")
@@ -85,7 +87,8 @@ func get(
 		versionLocalStoreIndexPath,
 		includeFilterRegEx,
 		excludeFilterRegEx,
-		scanTarget)
+		scanTarget,
+		cacheTargetIndex)
 
 	storeStats = append(storeStats, downSyncStoreStats...)
 	timeStats = append(timeStats, downSyncTimeStats...)
@@ -104,6 +107,7 @@ type GetCmd struct {
 	TargetPathIncludeRegExOption
 	TargetPathExcludeRegExOption
 	ScanTargetOption
+	CacheTargetIndexOption
 }
 
 func (r *GetCmd) Run(ctx *Context) error {
@@ -117,7 +121,8 @@ func (r *GetCmd) Run(ctx *Context) error {
 		r.Validate,
 		r.IncludeFilterRegEx,
 		r.ExcludeFilterRegEx,
-		r.ScanTarget)
+		r.ScanTarget,
+		r.CacheTargetIndex)
 	ctx.StoreStats = append(ctx.StoreStats, storeStats...)
 	ctx.TimeStats = append(ctx.TimeStats, timeStats...)
 	return err

commands/cmd_pack.go

@@ -58,7 +58,7 @@ func pack(
 		sourceNameSplit := strings.Split(sourceName, ".")
 		resolvedTargetPath = sourceNameSplit[0]
 		if resolvedTargetPath == "" {
-			err = fmt.Errorf("Unable to resolve target path using `%s` as base", sourceFolderPath)
+			err = fmt.Errorf("unable to resolve target path using `%s` as base", sourceFolderPath)
 			return storeStats, timeStats, errors.Wrap(err, fname)
 		}
 		resolvedTargetPath += ".la"

commands/cmd_printstore_test.go

@@ -2,41 +2,9 @@ package commands
 
 import (
 	"io/ioutil"
-	"runtime"
 	"testing"
-
-	"github.com/alecthomas/kong"
 )
 
-func runPrintStore(t *testing.T, storeIndexPath string, compact bool, details bool) {
-	parser, err := kong.New(&Cli)
-	if err != nil {
-		t.Errorf("kong.New(Cli) failed with %s", err)
-	}
-	args := []string{
-		"print-store",
-		"--store-index-path", storeIndexPath,
-	}
-	if compact {
-		args = append(args, "--compact")
-	}
-	if details {
-		args = append(args, "--details")
-	}
-	ctx, err := parser.Parse(args)
-	if err != nil {
-		t.Errorf("parser.Parse() failed with %s", err)
-	}
-
-	context := &Context{
-		NumWorkerCount: runtime.NumCPU(),
-	}
-	err = ctx.Run(context)
-	if err != nil {
-		t.Errorf("ctx.Run(context) failed with %s", err)
-	}
-}
-
 func TestPrintStoreIndex(t *testing.T) {
 	testPath, _ := ioutil.TempDir("", "test")
 	fsBlobPathPrefix := "fsblob://" + testPath