Test/verify no build index on downsync (#175)

* Add DeleteByURI the let the blob store follow the rules for locking etc. * Add TestDownsyncMissingIndex test * TestDownsyncNoTargetPath test * Remove redundant log
DanEngelbrecht · Oct 13, 2021 · 033c6eb · 033c6eb
1 parent e80ef1b
commit 033c6eb
Show file tree

Hide file tree

Showing 3 changed files with 104 additions and 1 deletion.
diff --git a/commands/cmd_clonestore.go b/commands/cmd_clonestore.go
@@ -127,7 +127,6 @@ func downloadFromZip(targetPath string, sourceFileZipPath string) error {
 		}
 		defer func() {
 			if err := rc.Close(); err != nil {
-				log.WithError(err).Error(fname)
 				panic(errors.Wrap(err, fname))
 			}
 		}()

diff --git a/commands/cmd_downsync_test.go b/commands/cmd_downsync_test.go
@@ -5,6 +5,8 @@ import (
 	"os"
 	"path"
 	"testing"
+
+	"github.com/DanEngelbrecht/golongtail/longtailutils"
 )
 
 func TestDownsync(t *testing.T) {
@@ -32,6 +34,34 @@ func TestDownsync(t *testing.T) {
 	validateContent(t, fsBlobPathPrefix, "version/current", v3FilesCreate)
 }
 
+func TestDownsyncNoTargetPath(t *testing.T) {
+	testPath, _ := ioutil.TempDir("", "test")
+	fsBlobPathPrefix := "fsblob://" + testPath
+	createVersionData(t, fsBlobPathPrefix)
+	executeCommandLine("upsync", "--source-path", testPath+"/version/v1", "--target-path", fsBlobPathPrefix+"/index/v1b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	executeCommandLine("upsync", "--source-path", testPath+"/version/v2", "--target-path", fsBlobPathPrefix+"/index/v2b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	executeCommandLine("upsync", "--source-path", testPath+"/version/v3", "--target-path", fsBlobPathPrefix+"/index/v3b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+
+	popd, _ := os.Getwd()
+	defer os.Chdir(popd)
+	os.Chdir(path.Join(testPath, "version"))
+	cmd, err := executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v1b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	validateContent(t, fsBlobPathPrefix, "version/v1b", v1FilesCreate)
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v2b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	validateContent(t, fsBlobPathPrefix, "version/v2b", v2FilesCreate)
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v3b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	validateContent(t, fsBlobPathPrefix, "version/v3b", v3FilesCreate)
+}
+
 func TestDownsyncWithVersionLSI(t *testing.T) {
 	testPath, _ := ioutil.TempDir("", "test")
 	fsBlobPathPrefix := "fsblob://" + testPath
@@ -232,3 +262,48 @@ func TestDownsyncMissingChunks(t *testing.T) {
 		t.Errorf("%s: %s", cmd, err)
 	}
 }
+
+func TestDownsyncMissingIndex(t *testing.T) {
+	testPath, _ := ioutil.TempDir("", "test")
+	fsBlobPathPrefix := "fsblob://" + testPath
+	createVersionData(t, fsBlobPathPrefix)
+	executeCommandLine("upsync", "--source-path", testPath+"/version/v1", "--target-path", fsBlobPathPrefix+"/index/v1.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	executeCommandLine("upsync", "--source-path", testPath+"/version/v2", "--target-path", fsBlobPathPrefix+"/index/v2.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+	executeCommandLine("upsync", "--source-path", testPath+"/version/v3", "--target-path", fsBlobPathPrefix+"/index/v3.lvi", "--storage-uri", fsBlobPathPrefix+"/storage")
+
+	longtailutils.DeleteByURI(fsBlobPathPrefix + "/storage/store.lsi")
+
+	cmd, err := executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v1.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage")
+	if err == nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v2.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage")
+	if err == nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v3.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage")
+	if err == nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+
+	cmd, err = executeCommandLine("init-remote-store", "--storage-uri", fsBlobPathPrefix+"/storage", "--worker-count", "1")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v1.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage", "--version-local-store-index-path", fsBlobPathPrefix+"/index/v1.lsi", "--cache-path", testPath+"/cache", "--validate")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	validateContent(t, fsBlobPathPrefix, "version/current", v1FilesCreate)
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v2.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage", "--version-local-store-index-path", fsBlobPathPrefix+"/index/v2.lsi", "--cache-path", testPath+"/cache", "--validate")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	validateContent(t, fsBlobPathPrefix, "version/current", v2FilesCreate)
+	cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v3.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage", "--version-local-store-index-path", fsBlobPathPrefix+"/index/v3.lsi", "--cache-path", testPath+"/cache", "--validate")
+	if err != nil {
+		t.Errorf("%s: %s", cmd, err)
+	}
+	validateContent(t, fsBlobPathPrefix, "version/current", v3FilesCreate)
+}
diff --git a/longtailutils/longtailutils.go b/longtailutils/longtailutils.go
@@ -336,6 +336,35 @@ func WriteToURI(uri string, data []byte) error {
 	return nil
 }
 
+// DeleteByURI ...
+func DeleteByURI(uri string) error {
+	const fname = "DeleteByURI"
+	log := logrus.WithFields(logrus.Fields{
+		"fname": fname,
+		"uri":   uri,
+	})
+	log.Debug(fname)
+	uriParent, uriName := splitURI(uri)
+	blobStore, err := longtailstorelib.CreateBlobStoreForURI(uriParent)
+	if err != nil {
+		return errors.Wrap(err, fname)
+	}
+	client, err := blobStore.NewClient(context.Background())
+	if err != nil {
+		return errors.Wrap(err, fname)
+	}
+	defer client.Close()
+	object, err := client.NewObject(uriName)
+	if err != nil {
+		return errors.Wrap(err, fname)
+	}
+	err = object.Delete()
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return errors.Wrap(err, fname)
+	}
+	return nil
+}
+
 func ReadBlobWithRetry(
 	ctx context.Context,
 	client longtailstorelib.BlobClient,