diff --git a/commands/cmd_clonestore.go b/commands/cmd_clonestore.go index 3aa6df21..2ff6b6a6 100644 --- a/commands/cmd_clonestore.go +++ b/commands/cmd_clonestore.go @@ -127,7 +127,6 @@ func downloadFromZip(targetPath string, sourceFileZipPath string) error { } defer func() { if err := rc.Close(); err != nil { - log.WithError(err).Error(fname) panic(errors.Wrap(err, fname)) } }() diff --git a/commands/cmd_downsync_test.go b/commands/cmd_downsync_test.go index 3ab38a23..604a46fa 100644 --- a/commands/cmd_downsync_test.go +++ b/commands/cmd_downsync_test.go @@ -5,6 +5,8 @@ import ( "os" "path" "testing" + + "github.com/DanEngelbrecht/golongtail/longtailutils" ) func TestDownsync(t *testing.T) { @@ -32,6 +34,34 @@ func TestDownsync(t *testing.T) { validateContent(t, fsBlobPathPrefix, "version/current", v3FilesCreate) } +func TestDownsyncNoTargetPath(t *testing.T) { + testPath, _ := ioutil.TempDir("", "test") + fsBlobPathPrefix := "fsblob://" + testPath + createVersionData(t, fsBlobPathPrefix) + executeCommandLine("upsync", "--source-path", testPath+"/version/v1", "--target-path", fsBlobPathPrefix+"/index/v1b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + executeCommandLine("upsync", "--source-path", testPath+"/version/v2", "--target-path", fsBlobPathPrefix+"/index/v2b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + executeCommandLine("upsync", "--source-path", testPath+"/version/v3", "--target-path", fsBlobPathPrefix+"/index/v3b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + + popd, _ := os.Getwd() + defer os.Chdir(popd) + os.Chdir(path.Join(testPath, "version")) + cmd, err := executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v1b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + validateContent(t, fsBlobPathPrefix, "version/v1b", v1FilesCreate) + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v2b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + validateContent(t, fsBlobPathPrefix, "version/v2b", v2FilesCreate) + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v3b.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + validateContent(t, fsBlobPathPrefix, "version/v3b", v3FilesCreate) +} + func TestDownsyncWithVersionLSI(t *testing.T) { testPath, _ := ioutil.TempDir("", "test") fsBlobPathPrefix := "fsblob://" + testPath @@ -232,3 +262,48 @@ func TestDownsyncMissingChunks(t *testing.T) { t.Errorf("%s: %s", cmd, err) } } + +func TestDownsyncMissingIndex(t *testing.T) { + testPath, _ := ioutil.TempDir("", "test") + fsBlobPathPrefix := "fsblob://" + testPath + createVersionData(t, fsBlobPathPrefix) + executeCommandLine("upsync", "--source-path", testPath+"/version/v1", "--target-path", fsBlobPathPrefix+"/index/v1.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + executeCommandLine("upsync", "--source-path", testPath+"/version/v2", "--target-path", fsBlobPathPrefix+"/index/v2.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + executeCommandLine("upsync", "--source-path", testPath+"/version/v3", "--target-path", fsBlobPathPrefix+"/index/v3.lvi", "--storage-uri", fsBlobPathPrefix+"/storage") + + longtailutils.DeleteByURI(fsBlobPathPrefix + "/storage/store.lsi") + + cmd, err := executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v1.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage") + if err == nil { + t.Errorf("%s: %s", cmd, err) + } + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v2.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage") + if err == nil { + t.Errorf("%s: %s", cmd, err) + } + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v3.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage") + if err == nil { + t.Errorf("%s: %s", cmd, err) + } + + cmd, err = executeCommandLine("init-remote-store", "--storage-uri", fsBlobPathPrefix+"/storage", "--worker-count", "1") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v1.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage", "--version-local-store-index-path", fsBlobPathPrefix+"/index/v1.lsi", "--cache-path", testPath+"/cache", "--validate") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + validateContent(t, fsBlobPathPrefix, "version/current", v1FilesCreate) + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v2.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage", "--version-local-store-index-path", fsBlobPathPrefix+"/index/v2.lsi", "--cache-path", testPath+"/cache", "--validate") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + validateContent(t, fsBlobPathPrefix, "version/current", v2FilesCreate) + cmd, err = executeCommandLine("downsync", "--source-path", fsBlobPathPrefix+"/index/v3.lvi", "--target-path", testPath+"/version/current", "--storage-uri", fsBlobPathPrefix+"/storage", "--version-local-store-index-path", fsBlobPathPrefix+"/index/v3.lsi", "--cache-path", testPath+"/cache", "--validate") + if err != nil { + t.Errorf("%s: %s", cmd, err) + } + validateContent(t, fsBlobPathPrefix, "version/current", v3FilesCreate) +} diff --git a/longtailutils/longtailutils.go b/longtailutils/longtailutils.go index 13af8d46..b0a8bfec 100644 --- a/longtailutils/longtailutils.go +++ b/longtailutils/longtailutils.go @@ -336,6 +336,35 @@ func WriteToURI(uri string, data []byte) error { return nil } +// DeleteByURI ... +func DeleteByURI(uri string) error { + const fname = "DeleteByURI" + log := logrus.WithFields(logrus.Fields{ + "fname": fname, + "uri": uri, + }) + log.Debug(fname) + uriParent, uriName := splitURI(uri) + blobStore, err := longtailstorelib.CreateBlobStoreForURI(uriParent) + if err != nil { + return errors.Wrap(err, fname) + } + client, err := blobStore.NewClient(context.Background()) + if err != nil { + return errors.Wrap(err, fname) + } + defer client.Close() + object, err := client.NewObject(uriName) + if err != nil { + return errors.Wrap(err, fname) + } + err = object.Delete() + if err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.Wrap(err, fname) + } + return nil +} + func ReadBlobWithRetry( ctx context.Context, client longtailstorelib.BlobClient,