diff --git a/fs/file_handle.go b/fs/file_handle.go index c7837ae..7032bc9 100644 --- a/fs/file_handle.go +++ b/fs/file_handle.go @@ -9,7 +9,7 @@ import ( "github.com/cyverse/go-irodsclient/irods/types" ) -// FileHandle ... +// FileHandle is a handle for a file opened type FileHandle struct { id string filesystem *FileSystem @@ -285,8 +285,6 @@ func (handle *FileHandle) postprocessRename(newPath string, newEntry *Entry) err if handle.offset != newOffset { return fmt.Errorf("failed to seek to %d", handle.offset) } - - fmt.Printf("postprocessRename seeked - %s, offset %d\n", newPath, handle.offset) } fileHandle := &FileHandle{ diff --git a/fs/fs.go b/fs/fs.go index 2a8c107..04d35b2 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -673,8 +673,6 @@ func (fs *FileSystem) preprocessRenameFileHandle(srcPath string) ([]*FileHandle, handles := fs.fileHandleMap.PopByPath(srcPath) handlesLocked := []*FileHandle{} - fmt.Printf("preprocess rename - %s\n", srcPath) - errs := []error{} for _, handle := range handles { // lock handles @@ -717,14 +715,12 @@ func (fs *FileSystem) preprocessRenameFileHandleForDir(srcPath string) ([]*FileH } func (fs *FileSystem) postprocessRenameFileHandle(handles []*FileHandle, conn *connection.IRODSConnection, destPath string) error { - newEntry, err := fs.getDataObjectWithConnectin(conn, destPath) + newEntry, err := fs.getDataObjectWithConnection(conn, destPath) if err != nil { return err } errs := []error{} - fmt.Printf("postprocess rename - %s\n", destPath) - for _, handle := range handles { err := handle.postprocessRename(destPath, newEntry) if err != nil { @@ -755,7 +751,7 @@ func (fs *FileSystem) postprocessRenameFileHandleForDir(handles []*FileHandle, c errs = append(errs, err) } else { destFullPath := util.JoinPath(destPath, relPath) - newEntry, err := fs.getDataObjectWithConnectin(conn, destFullPath) + newEntry, err := fs.getDataObjectWithConnection(conn, destFullPath) if err != nil { errs = append(errs, err) } else { @@ -1181,7 +1177,7 @@ func (fs *FileSystem) OpenFile(path string, resource string, mode string) (*File var entry *Entry = nil if types.IsFileOpenFlagOpeningExisting(types.FileOpenMode(mode)) { // file may exists - entryExisting, err := fs.getDataObject(irodsPath) + entryExisting, err := fs.getDataObjectWithConnection(conn, irodsPath) if err == nil { entry = entryExisting } @@ -1227,33 +1223,41 @@ func (fs *FileSystem) CreateFile(path string, resource string, mode string) (*Fi return nil, err } + // create handle, err := irods_fs.CreateDataObject(conn, irodsPath, resource, mode, true) if err != nil { fs.session.ReturnConnection(conn) return nil, err } - // do not return connection here - entry := &Entry{ - ID: 0, - Type: FileEntry, - Name: util.GetIRODSPathFileName(irodsPath), - Path: irodsPath, - Owner: fs.account.ClientUser, - Size: 0, - CreateTime: time.Now(), - ModifyTime: time.Now(), - CheckSum: "", - Internal: nil, + // close - this is required to let other processes see the file existence + err = irods_fs.CloseDataObject(conn, handle) + if err != nil { + fs.session.ReturnConnection(conn) + return nil, err } + entry, err := fs.getDataObjectWithConnectionNoCache(conn, irodsPath) + if err != nil { + fs.session.ReturnConnection(conn) + return nil, err + } + + // re-open + handle, offset, err := irods_fs.OpenDataObject(conn, irodsPath, resource, mode) + if err != nil { + fs.session.ReturnConnection(conn) + return nil, err + } + + // do not return connection here fileHandle := &FileHandle{ id: xid.New().String(), filesystem: fs, connection: conn, irodsfilehandle: handle, entry: entry, - offset: 0, + offset: offset, openmode: types.FileOpenMode(mode), } @@ -1501,32 +1505,19 @@ func (fs *FileSystem) searchEntriesByMeta(metaName string, metaValue string) ([] return entries, nil } -// getDataObjectWithConnectin returns an entry for data object -func (fs *FileSystem) getDataObjectWithConnectin(conn *connection.IRODSConnection, path string) (*Entry, error) { - if fs.cache.HasNegativeEntryCache(path) { - return nil, types.NewFileNotFoundErrorf("could not find a data object") - } - - // check cache first - cachedEntry := fs.cache.GetEntryCache(path) - if cachedEntry != nil && cachedEntry.Type == FileEntry { - return cachedEntry, nil - } - - // otherwise, retrieve it and add it to cache +// getDataObjectWithConnectionNoCache returns an entry for data object +func (fs *FileSystem) getDataObjectWithConnectionNoCache(conn *connection.IRODSConnection, path string) (*Entry, error) { + // retrieve it and add it to cache collection, err := fs.getCollection(util.GetIRODSPathDirname(path)) if err != nil { return nil, err } - fmt.Printf("getDataObjectWithConnectin check data object - %s\n", path) dataobject, err := irods_fs.GetDataObjectMasterReplica(conn, collection.Internal.(*types.IRODSCollection), util.GetIRODSPathFileName(path)) if err != nil { return nil, err } - fmt.Printf("getDataObjectWithConnectin check data object found - %s\n", path) - if dataobject.ID > 0 { entry := &Entry{ ID: dataobject.ID, @@ -1550,6 +1541,22 @@ func (fs *FileSystem) getDataObjectWithConnectin(conn *connection.IRODSConnectio return nil, types.NewFileNotFoundErrorf("could not find a data object") } +// getDataObjectWithConnection returns an entry for data object +func (fs *FileSystem) getDataObjectWithConnection(conn *connection.IRODSConnection, path string) (*Entry, error) { + if fs.cache.HasNegativeEntryCache(path) { + return nil, types.NewFileNotFoundErrorf("could not find a data object") + } + + // check cache first + cachedEntry := fs.cache.GetEntryCache(path) + if cachedEntry != nil && cachedEntry.Type == FileEntry { + return cachedEntry, nil + } + + // otherwise, retrieve it and add it to cache + return fs.getDataObjectWithConnectionNoCache(conn, path) +} + // getDataObjectNoCache returns an entry for data object func (fs *FileSystem) getDataObjectNoCache(path string) (*Entry, error) { // retrieve it and add it to cache @@ -1564,14 +1571,11 @@ func (fs *FileSystem) getDataObjectNoCache(path string) (*Entry, error) { } defer fs.session.ReturnConnection(conn) - fmt.Printf("getDataObjectNoCache check data object - %s\n", path) dataobject, err := irods_fs.GetDataObjectMasterReplica(conn, collection.Internal.(*types.IRODSCollection), util.GetIRODSPathFileName(path)) if err != nil { return nil, err } - fmt.Printf("getDataObjectNoCache check data object found - %s\n", path) - if dataobject.ID > 0 { entry := &Entry{ ID: dataobject.ID, diff --git a/irods/common/api_number.go b/irods/common/api_number.go index 7a1e07e..9cbb359 100644 --- a/irods/common/api_number.go +++ b/irods/common/api_number.go @@ -186,7 +186,8 @@ const ( AUTH_PLUG_REQ_AN APINumber = 1201 AUTH_PLUG_RESP_AN APINumber = 1202 - ATOMIC_APPLY_METADATA_OPERATIONS_APN APINumber = 20002 GET_FILE_DESCRIPTOR_INFO_APN APINumber = 20000 + ATOMIC_APPLY_METADATA_OPERATIONS_APN APINumber = 20002 REPLICA_CLOSE_APN APINumber = 20004 + TOUCH_APN APINumber = 20007 ) diff --git a/test/server/docker-compose.yml b/test/server/docker-compose.yml index f95e28c..eefc0ab 100644 --- a/test/server/docker-compose.yml +++ b/test/server/docker-compose.yml @@ -11,7 +11,7 @@ services: POSTGRESQL_USER: "irods" POSTGRESQL_DATABASE: "ICAT" irods: - image: cyverse/irods-test:v4.2.10 + image: cyverse/irods-test:v4.2.11 container_name: irods_test restart: always ports: diff --git a/test/testcases/fs_test.go b/test/testcases/fs_test.go index 18480a2..e402ddc 100644 --- a/test/testcases/fs_test.go +++ b/test/testcases/fs_test.go @@ -20,6 +20,7 @@ func TestFS(t *testing.T) { t.Run("test ListEntriesByMeta", testListEntriesByMeta) t.Run("test ListACLs", testListACLs) t.Run("test ReadWrite", testReadWrite) + t.Run("test CreateStat", testCreateStat) t.Run("test WriteRename", testWriteRename) t.Run("test WriteRenameDir", testWriteRenameDir) } @@ -31,13 +32,13 @@ func testListEntries(t *testing.T) { fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") - fs, err := fs.NewFileSystem(account, fsConfig) + filesystem, err := fs.NewFileSystem(account, fsConfig) assert.NoError(t, err) - defer fs.Release() + defer filesystem.Release() homedir := fmt.Sprintf("/%s/home/%s", account.ClientZone, account.ClientUser) - collections, err := fs.List(homedir) + collections, err := filesystem.List(homedir) assert.NoError(t, err) collectionPaths := []string{} @@ -62,9 +63,9 @@ func testListEntriesByMeta(t *testing.T) { fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") - fs, err := fs.NewFileSystem(account, fsConfig) + filesystem, err := fs.NewFileSystem(account, fsConfig) assert.NoError(t, err) - defer fs.Release() + defer filesystem.Release() for _, testFilePath := range GetTestFiles() { sha1sum := sha1.New() @@ -74,7 +75,7 @@ func testListEntriesByMeta(t *testing.T) { hashBytes := sha1sum.Sum(nil) hashString := hex.EncodeToString(hashBytes) - entries, err := fs.SearchByMeta("hash", hashString) + entries, err := filesystem.SearchByMeta("hash", hashString) assert.NoError(t, err) assert.Equal(t, 1, len(entries)) @@ -89,13 +90,13 @@ func testListACLs(t *testing.T) { fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") - fs, err := fs.NewFileSystem(account, fsConfig) + filesystem, err := fs.NewFileSystem(account, fsConfig) assert.NoError(t, err) - defer fs.Release() + defer filesystem.Release() objectPath := GetTestFiles()[0] - acls, err := fs.ListACLsWithGroupUsers(objectPath) + acls, err := filesystem.ListACLsWithGroupUsers(objectPath) assert.NoError(t, err) assert.GreaterOrEqual(t, len(acls), 1) @@ -111,9 +112,9 @@ func testReadWrite(t *testing.T) { fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") - fs, err := fs.NewFileSystem(account, fsConfig) + filesystem, err := fs.NewFileSystem(account, fsConfig) assert.NoError(t, err) - defer fs.Release() + defer filesystem.Release() homedir := fmt.Sprintf("/%s/home/%s", account.ClientZone, account.ClientUser) @@ -123,7 +124,7 @@ func testReadWrite(t *testing.T) { text := "HELLO WORLD!" // create - handle, err := fs.CreateFile(newDataObjectPath, "", "w") + handle, err := filesystem.CreateFile(newDataObjectPath, "", "w") assert.NoError(t, err) err = handle.Write([]byte(text)) @@ -132,10 +133,10 @@ func testReadWrite(t *testing.T) { err = handle.Close() assert.NoError(t, err) - assert.True(t, fs.Exists(newDataObjectPath)) + assert.True(t, filesystem.Exists(newDataObjectPath)) // read - newHandle, err := fs.OpenFile(newDataObjectPath, "", "r") + newHandle, err := filesystem.OpenFile(newDataObjectPath, "", "r") assert.NoError(t, err) readData, err := newHandle.Read(1024) @@ -147,10 +148,67 @@ func testReadWrite(t *testing.T) { assert.Equal(t, text, string(readData)) // delete - err = fs.RemoveFile(newDataObjectPath, true) + err = filesystem.RemoveFile(newDataObjectPath, true) assert.NoError(t, err) - assert.False(t, fs.Exists(newDataObjectPath)) + assert.False(t, filesystem.Exists(newDataObjectPath)) +} + +func testCreateStat(t *testing.T) { + account := GetTestAccount() + + account.ClientServerNegotiation = false + + fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") + + filesystem, err := fs.NewFileSystem(account, fsConfig) + assert.NoError(t, err) + defer filesystem.Release() + + homedir := fmt.Sprintf("/%s/home/%s", account.ClientZone, account.ClientUser) + + newDataObjectFilename := "testobj_create1234" + newDataObjectPath := homedir + "/" + newDataObjectFilename + + text := "HELLO WORLD" + + // create + handle, err := filesystem.CreateFile(newDataObjectPath, "", "w") + assert.NoError(t, err) + + // stat + stat, err := filesystem.Stat(newDataObjectPath) + assert.NoError(t, err) + assert.NotEmpty(t, stat.ID) + assert.Equal(t, fs.FileEntry, stat.Type) + + // write + err = handle.Write([]byte(text)) + assert.NoError(t, err) + + // close + err = handle.Close() + assert.NoError(t, err) + + assert.True(t, filesystem.Exists(newDataObjectPath)) + + // read + newHandle, err := filesystem.OpenFile(newDataObjectPath, "", "r") + assert.NoError(t, err) + + readData, err := newHandle.Read(1024) + assert.NoError(t, err) + + err = newHandle.Close() + assert.NoError(t, err) + + assert.Equal(t, text, string(readData)) + + // delete + err = filesystem.RemoveFile(newDataObjectPath, true) + assert.NoError(t, err) + + assert.False(t, filesystem.Exists(newDataObjectPath)) } func testWriteRename(t *testing.T) { @@ -160,9 +218,9 @@ func testWriteRename(t *testing.T) { fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") - fs, err := fs.NewFileSystem(account, fsConfig) + filesystem, err := fs.NewFileSystem(account, fsConfig) assert.NoError(t, err) - defer fs.Release() + defer filesystem.Release() homedir := fmt.Sprintf("/%s/home/%s", account.ClientZone, account.ClientUser) @@ -174,7 +232,7 @@ func testWriteRename(t *testing.T) { text2 := " WORLD!" // create - handle, err := fs.CreateFile(newDataObjectPath, "", "w") + handle, err := filesystem.CreateFile(newDataObjectPath, "", "w") assert.NoError(t, err) // write @@ -182,11 +240,9 @@ func testWriteRename(t *testing.T) { assert.NoError(t, err) // rename - err = fs.RenameFile(newDataObjectPath, newDataObjectPathRenameTarget) + err = filesystem.RenameFile(newDataObjectPath, newDataObjectPathRenameTarget) assert.NoError(t, err) - fmt.Printf("rename - %s done\n", newDataObjectPathRenameTarget) - // write again err = handle.Write([]byte(text2)) assert.NoError(t, err) @@ -195,10 +251,10 @@ func testWriteRename(t *testing.T) { err = handle.Close() assert.NoError(t, err) - assert.True(t, fs.Exists(newDataObjectPathRenameTarget)) + assert.True(t, filesystem.Exists(newDataObjectPathRenameTarget)) // read - newHandle, err := fs.OpenFile(newDataObjectPathRenameTarget, "", "r") + newHandle, err := filesystem.OpenFile(newDataObjectPathRenameTarget, "", "r") assert.NoError(t, err) readData, err := newHandle.Read(1024) @@ -210,10 +266,10 @@ func testWriteRename(t *testing.T) { assert.Equal(t, text1+text2, string(readData)) // delete - err = fs.RemoveFile(newDataObjectPathRenameTarget, true) + err = filesystem.RemoveFile(newDataObjectPathRenameTarget, true) assert.NoError(t, err) - assert.False(t, fs.Exists(newDataObjectPathRenameTarget)) + assert.False(t, filesystem.Exists(newDataObjectPathRenameTarget)) } func testWriteRenameDir(t *testing.T) { @@ -223,14 +279,14 @@ func testWriteRenameDir(t *testing.T) { fsConfig := fs.NewFileSystemConfigWithDefault("go-irodsclient-test") - fs, err := fs.NewFileSystem(account, fsConfig) + filesystem, err := fs.NewFileSystem(account, fsConfig) assert.NoError(t, err) - defer fs.Release() + defer filesystem.Release() homedir := fmt.Sprintf("/%s/home/%s", account.ClientZone, account.ClientUser) newdir := fmt.Sprintf("%s/testdir", homedir) - err = fs.MakeDir(newdir, true) + err = filesystem.MakeDir(newdir, true) assert.NoError(t, err) newDataObjectFilename := "testobj1234" @@ -243,7 +299,7 @@ func testWriteRenameDir(t *testing.T) { text2 := " WORLD!" // create - handle, err := fs.CreateFile(newDataObjectPath, "", "w") + handle, err := filesystem.CreateFile(newDataObjectPath, "", "w") assert.NoError(t, err) // write @@ -251,11 +307,9 @@ func testWriteRenameDir(t *testing.T) { assert.NoError(t, err) // rename - err = fs.RenameDir(newdir, newdirRenameTarget) + err = filesystem.RenameDir(newdir, newdirRenameTarget) assert.NoError(t, err) - fmt.Printf("rename dir - %s done\n", newdirRenameTarget) - // write again err = handle.Write([]byte(text2)) assert.NoError(t, err) @@ -264,10 +318,10 @@ func testWriteRenameDir(t *testing.T) { err = handle.Close() assert.NoError(t, err) - assert.True(t, fs.Exists(newDataObjectPathRenameTarget)) + assert.True(t, filesystem.Exists(newDataObjectPathRenameTarget)) // read - newHandle, err := fs.OpenFile(newDataObjectPathRenameTarget, "", "r") + newHandle, err := filesystem.OpenFile(newDataObjectPathRenameTarget, "", "r") assert.NoError(t, err) readData, err := newHandle.Read(1024) @@ -279,11 +333,11 @@ func testWriteRenameDir(t *testing.T) { assert.Equal(t, text1+text2, string(readData)) // delete - err = fs.RemoveFile(newDataObjectPathRenameTarget, true) + err = filesystem.RemoveFile(newDataObjectPathRenameTarget, true) assert.NoError(t, err) - assert.False(t, fs.Exists(newDataObjectPathRenameTarget)) + assert.False(t, filesystem.Exists(newDataObjectPathRenameTarget)) - err = fs.RemoveDir(newdirRenameTarget, true, true) + err = filesystem.RemoveDir(newdirRenameTarget, true, true) assert.NoError(t, err) }