Skip to content

Commit

Permalink
HPCC-31212 Add ListPlaneDirectory to fileservices
Browse files Browse the repository at this point in the history 
The ListPlaneDirectory is added to list DropZone files using
DropZone name and path. It should replace the existing
RemoteDirectory.
1. Add the fsListPlaneDirectory.
2. Move the checkExternalFileRights so that it can be used by
fsRemoteDirectory and fsListPlaneDirectory.
3. Add new listPlaneDirectory which contains the shared code
for fsRemoteDirectory and fsListPlaneDirectory.
4. Revise the fsRemoteDirectory.

Signed-off-by: wangkx <[email protected]>
  • Loading branch information
wangkx committed Feb 9, 2024
1 parent 31baac7 commit ff83485
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 35 deletions.
14 changes: 13 additions & 1 deletion ecllibrary/std/File.ecl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,19 @@ EXPORT SetFileDescription(varstring lfn, varstring val) :=
* @param recurse Whether to include files from subdirectories under the directory. Defaults to FALSE.
*/
EXPORT dataset(FsFilenameRecord) RemoteDirectory(varstring machineIP, varstring dir, varstring mask='*', boolean recurse=FALSE) :=
lib_fileservices.FileServices.RemoteDirectory(machineIP, dir, mask, recurse);
lib_fileservices.FileServices.RemoteDirectory(machineIP, dir, mask, recurse); //This function is replaced by ListPlaneDirectory.

/**
* Returns a dataset containing a list of files from the specified planeName and directory.
*
* @param planeName The name of the data plane containing the file.
* @param dir The path to the directory to read. This must be in the appropriate format for the operating
* system running on the remote machine.
* @param mask The filemask specifying which files to include in the result. Defaults to '*' (all files).
* @param recurse Whether to include files from subdirectories under the directory. Defaults to FALSE.
*/
EXPORT dataset(FsFilenameRecord) ListPlaneDirectory(varstring planeName, varstring dir, varstring mask='*', boolean recurse=FALSE) :=
lib_fileservices.FileServices.ListPlaneDirectory(planename, dir, mask, recurse);

/**
* Returns a dataset of information about the logical files known to the system.
Expand Down
102 changes: 70 additions & 32 deletions plugins/fileservices/fileservices.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2092,18 +2092,32 @@ FILESERVICES_API char * FILESERVICES_CALL fsGetFileDescription(ICodeContext *ct
return CTXSTRDUP(parentCtx, "");
}

FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(size32_t & __lenResult,void * & __result, const char *machine, const char *dir, const char *mask, bool sub)
static void checkExternalFileRights(ICodeContext *ctx,const char *scope,bool rd,bool wr)
{
MemoryBuffer mb;
RemoteFilename rfn;
SocketEndpoint ep(machine);
if (ep.isNull()){
if (machine)
throw MakeStringException(-1, "RemoteDirectory: Could not resolve host '%s'", machine);
ep.setLocalHost(0);
Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
unsigned auditflags = 0;
if (rd)
auditflags |= (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED);
if (wr)
auditflags |= (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_WRITE_WANTED);
SecAccessFlags perm = queryDistributedFileDirectory().getFScopePermissions(scope,udesc,auditflags);
if (wr) {
if (!HASWRITEPERMISSION(perm)) {
throw makeStringExceptionV(-1,"Write permission denied for scope %s", scope);
}
}
if (rd) {
if (!HASREADPERMISSION(perm)) {
throw makeStringExceptionV(-1,"Read permission denied for scope %s", scope);
}
}
}

rfn.setPath(ep,dir);
static void listPlaneDirectory(CDfsLogicalFileName &dlfn,size32_t &lenResult,void *&result,const char *mask,bool sub)
{
MemoryBuffer mb;
RemoteFilename rfn;
dlfn.getExternalFilename(rfn);
Owned<IFile> f = createIFile(rfn);
if (f) {
StringBuffer s;
Expand All @@ -2122,8 +2136,53 @@ FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(size32_t & __lenResult
}
}
}
__lenResult = mb.length();
__result = mb.detach();
lenResult = mb.length();
result = mb.detach();
}

FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(ICodeContext *ctx,size32_t &__lenResult,void *&__result,const char *machine,const char *dir,const char *mask,bool sub)
{
if (containsRelPaths(dir)) //Detect a path like: a/../../../f
throw makeStringExceptionV(-1,"Invalid file path %s",dir);

SocketEndpoint ep(machine);
if (ep.isNull()) {
if (machine)
throw MakeStringException(-1,"Could not resolve host '%s'",machine);
ep.setLocalHost(0);
}

StringBuffer ipText;
CDfsLogicalFileName dlfn;
dlfn.setExternal(ep.getHostText(ipText),dir);
checkExternalFileRights(ctx,dlfn.get(),true,false);

listPlaneDirectory(dlfn,__lenResult,__result,mask,sub);
}

FILESERVICES_API void FILESERVICES_CALL fsListPlaneDirectory(ICodeContext *ctx,size32_t &lenResult,
void *&result,const char *dropZoneName,const char *dir,const char *mask,bool sub)
{
if (containsRelPaths(dir)) //Detect a path like: a/../../../f
throw makeStringExceptionV(-1,"Invalid file path %s",dir);

Owned<IPropertyTree> dropZone = getDropZonePlane(dropZoneName);
if (!dropZone)
throw makeStringExceptionV(-1,"DropZone '%s' not found.",dropZoneName);

if (isAbsolutePath(dir))
{
const char *relativePath = getRelativePath(dir,dropZone->queryProp("@prefix"));
if (nullptr == relativePath)
throw makeStringExceptionV(-1,"Invalid DropZone path %s.",dir);
dir = relativePath;
}

CDfsLogicalFileName dlfn;
dlfn.setPlaneExternal(dropZoneName,dir);
checkExternalFileRights(ctx,dlfn.get(),true,false);

listPlaneDirectory(dlfn,lenResult,result,mask,sub);
}

FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *mask, bool includenormal, bool includesuper, bool unknownszero, const char *foreigndali)
Expand Down Expand Up @@ -2726,27 +2785,6 @@ FILESERVICES_API char * FILESERVICES_CALL fsfResolveHostName(const char *hostna
return ret.detach();
}

static void checkExternalFileRights(ICodeContext *ctx,const char *scope,bool rd,bool wr)
{
Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
unsigned auditflags = 0;
if (rd)
auditflags |= (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED);
if (wr)
auditflags |= (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_WRITE_WANTED);
SecAccessFlags perm = queryDistributedFileDirectory().getFScopePermissions(scope,udesc,auditflags);
if (wr) {
if (!HASWRITEPERMISSION(perm)) {
throw makeStringExceptionV(-1,"Write permission denied for scope %s", scope);
}
}
if (rd) {
if (!HASREADPERMISSION(perm)) {
throw makeStringExceptionV(-1,"Read permission denied for scope %s", scope);
}
}
}

static void checkExternalFilePath(ICodeContext *ctx,IPropertyTree *plane,const char *host,
const char *path,bool rd,bool wr,RemoteFilename &rfn)
{
Expand Down
3 changes: 2 additions & 1 deletion plugins/fileservices/fileservices.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,8 @@ FILESERVICES_API char * FILESERVICES_CALL fsfMonitorLogicalFileName(ICodeContex
FILESERVICES_API char * FILESERVICES_CALL fsfMonitorFile(ICodeContext *ctx, const char *eventname, const char *ip, const char *filename, bool sub, int shotcount, const char * espServerIpPort);
FILESERVICES_API void FILESERVICES_CALL fsSetFileDescription(ICodeContext *ctx, const char *logicalfilename, const char *value);
FILESERVICES_API char * FILESERVICES_CALL fsGetFileDescription(ICodeContext *ctx, const char *logicalfilename);
FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(size32_t & __lenResult,void * & __result, const char *machineip, const char *dir, const char *mask, bool sub);
FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *machineip, const char *dir, const char *mask, bool sub);
FILESERVICES_API void FILESERVICES_CALL fsListPlaneDirectory(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *planename, const char *dir, const char *mask, bool sub);
FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *mask, bool includenormal, bool includesuper, bool unknownszero,const char *foreigndali);
FILESERVICES_API void FILESERVICES_CALL fsSuperFileContents(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *lsuperlfn, bool recurse);
FILESERVICES_API void FILESERVICES_CALL fsLogicalFileSuperOwners(ICodeContext *ctx,size32_t & __lenResult,void * & __result, const char *lfn);
Expand Down
3 changes: 2 additions & 1 deletion plugins/proxies/lib_fileservices.ecllib
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ export FileServices := SERVICE : plugin('fileservices'), time
varstring fReplicate(const varstring logicalName, integer4 timeOut=-1, const varstring espServerIpPort=GETENV('ws_fs_server')) : c,action,context,entrypoint='fsfReplicate';
varstring GetFileDescription(const varstring lfn) : c,context,entrypoint='fsGetFileDescription';
SetFileDescription(const varstring lfn,const varstring val) : c,action,context,entrypoint='fsSetFileDescription';
dataset(FsFilenameRecord) RemoteDirectory(const varstring machineIP,const varstring dir,const varstring mask='*',boolean sub=false) : c,entrypoint='fsRemoteDirectory';
dataset(FsFilenameRecord) RemoteDirectory(const varstring machineIP,const varstring dir,const varstring mask='*',boolean sub=false) : c,context,entrypoint='fsRemoteDirectory';
dataset(FsFilenameRecord) ListPlaneDirectory(const varstring planeName,const varstring dir,const varstring mask='*',boolean sub=false) : c,context,entrypoint='fsListPlaneDirectory';
dataset(FsLogicalFileInfoRecord) LogicalFileList(const varstring namepattern='*',boolean includenormal=true,boolean includesuper=false,boolean unknownszero=false,const varstring foreigndali='') : c,context,entrypoint='fsLogicalFileList';
dataset(FsLogicalFileNameRecord) SuperFileContents(const varstring lsuperfn,boolean recurse=false) : c,context,entrypoint='fsSuperFileContents';
dataset(FsLogicalFileNameRecord) LogicalFileSuperOwners(const varstring lfn) : c,context,entrypoint='fsLogicalFileSuperOwners';
Expand Down

0 comments on commit ff83485

Please sign in to comment.