Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC-31689 Add checkfilesize to daliadmin #18582

Merged
merged 1 commit into from
Apr 26, 2024
Merged

HPCC-31689 Add checkfilesize to daliadmin #18582

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions dali/daliadmin/daadmin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2264,6 +2264,101 @@ void getxref(const char *dst)
conn->close();
}

void checkFileSizeOne(IUserDescriptor *user, const char *lfn, bool fix)
{
try
{
CDfsLogicalFileName dlfn;
dlfn.set(lfn);
Owned<IDistributedFile> dFile = queryDistributedFileDirectory().lookup(dlfn, user, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, 30000); // 30 sec timeout
if (dFile)
{
if (dFile->querySuperFile())
WARNLOG("Skipping: file '%s' is a superfile", lfn);
else
{
bool fileLocked = false;
COnScopeExit ensureFileUnlock([&]() { if (fileLocked) dFile->unlockProperties(); });
unsigned numParts = dFile->numParts();
for (unsigned p=0; p<numParts; p++)
{
IDistributedFilePart &part = dFile->queryPart(p);
IPropertyTree &attrs = part.queryAttributes();
if (!attrs.hasProp("@size"))
{
if (fix)
{
offset_t partSize;
try
{
partSize = part.getFileSize(true, true);
if (!fileLocked)
{
// we lock the file once, so that the individual part lock/unlocks are effectively a NOP
dFile->lockProperties(30000);
fileLocked = true;
PROGLOG("File '%s' has missing @size attributes", lfn);
}
part.lockProperties(30000);
}
catch (IException *e)
{
EXCLOG(e);
e->Release();
continue;
mckellyln marked this conversation as resolved.
Show resolved Hide resolved
}
COnScopeExit ensurePartUnlock([&]() { part.unlockProperties(); });
PROGLOG("Part %u: Setting @size to %" I64F "u", p+1, partSize);
attrs.setPropInt64("@size", partSize);
}
else
PROGLOG("File '%s' missing @size on part %u", lfn, p+1);
}
}
}
}
else
WARNLOG("File '%s' not found", lfn);
}
catch (IException *e)
{
EXCLOG(e);
e->Release();
}
}

void checkFileSize(IUserDescriptor *user, const char *lfnPattern, bool fix)
{
if (containsWildcard(lfnPattern))
{
unsigned count = 0;
Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator(lfnPattern, user, true, false); // no supers
CCycleTimer timer;
if (iter->first())
{
while (true)
{
IPropertyTree &attr = iter->query();
const char *lfn = attr.queryProp("@name");
checkFileSizeOne(user, lfn, fix);
++count;

if (!iter->next())
break;
else if (timer.elapsedCycles() >= queryOneSecCycles()*10) // log every 10 secs
{
PROGLOG("Processed %u files", count);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this tracing become excessive (e.g. if run as an esp service)?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is never intended to be run as a service - it's very specific to the missing @SiZe issue..
I think if it ever becomes something else, excessive tracing/other issues can be looked at then.

timer.reset();
}
}
}
PROGLOG("Total files processed %u files", count);
}
else
checkFileSizeOne(user, lfnPattern, fix);
}


struct CTreeItem : public CInterface
{
String *tail;
Expand Down
1 change: 1 addition & 0 deletions dali/daliadmin/daadmin.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ extern DALIADMIN_API void listmatches(const char *path, const char *match, const
extern DALIADMIN_API void dfsreplication(const char *clusterMask, const char *lfnMask, unsigned redundancy, bool dryRun);
extern DALIADMIN_API void migrateFiles(const char *srcGroup, const char *tgtGroup, const char *filemask, const char *_options);
extern DALIADMIN_API void getxref(const char *dst);
extern DALIADMIN_API void checkFileSize(IUserDescriptor *user, const char *lfnPattern, bool fix);

extern DALIADMIN_API void listworkunits(const char *test, const char *min, const char *max);
extern DALIADMIN_API void workunittimings(const char *wuid);
Expand Down
56 changes: 32 additions & 24 deletions dali/daliadmin/daliadmin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,49 +55,53 @@ void usage(const char *exe)
printf(" count <xpath> -- counts xpath matches\n");
printf("\n");
printf("Logical File meta information commands:\n");
printf(" dfsfile <logicalname> -- get meta information for file\n");
printf(" dfsmeta <logicalname> <storage> -- get new meta information for file\n");
printf(" setdfspartattr <logicalname> <part> <attribute> [<value>] -- set attribute of a file part to value, or delete the attribute if not provided\n");
printf(" dfspart <logicalname> <part> -- get meta information for part num\n");
printf(" checksuperfile <superfilename> [fix=true|false] -- check superfile links consistent and optionally fix\n");
printf(" checksubfile <subfilename> -- check subfile links to parent consistent\n");
printf(" checkfilesize <logicalfilemask> [fix=true|false] -- check file size attributes and optionally fix");
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NB: this is the new command, the other changes here are only alphabetic reordering

printf(" cleanscopes -- remove empty scopes\n");
printf(" clusternodes <clustername> [filename] -- get IPs for cluster group. Written to optional filename if provided\n");
printf(" dfscheck -- verify dfs file information is valid\n");
printf(" dfscompratio <logicalname> -- returns compression ratio of file\n");
printf(" dfscsv <logicalnamemask> -- get csv info. for files matching mask\n");
printf(" dfsexists <logicalname> -- sets return value to 0 if file exists\n");
printf(" dfsfile <logicalname> -- get meta information for file\n");
printf(" dfsgroup <logicalgroupname> [filename] -- get IPs for logical group (aka cluster). Written to optional filename if provided\n");
printf(" clusternodes <clustername> [filename] -- get IPs for cluster group. Written to optional filename if provided\n");
printf(" dfsls [<logicalname>] [options]-- get list of files within a scope (options=lrs)\n");
printf(" dfsmap <logicalname> -- get part files (primary and replicates)\n");
printf(" dfsexists <logicalname> -- sets return value to 0 if file exists\n");
printf(" dfsmeta <logicalname> <storage> -- get new meta information for file\n");
printf(" dfsparents <logicalname> -- list superfiles containing file\n");
printf(" dfspart <logicalname> <part> -- get meta information for part num\n");
printf(" dfsperm <logicalname> -- returns LDAP permission for file\n");
printf(" dfsreplication <clustermask> <logicalnamemask> <redundancy-count> [dryrun] -- set redundancy for files matching mask, on specified clusters only\n");
printf(" dfsscopes <mask> -- lists logical scopes (mask = * for all)\n");
printf(" dfsunlink <logicalname> -- unlinks file from all super parents\n");
printf(" dfsverify <logicalname> -- verifies parts exist, returns 0 if ok\n");
printf(" setprotect <logicalname> <id> -- overwrite protects logical file\n");
printf(" unprotect <logicalname> <id> -- unprotect (if id=* then clear all)\n");
printf(" listprotect <logicalnamemask> <id-mask> -- list protected files\n");
printf(" checksuperfile <superfilename> [fix=true|false] -- check superfile links consistent and optionally fix\n");
printf(" checksubfile <subfilename> -- check subfile links to parent consistent\n");
printf(" holdlock <logicalfile> <read|write> -- hold a lock to the logical-file until a key is pressed");
printf(" listexpires <logicalnamemask> -- lists logical files with expiry value\n");
printf(" listprotect <logicalnamemask> <id-mask> -- list protected files\n");
printf(" listrelationships <primary> <secondary>\n");
printf(" dfsperm <logicalname> -- returns LDAP permission for file\n");
printf(" dfscompratio <logicalname> -- returns compression ratio of file\n");
printf(" dfsscopes <mask> -- lists logical scopes (mask = * for all)\n");
printf(" cleanscopes -- remove empty scopes\n");
printf(" normalizefilenames [<logicalnamemask>] -- normalize existing logical filenames that match, e.g. .::.::scope::.::name -> scope::name\n");
printf(" dfsreplication <clustermask> <logicalnamemask> <redundancy-count> [dryrun] -- set redundancy for files matching mask, on specified clusters only\n");
printf(" holdlock <logicalfile> <read|write> -- hold a lock to the logical-file until a key is pressed");
printf(" setdfspartattr <logicalname> <part> <attribute> [<value>] -- set attribute of a file part to value, or delete the attribute if not provided\n");
printf(" setprotect <logicalname> <id> -- overwrite protects logical file\n");
printf(" unprotect <logicalname> <id> -- unprotect (if id=* then clear all)\n");
printf("\n");
printf("Workunit commands:\n");
printf(" listworkunits [<prop>=<val> [<lower> [<upper>]]] -- list workunits that match prop=val in workunit name range lower to upper\n");
printf(" listmatches <connection xpath> [<match xpath>=<val> [<property xpaths>]] -- <property xpaths> is comma separated list of xpaths\n");
printf(" workunittimings <WUID>\n");
printf("\n");
printf("Other dali server and misc commands:\n");
printf(" serverlist <mask> -- list server IPs (mask optional)\n");
printf(" clusterlist <mask> -- list clusters (mask optional)\n");
printf(" auditlog <fromdate> <todate> <match>\n");
printf(" cleanglobalwuid [dryrun] [noreconstruct]\n");
printf(" clusterlist <mask> -- list clusters (mask optional)\n");
printf(" coalesce -- force transaction coalesce\n");
printf(" mpping <server-ip> -- time MP connect\n");
printf(" dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
printf(" daliping [ <num> ] -- time dali server connect\n");
printf(" getxref <destxmlfile> -- get all XREF information\n");
printf(" dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
printf(" migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
printf(" mpping <server-ip> -- time MP connect\n");
printf(" serverlist <mask> -- list server IPs (mask optional)\n");
printf(" translatetoxpath logicalfile [File|SuperFile|Scope]\n");
printf(" unlock <xpath or logicalfile> <[path|file]> -- unlocks either matching xpath(s) or matching logical file(s), can contain wildcards\n");
printf(" validatestore [fix=<true|false>]\n"
" [verbose=<true|false>]\n"
Expand All @@ -106,9 +110,6 @@ void usage(const char *exe)
printf(" wuidcompress <wildcard> <type> -- scan workunits that match <wildcard> and compress resources of <type>\n");
printf(" wuiddecompress <wildcard> <type> -- scan workunits that match <wildcard> and decompress resources of <type>\n");
printf(" xmlsize <filename> [<percentage>] -- analyse size usage in xml file, display individual items above 'percentage' \n");
printf(" migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
printf(" translatetoxpath logicalfile [File|SuperFile|Scope]\n");
printf(" cleanglobalwuid [dryrun] [noreconstruct]\n");
printf("\n");
printf("Common options\n");
printf(" server=<dali-server-ip> -- server ip\n");
Expand Down Expand Up @@ -148,6 +149,8 @@ int main(int argc, const char* argv[])
StringBuffer tmps;
for (int i=1;i<argc;i++) {
const char *param = argv[i];
if (startsWith(param, "--"))
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NB: this is added so --config=/etc/config/.. can be added to the command line, which will be needed on a AKS system, to pickup plane configuration (from global) for path striping.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to add param to params here ?
params.append(param);

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok - I see now loadConfiguration() uses argv so we just skip param here.

continue; // handled by loadConfiguration
if ((memcmp(param,"server=",7)==0)||
(memcmp(param,"logfile=",8)==0)||
(memcmp(param,"rawlog=",7)==0)||
Expand Down Expand Up @@ -474,6 +477,11 @@ int main(int argc, const char* argv[])
CHECKPARAMS(1,1);
getxref(params.item(1));
}
else if (strieq(cmd,"checkfilesize")) {
CHECKPARAMS(1,1);
bool fix = props->getPropBool("fix");
checkFileSize(userDesc, params.item(1), fix);
}
else if (strieq(cmd,"dalilocks")) {
CHECKPARAMS(0,2);
bool filesonly = false;
Expand Down
Loading