HPCC-29215 Add DFU Copy Ensure functionality

Add an 'ensure' option to dfu copy request, which copies the file if necessary, but published the meta information only if the physical files already exist. Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Oct 20, 2023 · ba205f1 · ba205f1
1 parent a05215b
commit ba205f1
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 6 deletions.
diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp
@@ -1442,6 +1442,8 @@ class CDFUengine: public CInterface, implements IDFUengine
                 }
                 break;
             }
+
+            bool ensureLfnAlreadyPublished = false;
             // fill dstfile for commands that need it
             switch (cmd) {
             case DFUcmd_copymerge:
@@ -1542,6 +1544,14 @@ class CDFUengine: public CInterface, implements IDFUengine
                                 Owned<IDistributedFile> oldfile = wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE);
                                 if (oldfile)
                                 {
+                                    if (options->getEnsure())
+                                    {
+                                        // logical file already exists.
+                                        ensureLfnAlreadyPublished = true;
+                                        dstFile.setown(oldfile.getClear());
+                                        dstName.set(tmp);
+                                        break;
+                                    }
                                     StringBuffer reason;
                                     bool canRemove = oldfile->canRemove(reason);
                                     oldfile.clear();
@@ -1700,12 +1710,35 @@ class CDFUengine: public CInterface, implements IDFUengine
                             }
                         }
                         else {
-                            fsys.copy(srcFile,dstFile,recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid);
-                            if (!abortnotify.abortRequested()) {
-                                if (needrep)
-                                    replicating = true;
+                            bool performCopy = true;
+                            if (options->getEnsure())
+                            {
+                                if (ensureLfnAlreadyPublished)
+                                    performCopy = false;
                                 else
-                                    dstFile->attach(dstName.get(),userdesc);
+                                {
+                                    if (dstFile->existsPhysicalPartFiles(0))
+                                    {
+                                        dstFile->attach(dstName.get(), userdesc);
+                                        performCopy = false;
+                                    }
+                                }
+                                if (!performCopy)
+                                {
+                                    feedback.repmode=cProgressReporter::REPnone;
+                                    feedback.displaySummary(nullptr, 0);
+                                    Audit("COPYENSURE", userdesc, srcFile?srcName.str():nullptr, dstName.get());
+                                }
+                            }
+                            if (performCopy)
+                            {
+                                fsys.copy(srcFile,dstFile,recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid);
+                                if (!abortnotify.abortRequested()) {
+                                    if (needrep)
+                                        replicating = true;
+                                    else
+                                        dstFile->attach(dstName.get(),userdesc);
+                                }
                                 Audit("COPY",userdesc,srcFile?srcName.str():NULL,dstName.get());
                             }
                         }

diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp
@@ -2002,6 +2002,11 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
         return queryRoot()->getPropInt("@overwrite")!=0;
     }
 
+    bool getEnsure() const
+    {
+        return queryRoot()->getPropInt("@ensure")!=0;
+    }
+
     DFUreplicateMode getReplicateMode(StringBuffer &cluster, bool &repeatlast,bool &onlyrepeated) const
     {
         repeatlast = false;
@@ -2146,7 +2151,7 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
         queryRoot()->setPropInt("@throttle",val);
     }
 
-    void setTransferBufferSize(unsigned val)
+    void setTransferBufferSize(size32_t val)
     {
         queryRoot()->setPropInt("@transferBufferSize",val);
     }
@@ -2161,6 +2166,11 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions
         queryRoot()->setPropInt("@overwrite",val?1:0);
     }
 
+    void setEnsure(bool val=true)
+    {
+        queryRoot()->setPropInt("@ensure",val?1:0);
+    }
+
     void setReplicateMode(DFUreplicateMode val,const char *cluster=NULL,bool repeatlast=false,bool onlyrepeated=false)
     {
         queryRoot()->setPropInt("@replicatemode",(int)val);

diff --git a/dali/dfu/dfuwu.hpp b/dali/dfu/dfuwu.hpp
@@ -153,6 +153,7 @@ interface IConstDFUoptions : extends IInterface
     virtual size32_t getTransferBufferSize() const = 0;
     virtual bool getVerify() const = 0;
     virtual bool getOverwrite() const = 0;
+    virtual bool getEnsure() const = 0;
     virtual DFUreplicateMode getReplicateMode(StringBuffer &cluster, bool &repeatlast,bool &onlyrepeated) const = 0;
     virtual const char *queryPartFilter() const = 0;
     virtual bool getKeepHeader() const = 0;
@@ -195,6 +196,7 @@ interface IDFUoptions : extends IConstDFUoptions
     virtual void setTransferBufferSize(size32_t val) = 0;
     virtual void setVerify(bool val=true) = 0;
     virtual void setOverwrite(bool val=true) = 0;
+    virtual void setEnsure(bool val=true) = 0;
     virtual void setReplicateMode(DFUreplicateMode val,const char *cluster=NULL,bool repeatlast=false,bool onlyrepeated=false) = 0;
     virtual void setPartFilter(const char *filter) = 0;             // format n,n-n,n etc
     virtual void setKeepHeader(bool val=true) = 0;

diff --git a/esp/scm/ws_fs.ecm b/esp/scm/ws_fs.ecm
@@ -481,6 +481,7 @@ ESPrequest [nil_remove] Copy
     string srcusername;
     string srcpassword;
     bool   overwrite;
+    bool   ensure;
     bool   replicate;
     int    ReplicateOffset(1);
 

diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp
@@ -2790,6 +2790,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         wuFSpecDest->setLogicalName(dstname);
         wuFSpecDest->setFileMask(fileMask.str());
         wuOptions->setOverwrite(req.getOverwrite());
+        wuOptions->setEnsure(req.getEnsure());
         wuOptions->setPreserveCompression(req.getPreserveCompression());
         if (!req.getExpireDays_isNull())
             wuOptions->setExpireDays(req.getExpireDays());