From e711f3bbc60c3aa170116dd583baec35661be6a5 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 10 Oct 2024 11:19:21 +0200 Subject: [PATCH 01/56] Add ExitInfo --- src/libcommon/utility/types.h | 15 ++++++++++++++ test/libcommon/utility/testtypes.cpp | 30 ++++++++++++++++++++++++++++ test/libcommon/utility/testtypes.h | 2 ++ 3 files changed, 47 insertions(+) diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 6b61d593b..3a3e14ab8 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -219,6 +219,21 @@ enum class ExitCause { }; std::string toString(ExitCause e); +struct ExitInfo { + ExitInfo() = default; + ExitInfo(const ExitCode &code, const ExitCause &cause) : _code(code), _cause(cause) {} + ExitInfo(const ExitCode &code) : _code(code) {} + const ExitCode &code() const { return _code; } + const ExitCause &cause() const { return _cause; } + operator ExitCode() const { return _code; } + operator ExitCause() const { return _cause; } + operator bool() const { return _code == ExitCode::Ok; } + + private: + ExitCode _code{ExitCode::Unknown}; + ExitCause _cause{ExitCause::Unknown}; +}; + // Conflict types ordered by priority enum class ConflictType { None, diff --git a/test/libcommon/utility/testtypes.cpp b/test/libcommon/utility/testtypes.cpp index 398180679..2720b8f79 100644 --- a/test/libcommon/utility/testtypes.cpp +++ b/test/libcommon/utility/testtypes.cpp @@ -70,4 +70,34 @@ void TestTypes::testStreamConversion() { } CPPUNIT_ASSERT(previousLine.find("Test log of enumClass: Unknown(0)") != std::string::npos); } +void TestTypes::testExitInfo() { + ExitInfo ei; + ExitCode ec = ei; + ExitCause eca = ei; + CPPUNIT_ASSERT_EQUAL(ExitCode::Unknown, ei.code()); + CPPUNIT_ASSERT_EQUAL(ExitCode::Unknown, ec); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); + + ei = {ExitCode::Ok}; + ec = ei; + eca = ei; + CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, ei.code()); + CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, ec); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); + + ei = {ExitCode::NetworkError, ExitCause::ConnectionRefused}; + ec = ei; + eca = ei; + CPPUNIT_ASSERT_EQUAL(ExitCode::NetworkError, ei.code()); + CPPUNIT_ASSERT_EQUAL(ExitCode::NetworkError, ec); + CPPUNIT_ASSERT_EQUAL(ExitCause::ConnectionRefused, ei.cause()); + CPPUNIT_ASSERT_EQUAL(ExitCause::ConnectionRefused, eca); + + ec = ExitCode::BackError; + ei = ec; + CPPUNIT_ASSERT_EQUAL(ExitCode::BackError, ei.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); +} } // namespace KDC diff --git a/test/libcommon/utility/testtypes.h b/test/libcommon/utility/testtypes.h index 88a6e7bda..cfc693181 100644 --- a/test/libcommon/utility/testtypes.h +++ b/test/libcommon/utility/testtypes.h @@ -27,10 +27,12 @@ class TestTypes : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(TestTypes); CPPUNIT_TEST(testOtherSide); CPPUNIT_TEST(testStreamConversion); + CPPUNIT_TEST(testExitInfo); CPPUNIT_TEST_SUITE_END(); protected: void testOtherSide(); void testStreamConversion(); + void testExitInfo(); }; } // namespace KDC From c85f04608716e6a8922f1825f07fcf1caaf17559 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 10 Oct 2024 16:41:23 +0200 Subject: [PATCH 02/56] Fix testtypes for ExitInfo. --- test/libcommon/utility/testtypes.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/libcommon/utility/testtypes.cpp b/test/libcommon/utility/testtypes.cpp index 2720b8f79..a12dec925 100644 --- a/test/libcommon/utility/testtypes.cpp +++ b/test/libcommon/utility/testtypes.cpp @@ -87,13 +87,13 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); - ei = {ExitCode::NetworkError, ExitCause::ConnectionRefused}; + ei = {ExitCode::NetworkError, ExitCause::DbAccessError}; ec = ei; eca = ei; CPPUNIT_ASSERT_EQUAL(ExitCode::NetworkError, ei.code()); CPPUNIT_ASSERT_EQUAL(ExitCode::NetworkError, ec); - CPPUNIT_ASSERT_EQUAL(ExitCause::ConnectionRefused, ei.cause()); - CPPUNIT_ASSERT_EQUAL(ExitCause::ConnectionRefused, eca); + CPPUNIT_ASSERT_EQUAL(ExitCause::DbAccessError, ei.cause()); + CPPUNIT_ASSERT_EQUAL(ExitCause::DbAccessError, eca); ec = ExitCode::BackError; ei = ec; From db3ba37cf59ce2d02ba4e962c76a93a1a4e18eb6 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Wed, 16 Oct 2024 11:25:12 +0200 Subject: [PATCH 03/56] ExecutorWorker::initSyncFileItem return type from bool to void. --- src/libsyncengine/propagation/executor/executorworker.cpp | 6 +++--- src/libsyncengine/propagation/executor/executorworker.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 21474a91a..488105b8c 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -233,13 +233,13 @@ void ExecutorWorker::initProgressManager() { continue; // Do not notify UI of progress in case of pseudo conflicts } - if (initSyncFileItem(syncOp, syncItem)) { + initSyncFileItem(syncOp, syncItem); _syncPal->initProgress(syncItem); } } } -bool ExecutorWorker::initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem) { +void ExecutorWorker::initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem) { syncItem.setType(syncOp->affectedNode()->type()); syncItem.setConflict(syncOp->conflict().type()); syncItem.setInconsistency(syncOp->affectedNode()->inconsistencyType()); @@ -277,7 +277,7 @@ bool ExecutorWorker::initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem) } } - return true; + return; } void ExecutorWorker::logCorrespondingNodeErrorMsg(const SyncOpPtr syncOp) { diff --git a/src/libsyncengine/propagation/executor/executorworker.h b/src/libsyncengine/propagation/executor/executorworker.h index 26db3131b..e47625b0b 100644 --- a/src/libsyncengine/propagation/executor/executorworker.h +++ b/src/libsyncengine/propagation/executor/executorworker.h @@ -67,7 +67,7 @@ class ExecutorWorker : public OperationProcessor { private: void initProgressManager(); - bool initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem); + void initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem); void handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &hasError, bool &ignored); void checkAlreadyExcluded(const SyncPath &absolutePath, const NodeId &parentId); From 87daef38d35016c785c4f3671e8aee64e82170af Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Wed, 16 Oct 2024 15:27:37 +0200 Subject: [PATCH 04/56] Refactoring Error Handling with ExitInfo in ExecutorWorker. --- .../propagation/executor/executorworker.cpp | 785 ++++++------------ .../propagation/executor/executorworker.h | 59 +- test/server/workers/testworkers.cpp | 94 +-- 3 files changed, 340 insertions(+), 598 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 488105b8c..e61508f62 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -55,8 +55,7 @@ void ExecutorWorker::executorCallback(UniqueId jobId) { } void ExecutorWorker::execute() { - _executorExitCode = ExitCode::Unknown; - _executorExitCause = ExitCause::Unknown; + ExitInfo executorExitInfo = ExitCode::Ok; _snapshotToInvalidate = false; _jobToSyncOpMap.clear(); @@ -73,12 +72,11 @@ void ExecutorWorker::execute() { initProgressManager(); uint64_t changesCounter = 0; - bool hasError = false; while (!_opList.empty()) { // Same loop twice because we might reschedule the jobs after a pause TODO : refactor double loop // Create all the jobs while (!_opList.empty()) { - if (!deleteFinishedAsyncJobs()) { - hasError = true; + if (ExitInfo exitInfo = deleteFinishedAsyncJobs(); !exitInfo) { + executorExitInfo = exitInfo; cancelAllOngoingJobs(); break; } @@ -127,32 +125,30 @@ void ExecutorWorker::execute() { bool bypassProgressComplete = false; switch (syncOp->type()) { case OperationType::Create: { - handleCreateOp(syncOp, job, hasError, ignored); + executorExitInfo = handleCreateOp(syncOp, job, ignored); break; } case OperationType::Edit: { - handleEditOp(syncOp, job, hasError, ignored); + executorExitInfo = handleEditOp(syncOp, job, ignored); break; } case OperationType::Move: { - handleMoveOp(syncOp, hasError, ignored, bypassProgressComplete); + executorExitInfo = handleMoveOp(syncOp, ignored, bypassProgressComplete); break; } case OperationType::Delete: { - handleDeleteOp(syncOp, hasError, ignored, bypassProgressComplete); + executorExitInfo = handleDeleteOp(syncOp, ignored, bypassProgressComplete); break; } default: { LOGW_SYNCPAL_WARN(_logger, L"Unknown operation type: " << syncOp->type() << L" on file " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - hasError = true; + executorExitInfo = ExitCode::DataError; } } - if (hasError) { + if (!executorExitInfo) { if (!bypassProgressComplete) { setProgressComplete(syncOp, SyncFileStatus::Error); } @@ -192,11 +188,10 @@ void ExecutorWorker::execute() { } } - waitForAllJobsToFinish(hasError); - } - - if (!hasError) { - _executorExitCode = ExitCode::Ok; + if (ExitInfo exitInfo = waitForAllJobsToFinish(); !exitInfo) { + executorExitInfo = exitInfo; + break; + } } _syncPal->_syncOps->clear(); @@ -215,8 +210,8 @@ void ExecutorWorker::execute() { _syncPal->vfsCleanUpStatuses(); - setExitCause(_executorExitCause); - setDone(_executorExitCode); + setExitCause(executorExitInfo.cause()); + setDone(executorExitInfo.code()); LOG_SYNCPAL_DEBUG(_logger, "Worker stopped: name=" << name().c_str()); } @@ -234,8 +229,7 @@ void ExecutorWorker::initProgressManager() { } initSyncFileItem(syncOp, syncItem); - _syncPal->initProgress(syncItem); - } + _syncPal->initProgress(syncItem); } } @@ -303,15 +297,13 @@ void ExecutorWorker::setProgressComplete(const SyncOpPtr syncOp, SyncFileStatus _syncPal->setProgressComplete(relativeLocalFilePath, status); } -// !!! When returning with hasError == true, _executorExitCode and _executorExitCause must be set !!! -void ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &hasError, bool &ignored) { +ExitInfo ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &ignored) { // The execution of the create operation consists of three steps: // 1. If omit-flag is False, propagate the file or directory to target replica, because the object is missing there. // 2. Insert a new entry into the database, to avoid that the object is detected again by compute_ops() on the next sync // iteration. // 3. Update the update tree structures to ensure that follow-up operations can execute correctly, as they are based on the // information in these structures. - hasError = false; ignored = false; SyncPath relativeLocalFilePath = syncOp->nodePath(ReplicaSide::Local); @@ -319,11 +311,9 @@ void ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptrlocalPath() / relativeLocalFilePath; if (isLiteSyncActivated() && !syncOp->omit()) { bool isDehydratedPlaceholder = false; - if (!checkLiteSyncInfoForCreate(syncOp, absoluteLocalFilePath, isDehydratedPlaceholder)) { + if (ExitInfo exitInfo = checkLiteSyncInfoForCreate(syncOp, absoluteLocalFilePath, isDehydratedPlaceholder); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Error in checkLiteSyncInfoForCreate"); - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + return exitInfo; } if (isDehydratedPlaceholder) { @@ -335,53 +325,40 @@ void ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptrdeleteNode(syncOp->affectedNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: affectedNode name=" << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - hasError = true; - return; + return ExitCode::DataError; } if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { logCorrespondingNodeErrorMsg(syncOp); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - hasError = true; - return; + return ExitCode::DataError; } - return; + return ExitCode::Ok; } } if (syncOp->omit()) { // Do not generate job, only push changes in DB and update tree std::shared_ptr node; - if (!propagateCreateToDbAndTree( + if (ExitInfo exitInfo = propagateCreateToDbAndTree( syncOp, syncOp->correspondingNode()->id().has_value() ? *syncOp->correspondingNode()->id() : std::string(), - syncOp->affectedNode()->lastmodified(), node)) { + syncOp->affectedNode()->lastmodified(), node); + !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + return exitInfo; } } else { if (!isLiteSyncActivated() && !enoughLocalSpace(syncOp)) { - _syncPal->addError(Error(_syncPal->syncDbId(), name(), _executorExitCode, _executorExitCause)); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::NotEnoughDiskSpace; - hasError = true; - return; + _syncPal->addError(Error(_syncPal->syncDbId(), name(), ExitCode::SystemError, ExitCause::NotEnoughDiskSpace)); + return {ExitCode::SystemError, ExitCause::NotEnoughDiskSpace}; } bool exists = false; if (!hasRight(syncOp, exists)) { if (syncOp->targetSide() == ReplicaSide::Remote) { if (!exists) { - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::UnexpectedFileSystemEvent; - hasError = true; - return; + return {ExitCode::DataError, ExitCause::UnexpectedFileSystemEvent}; } // Ignore operation @@ -399,26 +376,21 @@ void ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptrdeleteNode(syncOp->affectedNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - hasError = true; - return; + return ExitCode::DataError; } } ignored = true; - return; + return ExitCode::Ok; } - if (!generateCreateJob(syncOp, job)) { - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + if (ExitInfo exitInfo = generateCreateJob(syncOp, job); !exitInfo) { + return exitInfo; } if (job && syncOp->affectedNode()->type() == NodeType::Directory) { // Propagate the directory creation immediately in order to avoid blocking other dependant job creation - if (!runCreateDirJob(syncOp, job)) { + if (ExitInfo exitInfo = runCreateDirJob(syncOp, job); !exitInfo) { std::shared_ptr createDirJob = std::dynamic_pointer_cast(job); if (createDirJob && (createDirJob->getStatusCode() == Poco::Net::HTTPResponse::HTTP_BAD_REQUEST || createDirJob->getStatusCode() == Poco::Net::HTTPResponse::HTTP_FORBIDDEN)) { @@ -426,31 +398,26 @@ void ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptrtargetSide() == ReplicaSide::Local) { - _executorExitCode = job->exitCode() == ExitCode::NeedRestart ? ExitCode::DataError : job->exitCode(); - _executorExitCause = - job->exitCode() == ExitCode::NeedRestart ? ExitCause::FileAlreadyExist : job->exitCause(); + return job->exitCode() == ExitCode::NeedRestart ? ExitInfo(ExitCode::DataError, ExitCause::FileAlreadyExist) + : ExitInfo(job->exitCode(), job->exitCause()); } else if (syncOp->targetSide() == ReplicaSide::Remote) { - _executorExitCode = job->exitCode() == ExitCode::NeedRestart ? ExitCode::BackError : job->exitCode(); - _executorExitCause = - job->exitCode() == ExitCode::NeedRestart ? ExitCause::FileAlreadyExist : job->exitCause(); + return job->exitCode() == ExitCode::NeedRestart ? ExitInfo(ExitCode::BackError, ExitCause::FileAlreadyExist) + : ExitInfo(job->exitCode(), job->exitCause()); } - - hasError = true; - return; + return ExitCode::LogicError; } - _executorExitCode = - convertToPlaceholder(relativeLocalFilePath, syncOp->targetSide() == ReplicaSide::Remote, _executorExitCause); - if (_executorExitCode != ExitCode::Ok) { + if (ExitInfo exitInfo = convertToPlaceholder(relativeLocalFilePath, syncOp->targetSide() == ReplicaSide::Remote); + !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to convert to placeholder for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - hasError = true; - return; + return exitInfo; } job.reset(); } } + return ExitCode::Ok; } void ExecutorWorker::checkAlreadyExcluded(const SyncPath &absolutePath, const NodeId &parentId) { @@ -488,13 +455,10 @@ void ExecutorWorker::checkAlreadyExcluded(const SyncPath &absolutePath, const No // The item already exist, exclude it PlatformInconsistencyCheckerUtility::renameLocalFile(absolutePath, PlatformInconsistencyCheckerUtility::SuffixTypeBlacklisted); - - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::FileAlreadyExist; } // !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr &job) noexcept { +ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr &job) noexcept { // 1. If omit-flag is False, propagate the file or directory to replica Y, because the object is missing there. std::shared_ptr newCorrespondingParentNode = nullptr; if (affectedUpdateTree(syncOp)->rootNode() == syncOp->affectedNode()->parentNode()) { @@ -502,9 +466,7 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->parentNode()); @@ -512,9 +474,7 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } } @@ -529,11 +489,10 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->name()).c_str()); - return false; + return exitInfo; } FileStat fileStat; @@ -541,30 +500,24 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr newNode = nullptr; - if (!propagateCreateToDbAndTree(syncOp, std::to_string(fileStat.inode), syncOp->affectedNode()->lastmodified(), - newNode)) { + if (ExitInfo exitInfo = propagateCreateToDbAndTree(syncOp, std::to_string(fileStat.inode), + syncOp->affectedNode()->lastmodified(), newNode); + !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - // _executorExitCode and _executorExitCause are set by the above function - return false; + return exitInfo; } // Check for inconsistency @@ -588,9 +541,12 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr( @@ -615,9 +571,7 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptrsetAffectedFilePath(relativeLocalFilePath); @@ -628,9 +582,8 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptrnodePath(ReplicaSide::Local); SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; if (syncOp->affectedNode()->type() == NodeType::Directory) { - _executorExitCode = - convertToPlaceholder(relativeLocalFilePath, syncOp->targetSide() == ReplicaSide::Remote, _executorExitCause); - if (_executorExitCode != ExitCode::Ok) { + if (ExitInfo exitInfo = convertToPlaceholder(relativeLocalFilePath, syncOp->targetSide() == ReplicaSide::Remote); + !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to convert to placeholder for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); _syncPal->setRestart(true); @@ -640,7 +593,7 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->name()).c_str()); } - return false; + return exitInfo; } try { @@ -651,25 +604,20 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->name()).c_str()); - return false; + return exitInfo; } uint64_t filesize = 0; - if (!getFileSize(absoluteLocalFilePath, filesize)) { + if (ExitInfo exitInfo = getFileSize(absoluteLocalFilePath, filesize); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Error in ExecutorWorker::getFileSize for " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return exitInfo; } if (filesize > useUploadSessionThreshold) { @@ -683,9 +631,8 @@ bool ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptrsetVfsForceStatusCallback(vfsForceStatusCallback); } - return true; + return ExitCode::Ok; } // !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const SyncPath &path, bool &isDehydratedPlaceholder) { +ExitInfo ExecutorWorker::checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const SyncPath &path, bool &isDehydratedPlaceholder) { isDehydratedPlaceholder = false; if (syncOp->targetSide() == ReplicaSide::Remote) { if (syncOp->affectedNode()->type() == NodeType::Directory) { - return true; + return ExitCode::Ok; } bool isPlaceholder = false; @@ -749,9 +694,7 @@ bool ExecutorWorker::checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const SyncPath int progress = 0; if (!_syncPal->vfsStatus(path, isPlaceholder, isHydrated, isSyncing, progress)) { LOGW_SYNCPAL_WARN(_logger, L"Error in vfsStatus: " << Utility::formatSyncPath(path).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } if (isPlaceholder && !isHydrated && !isSyncing) { @@ -760,18 +703,15 @@ bool ExecutorWorker::checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const SyncPath } } - return true; + return ExitCode::Ok; } -ExitCode ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath, ExitCause &exitCause) { - exitCause = ExitCause::Unknown; - +ExitInfo ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath) { SyncFileItem syncItem; if (!_syncPal->getSyncFileItem(relativeLocalPath, syncItem)) { LOGW_SYNCPAL_WARN(_logger, L"Failed to retrieve SyncFileItem associated to item: " << Utility::formatSyncPath(relativeLocalPath).c_str()); - exitCause = ExitCause::InvalidSnapshot; - return ExitCode::DataError; + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } if (!_syncPal->vfsCreatePlaceholder(relativeLocalPath, syncItem)) { // TODO: vfs functions should output an ioError parameter @@ -788,13 +728,11 @@ ExitCode ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath, Ex if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - exitCause = ExitCause::NoSearchPermission; - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } if (exists) { - exitCause = ExitCause::InvalidSnapshot; - return ExitCode::DataError; + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } else { // Check if the parent folder exists on local replica bool parentExists = false; @@ -807,32 +745,26 @@ ExitCode ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath, Ex if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(absoluteLocalFilePath.parent_path()).c_str()); - exitCause = ExitCause::NoSearchPermission; - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } if (!parentExists) { - exitCause = ExitCause::InvalidSnapshot; - return ExitCode::DataError; + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } } - exitCause = ExitCause::FileAccessError; return ExitCode::SystemError; } return ExitCode::Ok; } -ExitCode ExecutorWorker::convertToPlaceholder(const SyncPath &relativeLocalPath, bool hydrated, ExitCause &exitCause) { - exitCause = ExitCause::Unknown; - +ExitInfo ExecutorWorker::convertToPlaceholder(const SyncPath &relativeLocalPath, bool hydrated) { SyncFileItem syncItem; if (!_syncPal->getSyncFileItem(relativeLocalPath, syncItem)) { LOGW_SYNCPAL_WARN(_logger, L"Failed to retrieve SyncFileItem associated to item: " << Utility::formatSyncPath(relativeLocalPath).c_str()); - exitCause = ExitCause::InvalidSnapshot; - return ExitCode::DataError; + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalPath; @@ -852,12 +784,10 @@ ExitCode ExecutorWorker::convertToPlaceholder(const SyncPath &relativeLocalPath, if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_SYNCPAL_WARN(_logger, L"Item does not exist anymore: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - exitCause = ExitCause::InvalidSnapshot; - return ExitCode::DataError; + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } else if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - exitCause = ExitCause::NoSearchPermission; - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } syncItem.setLocalNodeId(std::to_string(fileStat.inode)); @@ -878,106 +808,91 @@ ExitCode ExecutorWorker::convertToPlaceholder(const SyncPath &relativeLocalPath, if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - exitCause = ExitCause::NoSearchPermission; - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } if (!exists) { - exitCause = ExitCause::InvalidSnapshot; - return ExitCode::DataError; + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } - exitCause = ExitCause::FileAccessError; return ExitCode::SystemError; } if (!_syncPal->vfsSetPinState(absoluteLocalFilePath, hydrated ? PinState::AlwaysLocal : PinState::OnlineOnly)) { LOGW_SYNCPAL_WARN(_logger, L"Error in vfsSetPinState: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - exitCause = ExitCause::FileAccessError; - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } return ExitCode::Ok; } // !!! When returning with hasError == true, _executorExitCode and _executorExitCause must be set !!! -void ExecutorWorker::handleEditOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &hasError, bool &ignored) { +ExitInfo ExecutorWorker::handleEditOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &ignored) { // The execution of the edit operation consists of three steps: // 1. If omit-flag is False, propagate the file to replicaY, replacing the existing one. // 2. Insert a new entry into the database, to avoid that the object is detected again by compute_ops() on the next sync // iteration. // 3. If the omit flag is False, update the updatetreeY structure to ensure that follow-up operations can execute correctly, // as they are based on the information in this structure. - hasError = false; ignored = false; SyncPath relativeLocalFilePath = syncOp->nodePath(ReplicaSide::Local); if (relativeLocalFilePath.empty()) { - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - hasError = true; - return; + return ExitCode::DataError; } if (isLiteSyncActivated()) { SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; bool ignoreItem = false; bool isSyncing = false; - if (!checkLiteSyncInfoForEdit(syncOp, absoluteLocalFilePath, ignoreItem, isSyncing)) { + if (ExitInfo exitInfo = checkLiteSyncInfoForEdit(syncOp, absoluteLocalFilePath, ignoreItem, isSyncing); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Error in checkLiteSyncInfoForEdit"); - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + return exitInfo; } if (ignoreItem) { ignored = true; - return; + return ExitCode::Ok; } if (isSyncing) { - return; + return ExitCode::Ok; } } if (syncOp->omit()) { // Do not generate job, only push changes in DB and update tree std::shared_ptr node; - if (!propagateEditToDbAndTree( + if (ExitInfo exitInfo = propagateEditToDbAndTree( syncOp, syncOp->correspondingNode()->id().has_value() ? *syncOp->correspondingNode()->id() : std::string(), - syncOp->affectedNode()->lastmodified(), node)) { + syncOp->affectedNode()->lastmodified(), node); + !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + return exitInfo; } } else { if (!enoughLocalSpace(syncOp)) { - _syncPal->addError(Error(_syncPal->syncDbId(), name(), _executorExitCode, _executorExitCause)); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::NotEnoughDiskSpace; - hasError = true; - return; + _syncPal->addError(Error(_syncPal->syncDbId(), name(), ExitCode::SystemError, ExitCause::NotEnoughDiskSpace)); + return {ExitCode::SystemError, ExitCause::NotEnoughDiskSpace}; } bool exists = false; if (!hasRight(syncOp, exists)) { ignored = true; - return; + return ExitCode::Ok; } - if (!generateEditJob(syncOp, job)) { - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + if (ExitInfo exitInfo = generateEditJob(syncOp, job); !exitInfo) { + return exitInfo; } } + return ExitCode::Ok; } // !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptr &job) { +ExitInfo ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptr &job) { // 1. If omit-flag is False, propagate the file to replicaY, replacing the existing one. if (syncOp->targetSide() == ReplicaSide::Local) { SyncPath relativeLocalFilePath = syncOp->nodePath(ReplicaSide::Local); @@ -992,9 +907,7 @@ bool ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptrsetAffectedFilePath(relativeLocalFilePath); @@ -1024,24 +937,19 @@ bool ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptrlocalPath() / relativeLocalFilePath; uint64_t filesize; - if (!getFileSize(absoluteLocalFilePath, filesize)) { + if (ExitInfo exitInfo = getFileSize(absoluteLocalFilePath, filesize); !exitInfo) { LOGW_WARN(_logger, L"Error in ExecutorWorker::getFileSize for " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return exitInfo; } if (!syncOp->correspondingNode()->id()) { // Should not happen LOGW_SYNCPAL_WARN(_logger, L"Edit operation with empty corresponding node id for " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; SentryHandler::instance()->captureMessage(SentryLevel::Warning, "ExecutorWorker::generateEditJob", "Edit operation with empty corresponding node id"); - - return false; + return ExitCode::DataError; } if (filesize > useUploadSessionThreshold) { @@ -1055,9 +963,7 @@ bool ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptrsetAffectedFilePath(relativeLocalFilePath); } - - return true; + return ExitCode::Ok; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::fixModificationDate(SyncOpPtr syncOp, const SyncPath &absolutePath) { +ExitInfo ExecutorWorker::fixModificationDate(SyncOpPtr syncOp, const SyncPath &absolutePath) { const auto id = syncOp->affectedNode()->id().has_value() ? syncOp->affectedNode()->id().value() : ""; LOGW_SYNCPAL_DEBUG(_logger, L"Do not upload dehydrated placeholders: " << Utility::formatSyncPath(absolutePath).c_str() << L" (" << Utility::s2ws(id).c_str() << L")"); @@ -1100,15 +1002,11 @@ bool ExecutorWorker::fixModificationDate(SyncOpPtr syncOp, const SyncPath &absol DbNode dbNode; if (!_syncPal->_syncDb->node(*syncOp->correspondingNode()->idb(), dbNode, found)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::node"); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } if (!found) { LOG_SYNCPAL_DEBUG(_logger, "Failed to retrieve node for dbId=" << *syncOp->correspondingNode()->idb()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } bool exists = false; @@ -1121,11 +1019,11 @@ bool ExecutorWorker::fixModificationDate(SyncOpPtr syncOp, const SyncPath &absol << Utility::formatSyncPath(absolutePath).c_str()); } // If file does not exist anymore, do nothing special. This is fine, it will not generate EDIT operations anymore. - return true; + return ExitCode::Ok; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPath &absolutePath, bool &ignoreItem, bool &isSyncing) { +ExitInfo ExecutorWorker::checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPath &absolutePath, bool &ignoreItem, + bool &isSyncing) { ignoreItem = false; bool isPlaceholder = false; @@ -1134,34 +1032,27 @@ bool ExecutorWorker::checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPath & int progress = 0; if (!_syncPal->vfsStatus(absolutePath, isPlaceholder, isHydrated, isSyncingTmp, progress)) { LOGW_SYNCPAL_WARN(_logger, L"Error in vfsStatus: " << Utility::formatSyncPath(absolutePath).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } if (syncOp->targetSide() == ReplicaSide::Remote) { if (isPlaceholder && !isHydrated) { ignoreItem = true; return fixModificationDate(syncOp, absolutePath); - // _executorExitCode and _executorExitCause are set by the above function } } else { if (isPlaceholder) { PinState pinState = PinState::Unspecified; if (!_syncPal->vfsPinState(absolutePath, pinState)) { LOGW_SYNCPAL_WARN(_logger, L"Error in vfsPinState for file: " << Utility::formatSyncPath(absolutePath).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::InconsistentPinState; - return false; + return {ExitCode::SystemError, ExitCause::InconsistentPinState}; } switch (pinState) { case PinState::Inherited: { // TODO : what do we do in that case?? LOG_SYNCPAL_WARN(_logger, "Inherited pin state not implemented yet"); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::LogicError; } case PinState::AlwaysLocal: { if (isSyncingTmp) { @@ -1188,24 +1079,22 @@ bool ExecutorWorker::checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPath & default: { LOGW_SYNCPAL_DEBUG(_logger, L"Ignore EDIT for file: " << Path2WStr(absolutePath).c_str()); ignoreItem = true; - return true; + return ExitCode::Ok; } } } } - return true; + return ExitCode::Ok; } -// !!! When returning with hasError == true, _executorExitCode and _executorExitCause must be set !!! -void ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &hasError, bool &ignored, bool &bypassProgressComplete) { +ExitInfo ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { // The three execution steps are as follows: // 1. If omit-flag is False, move the object on replica Y (where it still needs to be moved) from uY to vY, changing the name // to nameX. // 2. Update the database entry, to avoid detecting the move operation again. // 3. If the omit flag is False, update the updatetreeY structure to ensure that follow-up operations can execute correctly, // as they are based on the information in this structure. - hasError = false; ignored = false; bypassProgressComplete = false; @@ -1213,8 +1102,7 @@ void ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &hasError, bool &ignore // Do not generate job, only push changes in DB and update tree if (syncOp->hasConflict()) { bool propagateChange = true; - hasError = propagateConflictToDbAndTree(syncOp, propagateChange); - // _executorExitCode and _executorExitCause are set by the above function + ExitInfo exitInfo = propagateConflictToDbAndTree(syncOp, propagateChange); Error err(_syncPal->syncDbId(), syncOp->conflict().localNode() != nullptr @@ -1232,35 +1120,31 @@ void ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &hasError, bool &ignore _syncPal->addError(err); - if (!propagateChange || hasError) { - return; + if (!propagateChange || !exitInfo) { + return exitInfo; } } - if (!propagateMoveToDbAndTree(syncOp)) { + if (ExitInfo exitInfo = propagateMoveToDbAndTree(syncOp); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + return exitInfo; } } else { bool exists = false; if (!hasRight(syncOp, exists)) { ignored = true; - return; + return ExitCode::Ok; } - if (!generateMoveJob(syncOp, ignored, bypassProgressComplete)) { - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + if (ExitInfo exitInfo = generateMoveJob(syncOp, ignored, bypassProgressComplete); !exitInfo) { + return exitInfo; } } + return ExitCode::Ok; } -// !!! When returning with hasError == true, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { +ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { bypassProgressComplete = false; // 1. If omit-flag is False, move the object on replica Y (where it still needs to be moved) from uY to vY, changing the name @@ -1277,9 +1161,7 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa if (!correspondingNode) { LOGW_SYNCPAL_WARN(_logger, L"Corresponding node not found for item with " << Utility::formatSyncPath(syncOp->affectedNode()->getPath()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } // Get the new parent node @@ -1288,9 +1170,7 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa if (!parentNode) { LOGW_SYNCPAL_WARN(_logger, L"Parent node not found for item with " << Utility::formatSyncPath(correspondingNode->getPath()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } relativeDestLocalFilePath = parentNode->getPath() / syncOp->newName(); @@ -1305,9 +1185,7 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa if (!correspondingNode) { LOGW_SYNCPAL_WARN(_logger, L"Corresponding node not found for item " << Utility::formatSyncPath(syncOp->affectedNode()->getPath()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } // Get the new parent node @@ -1316,9 +1194,7 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa if (!parentNode) { LOGW_SYNCPAL_WARN(_logger, L"Parent node not found for item " << Utility::formatSyncPath(correspondingNode->getPath()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } relativeDestLocalFilePath = parentNode->getPath() / syncOp->newName(); @@ -1344,9 +1220,7 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa : correspondingNodeInOtherTree(parentNode); if (!remoteParentNode) { LOGW_SYNCPAL_WARN(_logger, L"Parent node not found for item " << Path2WStr(parentNode->getPath()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } job = std::make_shared(_syncPal->driveDbId(), absoluteDestLocalFilePath, correspondingNode->id().has_value() ? *correspondingNode->id() : std::string(), @@ -1373,9 +1247,7 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa } catch (std::exception const &e) { LOGW_SYNCPAL_WARN(_logger, L"Error in MoveJob::MoveJob for driveDbId=" << _syncPal->driveDbId() << L" : " << Utility::s2ws(e.what()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } } @@ -1386,9 +1258,9 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa // Conflict fixing job finished successfully // Propagate changes to DB and update trees std::shared_ptr newNode = nullptr; - if (!propagateChangeToDbAndTree(syncOp, job, newNode)) { - // _executorExitCode and _executorExitCause are set by the above function - return false; + if (ExitInfo exitInfo = propagateChangeToDbAndTree(syncOp, job, newNode); !exitInfo) { + LOG_WARN(_logger, "Failed to propagate changes in DB or update tree for: " << syncOp->affectedNode()->name()); + return exitInfo; } // Send conflict notification @@ -1408,22 +1280,19 @@ bool ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypa _syncPal->addError(err); } - return true; + return ExitCode::Ok; } return handleFinishedJob(job, syncOp, syncOp->affectedNode()->getPath(), ignored, bypassProgressComplete); - // _executorExitCode and _executorExitCause are set by the above function } -// !!! When returning with hasError == true, _executorExitCode and _executorExitCause must be set !!! -void ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &hasError, bool &ignored, bool &bypassProgressComplete) { +ExitInfo ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { // The three execution steps are as follows: // 1. If omit-flag is False, delete the file or directory on replicaY, because the objects till exists there // 2. Remove the entry from the database. If nX is a directory node, also remove all entries for each node n ∈ S. This avoids // that the object(s) are detected again by compute_ops() on the next sync iteration // 3. Update the update tree structures to ensure that follow-up operations can execute correctly, as they are based on the // information in these structures - hasError = false; ignored = false; bypassProgressComplete = false; @@ -1433,8 +1302,7 @@ void ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &hasError, bool &igno syncOp->conflict().type() != ConflictType::EditDelete) { // Error message handled with move operation in case Edit-Delete conflict bool propagateChange = true; - hasError = propagateConflictToDbAndTree(syncOp, propagateChange); - // _executorExitCode and _executorExitCause are set by the above function + ExitInfo exitInfo = propagateConflictToDbAndTree(syncOp, propagateChange); Error err(_syncPal->syncDbId(), syncOp->conflict().localNode() != nullptr @@ -1452,35 +1320,32 @@ void ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &hasError, bool &igno _syncPal->addError(err); - if (!propagateChange || hasError) { - return; + if (!propagateChange || !exitInfo) { + return exitInfo; } } - if (!propagateDeleteToDbAndTree(syncOp)) { - // _executorExitCode and _executorExitCause are set by the above function + if (ExitInfo exitInfo = propagateDeleteToDbAndTree(syncOp); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - hasError = true; - return; + return exitInfo; } } else { bool exists = false; if (!hasRight(syncOp, exists)) { ignored = true; - return; + return ExitCode::Ok; } - if (!generateDeleteJob(syncOp, ignored, bypassProgressComplete)) { - // _executorExitCode and _executorExitCause are set by the above function - hasError = true; - return; + if (ExitInfo exitInfo = generateDeleteJob(syncOp, ignored, bypassProgressComplete); !exitInfo) { + return exitInfo; } } + return ExitCode::Ok; } // !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { +ExitInfo ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { bypassProgressComplete = false; // 1. If omit-flag is False, delete the file or directory on replicaY, because the objects till exists there @@ -1496,9 +1361,7 @@ bool ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &by int progress = 0; if (!_syncPal->vfsStatus(absoluteLocalFilePath, isPlaceholder, isHydrated, isSyncing, progress)) { LOGW_SYNCPAL_WARN(_logger, L"Error in vfsStatus: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } isDehydratedPlaceholder = isPlaceholder && !isHydrated; } @@ -1506,9 +1369,7 @@ bool ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &by NodeId remoteNodeId = syncOp->affectedNode()->id().has_value() ? syncOp->affectedNode()->id().value() : ""; if (remoteNodeId.empty()) { LOGW_SYNCPAL_WARN(_logger, L"Failed to retrieve node ID"); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } job = std::make_shared(_syncPal->syncInfo(), relativeLocalFilePath, isDehydratedPlaceholder, remoteNodeId); @@ -1520,9 +1381,7 @@ bool ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &by } catch (std::exception const &e) { LOGW_SYNCPAL_WARN(_logger, L"Error in DeleteJob::DeleteJob for driveDbId=" << _syncPal->driveDbId() << L" : " << Utility::s2ws(e.what()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } } @@ -1534,7 +1393,6 @@ bool ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &by job->setAffectedFilePath(relativeLocalFilePath); job->runSynchronously(); return handleFinishedJob(job, syncOp, relativeLocalFilePath, ignored, bypassProgressComplete); - // _executorExitCode and _executorExitCause are set by the above function } bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { @@ -1687,7 +1545,7 @@ bool ExecutorWorker::enoughLocalSpace(SyncOpPtr syncOp) { return true; } -void ExecutorWorker::waitForAllJobsToFinish(bool &hasError) { +ExitInfo ExecutorWorker::waitForAllJobsToFinish() { while (!_ongoingJobs.empty()) { if (stopAsked()) { cancelAllOngoingJobs(); @@ -1707,21 +1565,21 @@ void ExecutorWorker::waitForAllJobsToFinish(bool &hasError) { } } - if (!deleteFinishedAsyncJobs()) { - hasError = true; + if (ExitInfo exitInfo = deleteFinishedAsyncJobs(); !exitInfo) { cancelAllOngoingJobs(); - break; + return exitInfo; } sendProgress(); } + return ExitCode::Ok; } -bool ExecutorWorker::deleteFinishedAsyncJobs() { - bool hasError = false; +ExitInfo ExecutorWorker::deleteFinishedAsyncJobs() { + ExitInfo exitInfo = ExitCode::Ok; while (!_terminatedJobs.empty()) { // Delete all terminated jobs - if (!hasError && _ongoingJobs.find(_terminatedJobs.front()) != _ongoingJobs.end()) { + if (exitInfo && _ongoingJobs.find(_terminatedJobs.front()) != _ongoingJobs.end()) { auto onGoingJobIt = _ongoingJobs.find(_terminatedJobs.front()); if (onGoingJobIt == _ongoingJobs.end()) { LOGW_SYNCPAL_WARN(_logger, L"Terminated job not found"); @@ -1743,12 +1601,14 @@ bool ExecutorWorker::deleteFinishedAsyncJobs() { SyncPath relativeLocalPath = syncOp->nodePath(ReplicaSide::Local); bool ignored = false; bool bypassProgressComplete = false; - if (!handleFinishedJob(job, syncOp, relativeLocalPath, ignored, bypassProgressComplete)) { + if (ExitInfo handleFinishedJobExitInfo = + handleFinishedJob(job, syncOp, relativeLocalPath, ignored, bypassProgressComplete); + !handleFinishedJobExitInfo) { increaseErrorCount(syncOp); - hasError = true; + exitInfo = handleFinishedJobExitInfo; } - if (!hasError) { + if (exitInfo) { if (ignored) { setProgressComplete(syncOp, SyncFileStatus::Ignored); } else { @@ -1771,21 +1631,15 @@ bool ExecutorWorker::deleteFinishedAsyncJobs() { } _terminatedJobs.pop(); } - - return !hasError; + return exitInfo; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::handleManagedBackError(ExitCause jobExitCause, SyncOpPtr syncOp, bool isInconsistencyIssue, - bool downloadImpossible) { - _executorExitCode = ExitCode::Ok; - +ExitInfo ExecutorWorker::handleManagedBackError(ExitCause jobExitCause, SyncOpPtr syncOp, bool isInconsistencyIssue, + bool downloadImpossible) { if (jobExitCause == ExitCause::NotFound && !downloadImpossible) { // The operation failed because the destination does not exist anymore - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; LOG_SYNCPAL_DEBUG(_logger, "Destination does not exist anymore, restarting sync."); - return false; + return ExitCode::DataError; } if (jobExitCause == ExitCause::QuotaExceeded) { @@ -1799,18 +1653,14 @@ bool ExecutorWorker::handleManagedBackError(ExitCause jobExitCause, SyncOpPtr sy if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { LOGW_SYNCPAL_WARN( _logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } if (syncOp->correspondingNode()) { if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } } @@ -1835,7 +1685,7 @@ bool ExecutorWorker::handleManagedBackError(ExitCause jobExitCause, SyncOpPtr sy } _syncPal->addError(error); - return true; + return ExitCode::Ok; } namespace details { @@ -1848,16 +1698,15 @@ bool isManagedBackError(ExitCause exitCause) { } } // namespace details -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::handleFinishedJob(std::shared_ptr job, SyncOpPtr syncOp, const SyncPath &relativeLocalPath, - bool &ignored, bool &bypassProgressComplete) { +ExitInfo ExecutorWorker::handleFinishedJob(std::shared_ptr job, SyncOpPtr syncOp, const SyncPath &relativeLocalPath, + bool &ignored, bool &bypassProgressComplete) { ignored = false; bypassProgressComplete = false; if (job->exitCode() == ExitCode::NeedRestart) { cancelAllOngoingJobs(); _syncPal->setRestart(true); - return true; + return ExitCode::Ok; } NodeId locaNodeId; @@ -1875,13 +1724,16 @@ bool ExecutorWorker::handleFinishedJob(std::shared_ptr job, SyncOpP job->exitCode() == ExitCode::BackError && details::isManagedBackError(job->exitCause())) { return handleManagedBackError(job->exitCause(), syncOp, isInconsistencyIssue, networkJob && networkJob->isDownloadImpossible()); - // _executorExitCode and _executorExitCause are set by the above function } if (job->exitCode() != ExitCode::Ok) { if (networkJob && (networkJob->getStatusCode() == Poco::Net::HTTPResponse::HTTP_FORBIDDEN || networkJob->getStatusCode() == Poco::Net::HTTPResponse::HTTP_CONFLICT)) { - handleForbiddenAction(syncOp, relativeLocalPath, ignored); + if (ExitInfo exitInfo = handleForbiddenAction(syncOp, relativeLocalPath, ignored); !exitInfo) { + LOGW_SYNCPAL_WARN(_logger, + L"Error in handleForbiddenAction for item: " << Utility::formatSyncPath(relativeLocalPath)); + return exitInfo; + } } else if (job->exitCode() == ExitCode::SystemError && (job->exitCause() == ExitCause::FileAccessError || job->exitCause() == ExitCause::MoveToTrashFailed)) { LOGW_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(relativeLocalPath).c_str() @@ -1896,36 +1748,29 @@ bool ExecutorWorker::handleFinishedJob(std::shared_ptr job, SyncOpP if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } if (syncOp->correspondingNode()) { if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } } } else { // Cancel all queued jobs LOGW_SYNCPAL_WARN(_logger, - L"Cancelling jobs. exit code: " << _executorExitCode << L" exit cause: " << _executorExitCause); + L"Cancelling jobs. exit code: " << job->exitCode() << L" exit cause: " << job->exitCause()); cancelAllOngoingJobs(); - _executorExitCode = job->exitCode(); - _executorExitCause = job->exitCause(); - return false; + return {job->exitCode(), job->exitCause()}; } } else { // Propagate changes to DB and update trees std::shared_ptr newNode; - if (!propagateChangeToDbAndTree(syncOp, job, newNode)) { - // _executorExitCode and _executorExitCause are set by the above function + if (ExitInfo exitInfo = propagateChangeToDbAndTree(syncOp, job, newNode); !exitInfo) { cancelAllOngoingJobs(); - return false; + return exitInfo; } SyncFileStatus status = SyncFileStatus::Success; @@ -1950,10 +1795,10 @@ bool ExecutorWorker::handleFinishedJob(std::shared_ptr job, SyncOpP syncOp->affectedNode()->hasChangeEvent(OperationType::Delete); } - return true; + return ExitCode::Ok; } -void ExecutorWorker::handleForbiddenAction(SyncOpPtr syncOp, const SyncPath &relativeLocalPath, bool &ignored) { +ExitInfo ExecutorWorker::handleForbiddenAction(SyncOpPtr syncOp, const SyncPath &relativeLocalPath, bool &ignored) { ignored = false; const SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalPath; @@ -2018,12 +1863,14 @@ void ExecutorWorker::handleForbiddenAction(SyncOpPtr syncOp, const SyncPath &rel if (removeFromDb) { // Remove the node from DB and tree so it will be re-created at its original location on next sync - if (!propagateDeleteToDbAndTree(syncOp)) { + _syncPal->setRestart(true); + if (ExitInfo exitInfo = propagateDeleteToDbAndTree(syncOp); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + return exitInfo; } - _syncPal->setRestart(true); } + return ExitCode::Ok; } void ExecutorWorker::sendProgress() { @@ -2039,8 +1886,7 @@ void ExecutorWorker::sendProgress() { } } -// !!! When returning true (hasError), _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &propagateChange) { +ExitInfo ExecutorWorker::propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &propagateChange) { propagateChange = true; switch (syncOp->conflict().type()) { @@ -2061,10 +1907,9 @@ bool ExecutorWorker::propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &propag _syncPal->_syncDb->dbId(ReplicaSide::Local, effectiveNodeId, dbId, localNodeFoundInDb); if (localNodeFoundInDb) { // Remove local node from DB - if (!deleteFromDb(syncOp->conflict().localNode())) { - // _executorExitCode and _executorExitCause are set by the above function + if (ExitInfo exitInfo = deleteFromDb(syncOp->conflict().localNode()); !exitInfo) { propagateChange = false; - return true; + return exitInfo; } } } @@ -2116,17 +1961,16 @@ bool ExecutorWorker::propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &propag // Just apply normal behavior break; } - return false; + return ExitCode::Ok; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::propagateChangeToDbAndTree(SyncOpPtr syncOp, std::shared_ptr job, std::shared_ptr &node) { +ExitInfo ExecutorWorker::propagateChangeToDbAndTree(SyncOpPtr syncOp, std::shared_ptr job, + std::shared_ptr &node) { if (syncOp->hasConflict()) { bool propagateChange = true; - bool hasError = propagateConflictToDbAndTree(syncOp, propagateChange); - // _executorExitCode and _executorExitCause are set by the above function - if (!propagateChange || hasError) { - return !hasError; + ExitInfo exitInfo = propagateConflictToDbAndTree(syncOp, propagateChange); + if (!propagateChange || !exitInfo) { + return exitInfo; } } @@ -2157,41 +2001,33 @@ bool ExecutorWorker::propagateChangeToDbAndTree(SyncOpPtr syncOp, std::shared_pt if (!jobOk) { LOGW_SYNCPAL_WARN(_logger, L"Failed to cast upload job " << job->jobId()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::SystemError; } } if (syncOp->type() == OperationType::Create) { return propagateCreateToDbAndTree(syncOp, nodeId, modtime, node); - // _executorExitCode and _executorExitCause are set by the above function } else { return propagateEditToDbAndTree(syncOp, nodeId, modtime, node); - // _executorExitCode and _executorExitCause are set by the above function } } case OperationType::Move: { return propagateMoveToDbAndTree(syncOp); - // _executorExitCode and _executorExitCause are set by the above function } case OperationType::Delete: { return propagateDeleteToDbAndTree(syncOp); - // _executorExitCode and _executorExitCause are set by the above function } default: { LOGW_SYNCPAL_WARN(_logger, L"Unknown operation type " << syncOp->type() << L" on file " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::SystemError; } } + return ExitCode::LogicError; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, std::optional newLastModTime, - std::shared_ptr &node) { +ExitInfo ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, + std::optional newLastModTime, std::shared_ptr &node) { std::shared_ptr newCorrespondingParentNode = nullptr; if (affectedUpdateTree(syncOp)->rootNode() == syncOp->affectedNode()->parentNode()) { newCorrespondingParentNode = targetUpdateTree(syncOp)->rootNode(); @@ -2201,9 +2037,7 @@ bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId & if (!newCorrespondingParentNode) { LOG_SYNCPAL_WARN(_logger, "Corresponding parent node not found"); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } // 2. Insert a new entry into the database, to avoid that the object is detected again by compute_ops() on the next sync @@ -2220,9 +2054,7 @@ bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId & if (localId.empty() || remoteId.empty()) { LOGW_SYNCPAL_WARN(_logger, L"Empty " << (localId.empty() ? L"local" : L"remote") << L" id for item " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } DbNode dbNode(0, newCorrespondingParentNode->idb(), localName, remoteName, localId, remoteId, @@ -2264,9 +2096,7 @@ bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId & << (newCorrespondingParentNode->idb().has_value() ? newCorrespondingParentNode->idb().value() : -1)); if (!constraintError) { - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } // Manage DELETE events not reported by the folder watcher @@ -2277,26 +2107,20 @@ bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId & bool found = false; if (!_syncPal->_syncDb->dbId(ReplicaSide::Remote, remoteId, dbNodeId, found)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::dbId"); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } if (found) { // Delete old node if (!_syncPal->_syncDb->deleteNode(dbNodeId, found)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::deleteNode"); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } } // Create new node if (!_syncPal->_syncDb->insertNode(dbNode, newDbNodeId, constraintError)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::insertNode"); - _executorExitCode = constraintError ? ExitCode::DataError : ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {constraintError ? ExitCode::DataError : ExitCode::DbError, ExitCause::DbAccessError}; } // The snapshot must be invalidated before the next sync @@ -2317,11 +2141,9 @@ bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId & remoteName, syncOp->affectedNode()->type(), OperationType::None, newNodeId, newLastModTime, newLastModTime, syncOp->affectedNode()->size(), newCorrespondingParentNode)); if (node == nullptr) { - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::NotEnoughtMemory; std::cout << "Failed to allocate memory" << std::endl; LOG_SYNCPAL_ERROR(_logger, "Failed to allocate memory"); - return false; + return {ExitCode::SystemError, ExitCause::NotEnoughtMemory}; } std::shared_ptr updateTree = targetUpdateTree(syncOp); @@ -2331,34 +2153,27 @@ bool ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId & LOGW_SYNCPAL_WARN(_logger, L"Error in Node::insertChildren: node name=" << SyncName2WStr(node->name()).c_str() << L" parent node name=" << SyncName2WStr(newCorrespondingParentNode->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } // Affected node does not have a valid DB ID yet, update it syncOp->affectedNode()->setIdb(newDbNodeId); } - return true; + return ExitCode::Ok; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, std::optional newLastModTime, - std::shared_ptr &node) { +ExitInfo ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, + std::optional newLastModTime, std::shared_ptr &node) { DbNode dbNode; bool found = false; if (!_syncPal->_syncDb->node(*syncOp->correspondingNode()->idb(), dbNode, found)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::node"); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; - } - if (!found) { - LOG_SYNCPAL_DEBUG(_logger, "Failed to retrieve node for dbId=" << *syncOp->correspondingNode()->idb()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; + if (!found) { + LOG_SYNCPAL_DEBUG(_logger, "Failed to retrieve node for dbId=" << *syncOp->correspondingNode()->idb()); + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; + } } // 2. Update the database entry, to avoid detecting the edit operation again. @@ -2374,9 +2189,7 @@ bool ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &ne if (localId.empty() || remoteId.empty()) { LOGW_SYNCPAL_WARN(_logger, L"Empty " << (localId.empty() ? L"local" : L"remote") << L" id for item " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } dbNode.setNodeIdLocal(localId); @@ -2408,14 +2221,10 @@ bool ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &ne << SyncName2WStr(localName).c_str() << L", remote name: " << SyncName2WStr(remoteName).c_str() << L", parent DB ID: " << (dbNode.parentNodeId().has_value() ? dbNode.parentNodeId().value() : -1)); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } if (!found) { - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } // 3. If the omit flag is False, update the updatetreeY structure to ensure that follow-up operations can execute @@ -2428,34 +2237,28 @@ bool ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &ne } node = syncOp->correspondingNode(); - return true; + return ExitCode::Ok; } // !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { +ExitInfo ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { std::shared_ptr correspondingNode = syncOp->correspondingNode() ? syncOp->correspondingNode() : syncOp->affectedNode(); // No corresponding node => rename if (!correspondingNode || !correspondingNode->idb().has_value()) { LOG_SYNCPAL_WARN(_logger, "Invalid corresponding node"); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } DbNode dbNode; bool found = false; if (!_syncPal->_syncDb->node(*correspondingNode->idb(), dbNode, found)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::node"); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } if (!found) { LOG_SYNCPAL_DEBUG(_logger, "Failed to retrieve node for dbId=" << *correspondingNode->idb()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } // 2. Update the database entry, to avoid detecting the move operation again. @@ -2467,9 +2270,7 @@ bool ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { if (!parentNode) { LOGW_SYNCPAL_DEBUG(_logger, L"Failed to get corresponding parent node: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } std::string localId = syncOp->targetSide() == ReplicaSide::Local @@ -2486,9 +2287,7 @@ bool ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { if (localId.empty() || remoteId.empty()) { LOGW_SYNCPAL_WARN(_logger, L"Empty " << (localId.empty() ? L"local" : L"remote") << L" id for item " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } dbNode.setParentNodeId(parentNode->idb()); @@ -2519,14 +2318,10 @@ bool ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { << SyncName2WStr(localName).c_str() << L", remote name: " << SyncName2WStr(remoteName).c_str() << L", parent DB ID: " << (dbNode.parentNodeId().has_value() ? dbNode.parentNodeId().value() : -1)); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } if (!found) { - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } // 3. If the omit flag is False, update the updatetreeY structure to ensure that follow-up operations can execute @@ -2541,86 +2336,68 @@ bool ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { LOGW_SYNCPAL_WARN(_logger, L"Error in Node::setParentNode: node name=" << SyncName2WStr(parentNode->name()).c_str() << L" parent node name=" << SyncName2WStr(correspondingNode->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } if (!correspondingNode->parentNode()->insertChildren(correspondingNode)) { LOGW_SYNCPAL_WARN(_logger, L"Error in Node::insertChildren: node name=" << SyncName2WStr(correspondingNode->name()).c_str() << L" parent node name=" << SyncName2WStr(correspondingNode->parentNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } } - return true; + return ExitCode::Ok; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::propagateDeleteToDbAndTree(SyncOpPtr syncOp) { +ExitInfo ExecutorWorker::propagateDeleteToDbAndTree(SyncOpPtr syncOp) { // 2. Remove the entry from the database. If nX is a directory node, also remove all entries for each node n ∈ S. This // avoids that the object(s) are detected again by compute_ops() on the next sync iteration - if (!deleteFromDb(syncOp->affectedNode())) { - // _executorExitCode and _executorExitCause are set by the above function - return false; + if (ExitInfo exitInfo = deleteFromDb(syncOp->affectedNode()); !exitInfo) { + return exitInfo; } // 3. Remove nX and nY from the update tree structures. if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { LOGW_SYNCPAL_WARN( _logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } - return true; + return ExitCode::Ok; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! -bool ExecutorWorker::deleteFromDb(std::shared_ptr node) { +ExitInfo ExecutorWorker::deleteFromDb(std::shared_ptr node) { if (!node->idb().has_value()) { LOGW_SYNCPAL_WARN(_logger, L"Node " << SyncName2WStr(node->name()).c_str() << L" does not have a DB ID"); - - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } // Remove item (and children by cascade) from DB bool found = false; if (!_syncPal->_syncDb->deleteNode(*node->idb(), found)) { LOG_SYNCPAL_WARN(_logger, "Failed to remove node " << *node->idb() << " from DB"); - _executorExitCode = ExitCode::DbError; - _executorExitCause = ExitCause::DbAccessError; - return false; + return {ExitCode::DbError, ExitCause::DbAccessError}; } if (!found) { LOG_SYNCPAL_WARN(_logger, "Node DB ID " << *node->idb() << " not found"); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::DbEntryNotFound; - return false; + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } if (ParametersCache::isExtendedLogEnabled()) { LOGW_SYNCPAL_DEBUG(_logger, L"Item \"" << SyncName2WStr(node->name()).c_str() << L"\" removed from DB"); } - return true; + return ExitCode::Ok; } -bool ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptr job) { +ExitInfo ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptr job) { job->runSynchronously(); std::string errorCode; @@ -2631,7 +2408,6 @@ bool ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptrblacklistTemporarily( syncOp->affectedNode()->id().has_value() ? syncOp->affectedNode()->id().value() : std::string(), syncOp->affectedNode()->getPath(), ReplicaSide::Local); @@ -2644,25 +2420,24 @@ bool ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptrdeleteNode(syncOp->affectedNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - return false; + return ExitCode::DataError; } if (syncOp->correspondingNode()) { if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); - return false; + return ExitCode::DataError; } } - return true; + return ExitCode::Ok; } } if (job->exitCode() == ExitCode::NeedRestart) { // Special case: not an error but sync needs to be restarted - _executorExitCode = ExitCode::Ok; _syncPal->setRestart(true); - return false; + return ExitCode::NeedRestart; } else if ((syncOp->targetSide() == ReplicaSide::Local && job->exitCode() == ExitCode::DataError && job->exitCause() == ExitCause::FileAlreadyExist) || (syncOp->targetSide() == ReplicaSide::Remote && job->exitCode() == ExitCode::BackError && @@ -2674,12 +2449,10 @@ bool ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptrlocalPath() / localCreateDirJob->destFilePath(), PlatformInconsistencyCheckerUtility::SuffixTypeBlacklisted); } - return false; + return ExitCode::NeedRestart; } else if (job->exitCode() != ExitCode::Ok) { LOGW_SYNCPAL_WARN(_logger, L"Failed to create directory: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - _executorExitCode = job->exitCode(); - _executorExitCause = job->exitCause(); - return false; + return {job->exitCode(), job->exitCause()}; } NodeId newNodeId; @@ -2698,19 +2471,17 @@ bool ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->name()).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::ApiErr; - return false; + return {ExitCode::DataError, ExitCause::ApiErr}; } std::shared_ptr newNode = nullptr; - if (!propagateCreateToDbAndTree(syncOp, newNodeId, newModTime, newNode)) { + if (ExitInfo exitInfo = propagateCreateToDbAndTree(syncOp, newNodeId, newModTime, newNode); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - return false; + return exitInfo; } - return true; + return ExitCode::Ok; } void ExecutorWorker::cancelAllOngoingJobs(bool reschedule /*= false*/) { @@ -2781,38 +2552,30 @@ void ExecutorWorker::increaseErrorCount(SyncOpPtr syncOp) { } } -bool ExecutorWorker::getFileSize(const SyncPath &path, uint64_t &size) { +ExitInfo ExecutorWorker::getFileSize(const SyncPath &path, uint64_t &size) { IoError ioError = IoError::Unknown; if (!IoHelper::getFileSize(path, size, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::getFileSize for " << Utility::formatIoError(path, ioError).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::SystemError; } if (ioError == IoError::NoSuchFileOrDirectory) { // The synchronization will be re-started. LOGW_WARN(_logger, L"File doesn't exist: " << Utility::formatSyncPath(path).c_str()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::DataError; } if (ioError == IoError::AccessDenied) { // An action from the user is requested. LOGW_WARN(_logger, L"File search permission missing: " << Utility::formatSyncPath(path).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::NoSearchPermission; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } assert(ioError == IoError::Success); if (ioError != IoError::Success) { LOGW_WARN(_logger, L"Unable to read file size for " << Utility::formatSyncPath(path).c_str()); - _executorExitCode = ExitCode::SystemError; - _executorExitCause = ExitCause::Unknown; - return false; + return ExitCode::SystemError; } - return true; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/propagation/executor/executorworker.h b/src/libsyncengine/propagation/executor/executorworker.h index e47625b0b..1bfe85a17 100644 --- a/src/libsyncengine/propagation/executor/executorworker.h +++ b/src/libsyncengine/propagation/executor/executorworker.h @@ -69,15 +69,15 @@ class ExecutorWorker : public OperationProcessor { void initProgressManager(); void initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem); - void handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &hasError, bool &ignored); + ExitInfo handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &ignored); void checkAlreadyExcluded(const SyncPath &absolutePath, const NodeId &parentId); - bool generateCreateJob(SyncOpPtr syncOp, std::shared_ptr &job) noexcept; - bool checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const SyncPath &path, bool &isDehydratedPlaceholder); - ExitCode createPlaceholder(const SyncPath &relativeLocalPath, ExitCause &exitCause); - ExitCode convertToPlaceholder(const SyncPath &relativeLocalPath, bool hydrated, ExitCause &exitCause); + ExitInfo generateCreateJob(SyncOpPtr syncOp, std::shared_ptr &job) noexcept; + ExitInfo checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const SyncPath &path, bool &isDehydratedPlaceholder); + ExitInfo createPlaceholder(const SyncPath &relativeLocalPath); + ExitInfo convertToPlaceholder(const SyncPath &relativeLocalPath, bool hydrated); - void handleEditOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &hasError, bool &ignored); - bool generateEditJob(SyncOpPtr syncOp, std::shared_ptr &job); + ExitInfo handleEditOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &ignored); + ExitInfo generateEditJob(SyncOpPtr syncOp, std::shared_ptr &job); /** * This method aims to fix the last modification date of a local file using the date stored in DB. This allows us to fix @@ -86,39 +86,40 @@ class ExecutorWorker : public OperationProcessor { * @param absolutePath : absolute local path of the affected file. * @return `true` if the date is modified successfully. */ - bool fixModificationDate(SyncOpPtr syncOp, const SyncPath &absolutePath); - bool checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPath &absolutePath, bool &ignoreItem, + ExitInfo fixModificationDate(SyncOpPtr syncOp, const SyncPath &absolutePath); + ExitInfo checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPath &absolutePath, bool &ignoreItem, bool &isSyncing); // TODO : is called "check..." but perform some actions. Wording not // good, function probably does too much - void handleMoveOp(SyncOpPtr syncOp, bool &hasError, bool &ignored, bool &bypassProgressComplete); - bool generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete); + ExitInfo handleMoveOp(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete); + ExitInfo generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete); - void handleDeleteOp(SyncOpPtr syncOp, bool &hasError, bool &ignored, bool &bypassProgressComplete); - bool generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete); + ExitInfo handleDeleteOp(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete); + ExitInfo generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete); bool hasRight(SyncOpPtr syncOp, bool &exists); bool enoughLocalSpace(SyncOpPtr syncOp); - void waitForAllJobsToFinish(bool &hasError); - bool deleteFinishedAsyncJobs(); - bool handleManagedBackError(ExitCause jobExitCause, SyncOpPtr syncOp, bool isInconsistencyIssue, bool downloadImpossible); - bool handleFinishedJob(std::shared_ptr job, SyncOpPtr syncOp, const SyncPath &relativeLocalPath, + ExitInfo waitForAllJobsToFinish(); + ExitInfo deleteFinishedAsyncJobs(); + ExitInfo handleManagedBackError(ExitCause jobExitCause, SyncOpPtr syncOp, bool isInconsistencyIssue, + bool downloadImpossible); + ExitInfo handleFinishedJob(std::shared_ptr job, SyncOpPtr syncOp, const SyncPath &relativeLocalPath, bool &ignored, bool &bypassProgressComplete); - void handleForbiddenAction(SyncOpPtr syncOp, const SyncPath &relativeLocalPath, bool &ignored); + ExitInfo handleForbiddenAction(SyncOpPtr syncOp, const SyncPath &relativeLocalPath, bool &ignored); void sendProgress(); - bool propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &propagateChange); - bool propagateChangeToDbAndTree(SyncOpPtr syncOp, std::shared_ptr job, std::shared_ptr &node); - bool propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, std::optional newLastModTime, + ExitInfo propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &propagateChange); + ExitInfo propagateChangeToDbAndTree(SyncOpPtr syncOp, std::shared_ptr job, std::shared_ptr &node); + ExitInfo propagateCreateToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, std::optional newLastModTime, std::shared_ptr &node); - bool propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, std::optional newLastModTime, + ExitInfo propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId &newNodeId, std::optional newLastModTime, std::shared_ptr &node); - bool propagateMoveToDbAndTree(SyncOpPtr syncOp); - bool propagateDeleteToDbAndTree(SyncOpPtr syncOp); - bool deleteFromDb(std::shared_ptr node); + ExitInfo propagateMoveToDbAndTree(SyncOpPtr syncOp); + ExitInfo propagateDeleteToDbAndTree(SyncOpPtr syncOp); + ExitInfo deleteFromDb(std::shared_ptr node); - bool runCreateDirJob(SyncOpPtr syncOp, std::shared_ptr job); + ExitInfo runCreateDirJob(SyncOpPtr syncOp, std::shared_ptr job); void cancelAllOngoingJobs(bool reschedule = false); @@ -135,7 +136,7 @@ class ExecutorWorker : public OperationProcessor { void increaseErrorCount(SyncOpPtr syncOp); - bool getFileSize(const SyncPath &path, uint64_t &size); + ExitInfo getFileSize(const SyncPath &path, uint64_t &size); void logCorrespondingNodeErrorMsg(const SyncOpPtr syncOp); void setProgressComplete(const SyncOpPtr syncOp, SyncFileStatus status); @@ -145,10 +146,6 @@ class ExecutorWorker : public OperationProcessor { std::unordered_map _jobToSyncOpMap; std::unordered_map _syncOpToJobMap; std::list _opList; - - ExitCode _executorExitCode = ExitCode::Unknown; - ExitCause _executorExitCause = ExitCause::Unknown; - std::chrono::steady_clock::time_point _fileProgressTimer = std::chrono::steady_clock::now(); bool _snapshotToInvalidate = false; diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index e18ed7785..2345e3e2b 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -151,17 +151,13 @@ void TestWorkers::tearDown() { } void TestWorkers::testCreatePlaceholder() { - ExitCause exitCause{ExitCause::Unknown}; - ExitCode exitCode{ExitCode::Unknown}; - _syncPal->resetEstimateUpdates(); - + ExitInfo exitInfo; // Progress not intialized { - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(SyncPath("dummy"), exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::InvalidSnapshot); + exitInfo = _syncPal->_executorWorker->createPlaceholder(SyncPath("dummy")); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); } // Create folder operation @@ -174,17 +170,15 @@ void TestWorkers::testCreatePlaceholder() { _syncPal->initProgress(syncItem); // Folder doesn't exist (normal case) - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::Unknown); + exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); #if defined(__APPLE__) && defined(__WIN32) // Folder already exists - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::InvalidSnapshot); + exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); #endif } @@ -203,10 +197,9 @@ void TestWorkers::testCreatePlaceholder() { CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, false, false, false, ioError) && ioError == IoError::Success); - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(relativeFilePath, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::SystemError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::NoSearchPermission); + exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::SystemError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::NoSearchPermission); ioError = IoError::Unknown; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, true, true, true, ioError) && @@ -214,33 +207,27 @@ void TestWorkers::testCreatePlaceholder() { #endif // File doesn't exist (normal case) - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(relativeFilePath, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::Unknown); + exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); #if defined(__APPLE__) && defined(__WIN32) // File already exists - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(relativeFilePath, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::InvalidSnapshot); + exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); #endif } } void TestWorkers::testConvertToPlaceholder() { - ExitCause exitCause{ExitCause::Unknown}; - ExitCode exitCode{ExitCode::Unknown}; - _syncPal->resetEstimateUpdates(); - + ExitInfo exitInfo; // Progress not intialized { - ExitCause exitCause{ExitCause::Unknown}; - ExitCode exitCode = _syncPal->_executorWorker->convertToPlaceholder(SyncPath("dummy"), true, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::InvalidSnapshot); + exitInfo = _syncPal->_executorWorker->convertToPlaceholder(SyncPath("dummy"), true); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); } // Convert folder operation @@ -254,20 +241,18 @@ void TestWorkers::testConvertToPlaceholder() { #if defined(__APPLE__) && defined(__WIN32) // Folder doesn't exist - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::InvalidSnapshot); + exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); #endif // Folder already exists (normal case) std::error_code ec; CPPUNIT_ASSERT(std::filesystem::create_directory(_syncPal->localPath() / relativeFolderPath, ec) && ec.value() == 0); - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::Unknown); + exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); } // Convert file operation @@ -285,20 +270,18 @@ void TestWorkers::testConvertToPlaceholder() { CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, false, false, false, ioError) && ioError == IoError::Success); - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->createPlaceholder(relativeFilePath, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::SystemError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::NoSearchPermission); + exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::SystemError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::NoSearchPermission); ioError = IoError::Unknown; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, true, true, true, ioError) && ioError == IoError::Success); // File doesn't exist - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->convertToPlaceholder(relativeFilePath, true, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::InvalidSnapshot); + exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFilePath, true); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); #endif // File already exists (normal case) @@ -307,10 +290,9 @@ void TestWorkers::testConvertToPlaceholder() { ofs << "Some content.\n"; } - exitCause = ExitCause::Unknown; - exitCode = _syncPal->_executorWorker->convertToPlaceholder(relativeFilePath, true, exitCause); - CPPUNIT_ASSERT_EQUAL(exitCode, ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitCause, ExitCause::Unknown); + exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFilePath, true); + CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); } } From 9aa97f84d2cd8e33b37b844bc52a58a2a3d8e8b5 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 18 Oct 2024 15:57:56 +0200 Subject: [PATCH 05/56] Add int operator for ExitInfo. --- src/libcommon/utility/types.h | 44 ++++++++++++++++------------ test/libcommon/utility/testtypes.cpp | 5 ++++ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 3a3e14ab8..975e78475 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -111,6 +111,19 @@ static constexpr std::string_view clientSendEventFileName( "kdrive_client.send.event"); // Name of the debug file generated by Sentry before_send callback of the client } // namespace event_dump_files +// Concepts +template // Any enum class +concept EnumClass = std::is_enum_v; + +template // Any enum class that can be converted to (and from) int +concept IntableEnum = EnumClass && std::is_convertible_v, int>; + +template +inline constexpr int toInt(C e); + +template // Any enum class that can be printed (with enumClassToString) +concept PrintableEnum = EnumClass && requires(C e) { toString(e); }; + enum class AppType { None, Server, Client }; std::string toString(AppType e); @@ -221,13 +234,15 @@ std::string toString(ExitCause e); struct ExitInfo { ExitInfo() = default; - ExitInfo(const ExitCode &code, const ExitCause &cause) : _code(code), _cause(cause) {} + constexpr ExitInfo(const ExitCode &code, const ExitCause &cause) : _code(code), _cause(cause) {} ExitInfo(const ExitCode &code) : _code(code) {} const ExitCode &code() const { return _code; } const ExitCause &cause() const { return _cause; } operator ExitCode() const { return _code; } operator ExitCause() const { return _cause; } operator bool() const { return _code == ExitCode::Ok; } + constexpr explicit operator int() const { return toInt(_code) * 100 + toInt(_cause); } + constexpr bool operator==(const ExitInfo &other) const { return _code == other._code && _cause == other._cause; } private: ExitCode _code{ExitCode::Unknown}; @@ -407,8 +422,8 @@ struct ItemType { SyncPath targetPath; // The value of the data member `ioError` is `IoError::NoSuchFileOrDirectory` if // - the file or directory indicated by `path` doesn't exist - // - the file or directory indicated by `path` is a symlink or an alias (in which case `linkType` is different from - // `LinkType::Unknown`) and its target doesn't exist. + // - the file or directory indicated by `path` is a symlink or an alias (in which case `linkType` is different + // from `LinkType::Unknown`) and its target doesn't exist. IoError ioError{IoError::Success}; }; @@ -447,7 +462,8 @@ struct VersionInfo { std::string tag; // Version number. Example: 3.6.4 std::string changeLog; // List of changes in this version std::uint64_t buildVersion = 0; // Example: 20240816 - std::string buildMinOsVersion; // Optionnal. Minimum supported version of the OS. Examples: 10.15, 11, server 2005, ... + std::string buildMinOsVersion; // Optionnal. Minimum supported version of the OS. Examples: 10.15, 11, server + // 2005, ... std::string downloadUrl; // URL to download the version [[nodiscard]] bool isValid() const { @@ -470,7 +486,8 @@ enum class SentryConfidentialityLevel { }; std::string toString(SentryConfidentialityLevel e); -// Adding a new types here requires to add it in stringToAppStateValue and appStateValueToString in libcommon/utility/utility.cpp +// Adding a new types here requires to add it in stringToAppStateValue and appStateValueToString in +// libcommon/utility/utility.cpp using AppStateValue = std::variant; /* @@ -478,26 +495,17 @@ using AppStateValue = std::variant; */ // Concepts -template // Any enum class -concept EnumClass = std::is_enum_v; - -template // Any enum class that can be converted to (and from) int -concept IntableEnum = EnumClass && std::is_convertible_v, int>; - template // Any enum class we want to allow bitwise operations (OperationType & InconsistencyType) concept AllowBitWiseOpEnum = IntableEnum && (std::is_same_v || std::is_same_v); -template // Any enum class that can be printed (with enumClassToString) -concept PrintableEnum = EnumClass && requires(C e) { toString(e); }; - // Converters template -inline int toInt(C e) { +inline constexpr int toInt(C e) { return static_cast(e); } template -inline C fromInt(int e) { +inline constexpr C fromInt(int e) { return static_cast(e); } @@ -538,8 +546,8 @@ inline bool bitWiseEnumToBool(const C a) { } namespace typesUtility { -std::wstring stringToWideString(const std::string &str); // Convert string to wstring (We can't use the s2ws of Utility because - // it's in libCommonServer and it includes types.h) +std::wstring stringToWideString(const std::string &str); // Convert string to wstring (We can't use the s2ws of Utility + // because it's in libCommonServer and it includes types.h) } // namespace typesUtility // Stream Operator (toString) diff --git a/test/libcommon/utility/testtypes.cpp b/test/libcommon/utility/testtypes.cpp index a12dec925..7a678b2de 100644 --- a/test/libcommon/utility/testtypes.cpp +++ b/test/libcommon/utility/testtypes.cpp @@ -78,6 +78,7 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCode::Unknown, ec); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); + CPPUNIT_ASSERT_EQUAL(0, static_cast(ei)); ei = {ExitCode::Ok}; ec = ei; @@ -86,6 +87,7 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, ec); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); + CPPUNIT_ASSERT_EQUAL(100, static_cast(ei)); ei = {ExitCode::NetworkError, ExitCause::DbAccessError}; ec = ei; @@ -94,10 +96,13 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCode::NetworkError, ec); CPPUNIT_ASSERT_EQUAL(ExitCause::DbAccessError, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::DbAccessError, eca); + CPPUNIT_ASSERT_EQUAL(302, static_cast(ei)); + ec = ExitCode::BackError; ei = ec; CPPUNIT_ASSERT_EQUAL(ExitCode::BackError, ei.code()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); + CPPUNIT_ASSERT_EQUAL(700, static_cast(ei)); } } // namespace KDC From a58e28902894fee86903472399c5b481b8bd8c60 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 08:14:25 +0200 Subject: [PATCH 06/56] Remove duplicate exitCode/Cause for acess denied and use it exclusively for acess denied errors. --- src/gui/parametersdialog.cpp | 10 --------- src/libcommon/utility/types.cpp | 4 ---- src/libcommon/utility/types.h | 2 -- src/libcommonserver/io/iohelper_win.cpp | 4 ++-- src/libsyncengine/jobs/local/localcopyjob.cpp | 16 ++++++++++++-- .../jobs/local/localcreatedirjob.cpp | 16 +++++++++----- .../jobs/local/localdeletejob.cpp | 10 +++++++-- src/libsyncengine/jobs/local/localmovejob.cpp | 8 ++++++- .../jobs/network/API_v2/deletejob.cpp | 9 ++++++-- .../jobs/network/API_v2/downloadjob.cpp | 20 ++++++++++++----- .../jobs/network/API_v2/movejob.cpp | 6 +++++ .../upload_session/abstractuploadsession.cpp | 6 +++++ .../jobs/network/API_v2/uploadjob.cpp | 22 ++++++++++++------- .../localfilesystemobserverworker.cpp | 10 ++++----- src/server/logarchiver.cpp | 2 +- 15 files changed, 96 insertions(+), 49 deletions(-) diff --git a/src/gui/parametersdialog.cpp b/src/gui/parametersdialog.cpp index 2d213fe8f..66d7aa085 100644 --- a/src/gui/parametersdialog.cpp +++ b/src/gui/parametersdialog.cpp @@ -403,7 +403,6 @@ QString ParametersDialog::getAppErrorText(QString fctCode, ExitCode exitCode, Ex case ExitCode::Ok: case ExitCode::NeedRestart: case ExitCode::LogicError: - case ExitCode::NoWritePermission: case ExitCode::TokenRefreshed: case ExitCode::RateLimited: case ExitCode::InvalidSync: @@ -420,10 +419,6 @@ QString ParametersDialog::getAppErrorText(QString fctCode, ExitCode exitCode, Ex QString ParametersDialog::getSyncPalSystemErrorText(const QString &err, ExitCause exitCause) const { switch (exitCause) { - case ExitCause::NoSearchPermission: - return tr("The item misses search permission (error %1).
" - "Please check that you have search/exec access to the parent folder.") - .arg(err); case ExitCause::SyncDirDoesntExist: return tr("The synchronization folder is no longer accessible (error %1).
" "Synchronization will resume as soon as the folder is accessible.") @@ -577,11 +572,6 @@ QString ParametersDialog::getSyncPalErrorText(QString fctCode, ExitCode exitCode return tr("Nested synchronizations are prohibited (error %1).
" "You should only keep synchronizations whose folders are not nested.") .arg(err); - case ExitCode::NoWritePermission: - return tr( - "The app does not have write rights to the synchronization folder.
" - "The synchronization has been stopped."); - break; case ExitCode::LogicError: if (exitCause == ExitCause::FullListParsingError) { return tr("File name parsing error (error %1).
" diff --git a/src/libcommon/utility/types.cpp b/src/libcommon/utility/types.cpp index 061291918..a3232f713 100644 --- a/src/libcommon/utility/types.cpp +++ b/src/libcommon/utility/types.cpp @@ -101,8 +101,6 @@ std::string toString(const ExitCode e) { return "LogicError"; case ExitCode::TokenRefreshed: return "TokenRefreshed"; - case ExitCode::NoWritePermission: - return "NoWritePermission"; case ExitCode::RateLimited: return "RateLimited"; case ExitCode::InvalidSync: @@ -190,8 +188,6 @@ std::string toString(const ExitCause e) { return "NetworkTimeout"; case ExitCause::SocketsDefuncted: return "SocketsDefuncted"; - case ExitCause::NoSearchPermission: - return "NoSearchPermission"; case ExitCause::NotFound: return "NotFound"; case ExitCause::QuotaExceeded: diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 975e78475..654026723 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -178,7 +178,6 @@ enum class ExitCode { LogicError, // Consequence of faulty logic within the program such as violating logical preconditions or class // invariants and may be preventable TokenRefreshed, - NoWritePermission, RateLimited, InvalidSync, // The sync configuration is not valid InvalidOperation, @@ -224,7 +223,6 @@ enum class ExitCause { LiteSyncNotAllowed, NetworkTimeout, SocketsDefuncted, // macOS: sockets defuncted by kernel - NoSearchPermission, NotFound, QuotaExceeded, FullListParsingError, diff --git a/src/libcommonserver/io/iohelper_win.cpp b/src/libcommonserver/io/iohelper_win.cpp index 8c4a85951..5dda7f832 100644 --- a/src/libcommonserver/io/iohelper_win.cpp +++ b/src/libcommonserver/io/iohelper_win.cpp @@ -202,7 +202,8 @@ bool IoHelper::getFileStat(const SyncPath &path, FileStat *filestat, IoError &io ioError = IoError::NoSuchFileOrDirectory; return true; } - if (counter) { + ioError = dWordError2ioError(dwError, logger()); + if (counter && ioError != IoError::AccessDenied) { retry = true; Utility::msleep(10); LOGW_DEBUG(logger(), L"Retrying to get handle: " << Utility::formatSyncPath(path.parent_path()).c_str()); @@ -210,7 +211,6 @@ bool IoHelper::getFileStat(const SyncPath &path, FileStat *filestat, IoError &io continue; } - ioError = dWordError2ioError(dwError, logger()); LOGW_WARN(logger(), L"Error in CreateFileW: " << Utility::formatIoError(path.parent_path(), ioError).c_str()); return isExpectedError(ioError); diff --git a/src/libsyncengine/jobs/local/localcopyjob.cpp b/src/libsyncengine/jobs/local/localcopyjob.cpp index 39b3aef81..d1ae8e8af 100644 --- a/src/libsyncengine/jobs/local/localcopyjob.cpp +++ b/src/libsyncengine/jobs/local/localcopyjob.cpp @@ -37,6 +37,12 @@ bool LocalCopyJob::canRun() { if (!IoHelper::checkIfPathExists(_dest, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_dest, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_dest).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } @@ -53,6 +59,12 @@ bool LocalCopyJob::canRun() { if (!IoHelper::checkIfPathExists(_source, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_source, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_source).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } @@ -82,12 +94,12 @@ void LocalCopyJob::runJob() { << Utility::s2ws(fsError.what()).c_str() << L" (" << fsError.code().value() << L")"); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; } catch (...) { LOGW_WARN(_logger, L"Failed to copy item " << Path2WStr(_source).c_str() << L" to " << Path2WStr(_dest).c_str() << L": Unknown error"); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; } } diff --git a/src/libsyncengine/jobs/local/localcreatedirjob.cpp b/src/libsyncengine/jobs/local/localcreatedirjob.cpp index dd51c7a73..605c00b06 100644 --- a/src/libsyncengine/jobs/local/localcreatedirjob.cpp +++ b/src/libsyncengine/jobs/local/localcreatedirjob.cpp @@ -38,6 +38,12 @@ bool LocalCreateDirJob::canRun() { if (!IoHelper::checkIfPathExists(_destFilePath, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_destFilePath, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_destFilePath).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } @@ -58,7 +64,7 @@ void LocalCreateDirJob::runJob() { } IoError ioError = IoError::Success; - if (IoHelper::createDirectory(_destFilePath, ioError)) { + if (IoHelper::createDirectory(_destFilePath, ioError) && ioError == IoError::Success) { if (isExtendedLog()) { LOGW_DEBUG(_logger, L"Directory: " << Utility::formatSyncPath(_destFilePath).c_str() << L" created"); } @@ -68,14 +74,14 @@ void LocalCreateDirJob::runJob() { if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Search permission missing: =" << Utility::formatSyncPath(_destFilePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NoSearchPermission; + _exitCause = ExitCause::FileAccessError; return; } if (ioError != IoError::Success) { // Unexpected error LOGW_WARN(_logger, L"Failed to create directory: " << Utility::formatIoError(_destFilePath, ioError).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; return; } @@ -84,7 +90,7 @@ void LocalCreateDirJob::runJob() { if (!IoHelper::getFileStat(_destFilePath, &filestat, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::getFileStat: " << Utility::formatIoError(_destFilePath, ioError).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; return; } @@ -96,7 +102,7 @@ void LocalCreateDirJob::runJob() { } else if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(_destFilePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NoSearchPermission; + _exitCause = ExitCause::FileAccessError; return; } diff --git a/src/libsyncengine/jobs/local/localdeletejob.cpp b/src/libsyncengine/jobs/local/localdeletejob.cpp index 3b3403ab1..3e78f7431 100644 --- a/src/libsyncengine/jobs/local/localdeletejob.cpp +++ b/src/libsyncengine/jobs/local/localdeletejob.cpp @@ -102,6 +102,12 @@ bool LocalDeleteJob::canRun() { if (!IoHelper::checkIfPathExists(_absolutePath, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_absolutePath, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_absolutePath).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } @@ -176,10 +182,10 @@ void LocalDeleteJob::runJob() { LOGW_DEBUG(_logger, L"Delete item with " << Utility::formatSyncPath(_absolutePath).c_str()); std::error_code ec; std::filesystem::remove_all(_absolutePath, ec); - if (ec) { + if (ec) { // We consider this as a permission denied error LOGW_WARN(_logger, L"Failed to delete item with path " << Utility::formatStdError(_absolutePath, ec).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::FileAccessError; return; } diff --git a/src/libsyncengine/jobs/local/localmovejob.cpp b/src/libsyncengine/jobs/local/localmovejob.cpp index 74bb48d3f..bc0439669 100644 --- a/src/libsyncengine/jobs/local/localmovejob.cpp +++ b/src/libsyncengine/jobs/local/localmovejob.cpp @@ -39,6 +39,12 @@ bool LocalMoveJob::canRun() { if (!IoHelper::checkIfPathExists(_dest, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_dest, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_dest).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } @@ -79,7 +85,7 @@ void LocalMoveJob::runJob() { std::error_code ec; std::filesystem::rename(_source, _dest, ec); - if (ec.value() != 0) { + if (ec.value() != 0) { // We consider this as a permission denied error LOGW_WARN(_logger, L"Failed to rename " << Path2WStr(_source).c_str() << L" to " << Path2WStr(_dest).c_str() << L": " << Utility::s2ws(ec.message()).c_str() << L" (" << ec.value() << L")"); _exitCode = ExitCode::SystemError; diff --git a/src/libsyncengine/jobs/network/API_v2/deletejob.cpp b/src/libsyncengine/jobs/network/API_v2/deletejob.cpp index 7401e706f..9892fea39 100644 --- a/src/libsyncengine/jobs/network/API_v2/deletejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/deletejob.cpp @@ -56,6 +56,11 @@ bool DeleteJob::canRun() { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_absoluteLocalFilepath, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } @@ -67,7 +72,7 @@ bool DeleteJob::canRun() { LOGW_WARN(_logger, L"Error in IoHelper::getFileStat: " << Utility::formatIoError(_absoluteLocalFilepath, ioError).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; return false; } @@ -79,7 +84,7 @@ bool DeleteJob::canRun() { } else if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(_absoluteLocalFilepath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NoSearchPermission; + _exitCause = ExitCause::FileAccessError; return false; } diff --git a/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp b/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp index a7c9933f1..d5d43d39e 100644 --- a/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp @@ -194,7 +194,7 @@ bool DownloadJob::handleResponse(std::istream &is) { if (isLink) { // Create link LOG_DEBUG(_logger, "Create link: mimeType=" << mimeType.c_str()); - if (!createLink(mimeType, linkData)) { + if (!createLink(mimeType, linkData)) { // We consider this as a permission denied error _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; @@ -212,7 +212,7 @@ bool DownloadJob::handleResponse(std::istream &is) { if (!IoHelper::tempDirectoryPath(tmpPath, ioError)) { LOGW_WARN(_logger, L"Failed to get temporary directory path: " << Utility::formatIoError(tmpPath, ioError).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; return false; } @@ -400,7 +400,7 @@ bool DownloadJob::handleResponse(std::istream &is) { if (!IoHelper::getFileStat(_localpath, &filestat, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::getFileStat: " << Utility::formatIoError(_localpath, ioError).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::Unknown; return false; } @@ -412,7 +412,7 @@ bool DownloadJob::handleResponse(std::istream &is) { } else if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(_localpath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NoSearchPermission; + _exitCause = ExitCause::FileAccessError; return false; } @@ -550,13 +550,23 @@ bool DownloadJob::moveTmpFile(const SyncPath &path, bool &restartSync) { } } #endif - else if (ec) { + else if (ec.value() == ERROR_ACCESS_DENIED) { + _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::FileAccessError; + return false; + } else if (ec) { bool exists = false; IoError ioError = IoError::Success; if (!IoHelper::checkIfPathExists(_localpath.parent_path(), exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_localpath.parent_path(), ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_localpath.parent_path()).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } diff --git a/src/libsyncengine/jobs/network/API_v2/movejob.cpp b/src/libsyncengine/jobs/network/API_v2/movejob.cpp index 83f76b6cc..1ae64d9bf 100644 --- a/src/libsyncengine/jobs/network/API_v2/movejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/movejob.cpp @@ -58,6 +58,12 @@ bool MoveJob::canRun() { if (!IoHelper::checkIfPathExists(_destFilepath, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_destFilepath, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_destFilepath).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp b/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp index a511abf0b..f6f3c6e9c 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp @@ -168,6 +168,12 @@ bool AbstractUploadSession::canRun() { if (!IoHelper::checkIfPathExists(_filePath, exists, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_filePath, ioError).c_str()); _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_filePath).c_str()); + _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; } diff --git a/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp b/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp index 2f4714bef..8274b84b5 100644 --- a/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp @@ -66,7 +66,13 @@ bool UploadJob::canRun() { bool exists; IoError ioError = IoError::Success; if (!IoHelper::checkIfPathExists(_filePath, exists, ioError)) { - LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_filePath, ioError).c_str()); + LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(_filePath, ioError)); + _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + return false; + } + if (ioError == IoError::AccessDenied) { + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_filePath)); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; @@ -74,7 +80,7 @@ bool UploadJob::canRun() { if (!exists) { LOGW_DEBUG(_logger, - L"Item does not exist anymore. Aborting current sync and restart - path=" << Path2WStr(_filePath).c_str()); + L"Item does not exist anymore. Aborting current sync and restart - path=" << Path2WStr(_filePath)); _exitCode = ExitCode::NeedRestart; _exitCause = ExitCause::UnexpectedFileSystemEvent; return false; @@ -184,7 +190,7 @@ std::string UploadJob::getContentType(bool &canceled) { bool UploadJob::readFile() { std::ifstream file(_filePath, std::ios_base::in | std::ios_base::binary); if (!file.is_open()) { - LOGW_WARN(_logger, L"Failed to open file - path=" << Path2WStr(_filePath).c_str()); + LOGW_WARN(_logger, L"Failed to open file - path=" << Path2WStr(_filePath)); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; @@ -244,7 +250,7 @@ bool UploadJob::readLink() { if (!exists) { LOGW_DEBUG(_logger, L"File doesn't exist - path=" << Path2WStr(_filePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::NotFound; return false; } @@ -278,14 +284,14 @@ bool UploadJob::readLink() { if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_DEBUG(_logger, L"File doesn't exist - " << Utility::formatSyncPath(_filePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::NotFound; return false; } if (ioError == IoError::AccessDenied) { LOGW_DEBUG(_logger, L"File misses search permissions - " << Utility::formatSyncPath(_filePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NoSearchPermission; + _exitCause = ExitCause::FileAccessError; return false; } #endif @@ -303,7 +309,7 @@ bool UploadJob::readLink() { if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_DEBUG(_logger, L"File doesn't exist - path=" << Path2WStr(_filePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + _exitCause = ExitCause::NotFound; return false; } @@ -311,7 +317,7 @@ bool UploadJob::readLink() { if (ioError == IoError::AccessDenied) { LOGW_DEBUG(_logger, L"File with insufficient access rights - path=" << Path2WStr(_filePath).c_str()); _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NoSearchPermission; + _exitCause = ExitCause::FileAccessError; return false; } diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index f37394e97..5add3a8cf 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -641,22 +641,22 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (!IoHelper::getItemType(absoluteParentDirPath, itemType)) { LOGW_WARN(_logger, L"Error in IoHelper::getItemType: " << Utility::formatIoError(absoluteParentDirPath, itemType.ioError).c_str()); - setExitCause(ExitCause::FileAccessError); - return ExitCode::SystemError; + setExitCause(ExitCause::Unknown); + return {ExitCode::SystemError, ExitCause::Unknown}; } if (itemType.ioError == IoError::NoSuchFileOrDirectory) { LOGW_SYNCPAL_WARN(_logger, L"Sync localpath: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() << L" doesn't exist"); setExitCause(ExitCause::SyncDirDoesntExist); - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::SyncDirDoesntExist}; } if (itemType.ioError == IoError::AccessDenied) { LOGW_SYNCPAL_WARN(_logger, L"Sync localpath: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() << L" misses read permission"); setExitCause(ExitCause::SyncDirReadError); - return ExitCode::SystemError; + return {ExitCode::SystemError, ExitCause::SyncDirReadError}; } if (itemType.linkType != LinkType::None) { @@ -867,7 +867,7 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen } catch (std::filesystem::filesystem_error &e) { LOG_SYNCPAL_WARN(Log::instance()->getLogger(), "Error caught in LocalFileSystemObserverWorker::exploreDir: " << e.code() << " - " << e.what()); - setExitCause(ExitCause::FileAccessError); + setExitCause(ExitCause::Unknown); return ExitCode::SystemError; } catch (...) { LOG_SYNCPAL_WARN(Log::instance()->getLogger(), "Error caught in LocalFileSystemObserverWorker::exploreDir"); diff --git a/src/server/logarchiver.cpp b/src/server/logarchiver.cpp index 6656ad8aa..436d763bb 100644 --- a/src/server/logarchiver.cpp +++ b/src/server/logarchiver.cpp @@ -299,7 +299,7 @@ ExitCode LogArchiver::copyParmsDbTo(const SyncPath &outputPath, ExitCause &exitC LOGW_WARN(Log::instance()->getLogger(), L"Error in IoHelper::getDirectoryEntry: " << Utility::formatIoError(parmsDbPath, ioError).c_str()); if (ioError == IoError::NoSuchFileOrDirectory) { - exitCause = ExitCause::FileAccessError; + exitCause = ExitCause::NotFound; } return ExitCode::SystemError; } From d6a510f6fa2d247e8ba15727d91e120deff26553 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:06:08 +0200 Subject: [PATCH 07/56] Add isParentOf method to Node. --- .../update_detection/update_detector/node.cpp | 26 ++++++--- .../update_detection/update_detector/node.h | 3 +- test/libsyncengine/CMakeLists.txt | 1 + test/libsyncengine/test.cpp | 2 + .../update_detector/testnode.cpp | 54 +++++++++++++++++++ .../update_detector/testnode.h | 38 +++++++++++++ .../update_detector/testupdatetree.cpp | 10 ---- .../update_detector/testupdatetree.h | 2 - 8 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 test/libsyncengine/update_detection/update_detector/testnode.cpp create mode 100644 test/libsyncengine/update_detection/update_detector/testnode.h diff --git a/src/libsyncengine/update_detection/update_detector/node.cpp b/src/libsyncengine/update_detection/update_detector/node.cpp index c9978593c..80c967016 100644 --- a/src/libsyncengine/update_detection/update_detector/node.cpp +++ b/src/libsyncengine/update_detection/update_detector/node.cpp @@ -20,6 +20,7 @@ #include "libcommon/utility/utility.h" #include "libcommonserver/utility/utility.h" +#include "libcommonserver/log/log.h" namespace KDC { @@ -174,16 +175,25 @@ SyncPath Node::getPath() const { return path; } -bool Node::isParentValid(std::shared_ptr parentNode) const { - if (!parentNode) return true; // `parentNode` is the root node, hence a valid parent. Stop climbing up the tree. - - while (parentNode) { - if (parentNode->id() == _id) - return false; // This node is a parent of `parentNode`, hence `parentNode` is not a valid parent. - parentNode = parentNode->parentNode(); +bool Node::isParentOf(std::shared_ptr potentialChild) const { + if (!potentialChild) return false; // `parentNode` is the root node, + if (potentialChild->id().has_value() && potentialChild->id() == _id) return false; // `parentNode` is the root node + while (potentialChild) { + if (!potentialChild->id().has_value()) { + LOG_ERROR(Log::instance()->getLogger(), "Node::isParentOf Node has no id"); + assert(false && "Node has no id"); + return false; + } + if (*potentialChild->id() == *_id) + return true; // This node is a parent of `potentialChild` + potentialChild = potentialChild->parentNode(); } - return true; + return false; +} + +bool Node::isParentValid(std::shared_ptr parentNode) const { + return !isParentOf(parentNode); } } // namespace KDC diff --git a/src/libsyncengine/update_detection/update_detector/node.h b/src/libsyncengine/update_detection/update_detector/node.h index 1f6f82aab..eb0786f06 100644 --- a/src/libsyncengine/update_detection/update_detector/node.h +++ b/src/libsyncengine/update_detection/update_detector/node.h @@ -111,6 +111,7 @@ class Node { [[nodiscard]] bool isRoot() const; [[nodiscard]] bool isCommonDocumentsFolder() const; [[nodiscard]] bool isSharedFolder() const; + [[nodiscard]] bool isParentOf(std::shared_ptr potentialChild) const; [[nodiscard]] SyncPath getPath() const; @@ -141,7 +142,7 @@ class Node { bool _isTmp = false; [[nodiscard]] bool isParentValid(std::shared_ptr parentNode) const; - friend class TestUpdateTree; + friend class TestNode; }; } // namespace KDC diff --git a/test/libsyncengine/CMakeLists.txt b/test/libsyncengine/CMakeLists.txt index b99438623..a874658dc 100644 --- a/test/libsyncengine/CMakeLists.txt +++ b/test/libsyncengine/CMakeLists.txt @@ -34,6 +34,7 @@ set(testsyncengine_SRCS ## Update Detector update_detection/update_detector/testupdatetree.h update_detection/update_detector/testupdatetree.cpp update_detection/update_detector/testupdatetreeworker.h update_detection/update_detector/testupdatetreeworker.cpp + update_detection/update_detector/testnode.h update_detection/update_detector/testnode.cpp # Reconciliation reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.h reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.cpp reconciliation/conflict_finder/testconflictfinderworker.h reconciliation/conflict_finder/testconflictfinderworker.cpp diff --git a/test/libsyncengine/test.cpp b/test/libsyncengine/test.cpp index 57ff2ed90..f5d080014 100644 --- a/test/libsyncengine/test.cpp +++ b/test/libsyncengine/test.cpp @@ -27,6 +27,7 @@ #include "update_detection/file_system_observer/testsnapshot.h" #include "update_detection/file_system_observer/testcomputefsoperationworker.h" #include "update_detection/update_detector/testupdatetree.h" +#include "update_detection/update_detector/testnode.h" #include "update_detection/update_detector/testupdatetreeworker.h" #include "reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.h" #include "reconciliation/conflict_finder/testconflictfinderworker.h" @@ -62,6 +63,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION(TestFolderWatcher_mac); CPPUNIT_TEST_SUITE_REGISTRATION(TestSnapshotItemHandler); CPPUNIT_TEST_SUITE_REGISTRATION(TestRemoteFileSystemObserverWorker); CPPUNIT_TEST_SUITE_REGISTRATION(TestComputeFSOperationWorker); +CPPUNIT_TEST_SUITE_REGISTRATION(TestNode); CPPUNIT_TEST_SUITE_REGISTRATION(TestUpdateTree); CPPUNIT_TEST_SUITE_REGISTRATION(TestUpdateTreeWorker); CPPUNIT_TEST_SUITE_REGISTRATION(TestPlatformInconsistencyCheckerWorker); diff --git a/test/libsyncengine/update_detection/update_detector/testnode.cpp b/test/libsyncengine/update_detection/update_detector/testnode.cpp new file mode 100644 index 000000000..fe28045d6 --- /dev/null +++ b/test/libsyncengine/update_detection/update_detector/testnode.cpp @@ -0,0 +1,54 @@ +/* + * Infomaniak kDrive - Desktop + * Copyright (C) 2023-2024 Infomaniak Network SA + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include "testnode.h" +#include "update_detection/update_detector/node.h" + +using namespace CppUnit; + +namespace KDC { + +void TestNode::testIsParentValid() { + const auto node1 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("Dir 1"), NodeType::Directory, + OperationType::None, "l1", 0, 0, 12345, nullptr); + const auto node11 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("Dir 1.1"), NodeType::Directory, + OperationType::None, "l11", 0, 0, 12345, node1); + + CPPUNIT_ASSERT(node11->isParentValid(node1)); + CPPUNIT_ASSERT(!node1->isParentValid(node11)); +} + +void TestNode::testIsParentOf() { + + const auto node1 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("Dir 1"), NodeType::Directory, + OperationType::None, "l1", 0, 0, 12345, nullptr); + const auto node11 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("Dir 1.1"), NodeType::Directory, + OperationType::None, "l11", 0, 0, 12345, node1); + const auto node111 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("Dir 1.1.1"), NodeType::Directory, + OperationType::None, "l111", 0, 0, 12345, node11); + + + CPPUNIT_ASSERT(node1->isParentOf(node11)); + CPPUNIT_ASSERT(!node11->isParentOf(node1)); + CPPUNIT_ASSERT(!node11->isParentOf(node11)); + CPPUNIT_ASSERT(node1->isParentOf(node111)); + CPPUNIT_ASSERT(node11->isParentOf(node111)); +} + +} // namespace KDC diff --git a/test/libsyncengine/update_detection/update_detector/testnode.h b/test/libsyncengine/update_detection/update_detector/testnode.h new file mode 100644 index 000000000..bd78e6fcc --- /dev/null +++ b/test/libsyncengine/update_detection/update_detector/testnode.h @@ -0,0 +1,38 @@ +/* + * Infomaniak kDrive - Desktop + * Copyright (C) 2023-2024 Infomaniak Network SA + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "testincludes.h" + +using namespace CppUnit; + +namespace KDC { +class TestNode : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(TestNode); + CPPUNIT_TEST(testIsParentValid); + CPPUNIT_TEST(testIsParentOf); + CPPUNIT_TEST_SUITE_END(); + + protected: + void testIsParentValid(); + void testIsParentOf(); + +}; + +} // namespace KDC diff --git a/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp b/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp index 0b4872b51..80a11b9bf 100644 --- a/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp +++ b/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp @@ -67,16 +67,6 @@ void TestUpdateTree::testConstructors() { } } -void TestUpdateTree::testIsParentValid() { - const auto node1 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1"), NodeType::Directory, - OperationType::None, "l1", 0, 0, 12345, _myTree->rootNode()); - const auto node11 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1.1"), NodeType::Directory, - OperationType::None, "l11", 0, 0, 12345, node1); - - CPPUNIT_ASSERT(node11->isParentValid(node1)); - CPPUNIT_ASSERT(!node1->isParentValid(node11)); -} - void TestUpdateTree::testInsertionOfFileNamesWithDifferentEncodings() { CPPUNIT_ASSERT(_myTree->_nodes.empty()); auto node1 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::None, diff --git a/test/libsyncengine/update_detection/update_detector/testupdatetree.h b/test/libsyncengine/update_detection/update_detector/testupdatetree.h index a32c44918..07504a51f 100644 --- a/test/libsyncengine/update_detection/update_detector/testupdatetree.h +++ b/test/libsyncengine/update_detection/update_detector/testupdatetree.h @@ -29,7 +29,6 @@ class FSOperationSet; class TestUpdateTree : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(TestUpdateTree); CPPUNIT_TEST(testConstructors); - CPPUNIT_TEST(testIsParentValid); CPPUNIT_TEST(testAll); CPPUNIT_TEST(testChangeEvents); CPPUNIT_TEST(testInsertionOfFileNamesWithDifferentEncodings); @@ -41,7 +40,6 @@ class TestUpdateTree : public CppUnit::TestFixture { protected: void testConstructors(); - void testIsParentValid(); void testAll(); void testChangeEvents(); void testInsertionOfFileNamesWithDifferentEncodings(); From a86084cdc5812fec06e9794e5f00ad13306dd8e6 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:17:17 +0200 Subject: [PATCH 08/56] Handle access denied in localFileSystemObserver --- .../localfilesystemobserverworker.cpp | 197 +++++++----------- .../localfilesystemobserverworker.h | 2 +- 2 files changed, 81 insertions(+), 118 deletions(-) diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index 5add3a8cf..723ed658a 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -132,12 +132,18 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listitemId(relativePath); @@ -291,6 +284,14 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listlocalPath(), absolutePath); bool isWarning = false; @@ -591,21 +592,14 @@ void LocalFileSystemObserverWorker::sendAccessDeniedError(const SyncPath &absolu return; } - // Check if path exists - bool exists = false; - if (!IoHelper::checkIfPathExists(absolutePath, exists, ioError)) { - LOGW_WARN(_logger, L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(absolutePath, ioError).c_str()); - return; - } - - if (exists) { - NodeId nodeId; - if (!IoHelper::getNodeId(absolutePath, nodeId)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in IoHelper::getLocalNodeId: " << Utility::formatSyncPath(absolutePath).c_str()); - } else { - // Blacklist file/directory - _syncPal->blacklistTemporarily(nodeId, relativePath, ReplicaSide::Local); - } + NodeId nodeId; + if (!IoHelper::getNodeId(absolutePath, nodeId)) { + nodeId = _snapshot->itemId(relativePath); + } + if(!nodeId.empty()){ + _syncPal->handleAccessDeniedItem(absolutePath, nodeId); + } else { + LOGW_SYNCPAL_WARN(_logger, L"Failed to handle access denied error (nodeId not found) for " << Utility::formatSyncPath(absolutePath)); } Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, absolutePath, ConflictType::None, InconsistencyType::None, @@ -613,29 +607,9 @@ void LocalFileSystemObserverWorker::sendAccessDeniedError(const SyncPath &absolu _syncPal->addError(error); } -ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParentDirPath) { +ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParentDirPath) { // Check if root dir exists - bool readPermission = false; - bool writePermission = false; - bool execPermission = false; IoError ioError = IoError::Success; - if (!IoHelper::getRights(absoluteParentDirPath, readPermission, writePermission, execPermission, ioError)) { - LOGW_WARN(_logger, L"Error in Utility::getRights: " << Utility::formatSyncPath(absoluteParentDirPath).c_str()); - setExitCause(ExitCause::FileAccessError); - return ExitCode::SystemError; - } - - if (ioError == IoError::NoSuchFileOrDirectory) { - LOGW_WARN(_logger, L"Error in IoHelper::getItemType: " << Utility::formatIoError(absoluteParentDirPath, ioError).c_str()); - setExitCause(ExitCause::SyncDirDoesntExist); - return ExitCode::SystemError; - } - - if (!writePermission) { - LOGW_DEBUG(_logger, - L"Item: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() << L" doesn't have write permissions!"); - return ExitCode::NoWritePermission; - } ItemType itemType; if (!IoHelper::getItemType(absoluteParentDirPath, itemType)) { @@ -665,61 +639,67 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen // Process all files try { - std::error_code ec; - auto dirIt = std::filesystem::recursive_directory_iterator( - absoluteParentDirPath, std::filesystem::directory_options::skip_permission_denied, ec); - if (ec) { - LOGW_SYNCPAL_DEBUG(_logger, L"Error in exploreDir: " << Utility::formatStdError(ec).c_str()); - setExitCause(ExitCause::FileAccessError); - return ExitCode::SystemError; + IoHelper::DirectoryIterator dirIt; + if (!IoHelper::getDirectoryIterator(absoluteParentDirPath, true, ioError, dirIt)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in IoHelper::getDirectoryIterator: " + << Utility::formatIoError(absoluteParentDirPath, ioError).c_str()); + setExitCause(ExitCause::Unknown); + return {ExitCode::SystemError, ExitCause::Unknown}; } - for (; dirIt != std::filesystem::recursive_directory_iterator(); ++dirIt) { + + if (ioError == IoError::NoSuchFileOrDirectory) { + LOGW_SYNCPAL_WARN(_logger, + L"Sync localpath: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() << L" doesn't exist"); + setExitCause(ExitCause::SyncDirDoesntExist); + return {ExitCode::SystemError, ExitCause::SyncDirDoesntExist}; + } + + if (ioError == IoError::AccessDenied) { + LOGW_SYNCPAL_WARN(_logger, L"Sync localpath: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() + << L" misses read permission"); + setExitCause(ExitCause::SyncDirReadError); + return {ExitCode::SystemError, ExitCause::SyncDirReadError}; + } + + + DirectoryEntry entry; + bool endOfDirectory = false; + while (dirIt.next(entry, endOfDirectory, ioError) && !endOfDirectory && ioError == IoError::Success) { if (ParametersCache::isExtendedLogEnabled()) { - LOGW_SYNCPAL_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(dirIt->path()).c_str() << L" found"); + LOGW_SYNCPAL_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(entry.path()).c_str() << L" found"); } if (stopAsked()) { return ExitCode::Ok; } - const SyncPath absolutePath = dirIt->path(); + const SyncPath absolutePath = entry.path(); const SyncPath relativePath = CommonUtility::relativePath(_syncPal->localPath(), absolutePath); bool toExclude = false; bool denyFullControl = false; - -#ifdef _WIN32 - // skip_permission_denied doesn't work on Windows if "Full control = Deny" - try { - bool dummy = dirIt->exists(); - (void) (dummy); - } catch (std::filesystem::filesystem_error &) { - LOGW_SYNCPAL_INFO(_logger, L"Item: " << Utility::formatSyncPath(absolutePath).c_str() - << L" rejected because access is denied"); - toExclude = true; - denyFullControl = true; - } -#endif - bool isLink = false; if (!toExclude) { // Check if the directory entry is managed bool isManaged = false; IoError ioError = IoError::Success; - if (!Utility::checkIfDirEntryIsManaged(dirIt, isManaged, isLink, ioError)) { + if (!Utility::checkIfDirEntryIsManaged(entry, isManaged, isLink, ioError)) { LOGW_SYNCPAL_WARN(_logger, L"Error in Utility::checkIfDirEntryIsManaged: " << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; - } else if (ioError == IoError::NoSuchFileOrDirectory) { + } + if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_SYNCPAL_DEBUG(_logger, L"Directory entry does not exist anymore: " << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; - } else if (ioError == IoError::AccessDenied) { + } + if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_DEBUG(_logger, L"Directory misses search permission: " << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); + sendAccessDeniedError(absolutePath); continue; } @@ -740,7 +720,7 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (!success) { LOGW_SYNCPAL_DEBUG(_logger, L"Error in ExclusionTemplateCache::isExcluded: " << Utility::formatIoError(absolutePath, ioError).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; } if (isExcluded) { @@ -757,46 +737,22 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (!IoHelper::getFileStat(absolutePath, &fileStat, ioError)) { LOGW_SYNCPAL_DEBUG(_logger, L"Error in IoHelper::getFileStat: " << Utility::formatIoError(absolutePath, ioError).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; } if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_SYNCPAL_DEBUG(_logger, L"Directory entry does not exist anymore: " << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; } else if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_INFO(_logger, L"Item: " << Utility::formatSyncPath(absolutePath).c_str() << L" rejected because access is denied"); sendAccessDeniedError(absolutePath); toExclude = true; - } else { // Check access permissions - nodeId = std::to_string(fileStat.inode); - readPermission = false; - writePermission = false; - execPermission = false; - if (!IoHelper::getRights(absolutePath, readPermission, writePermission, execPermission, ioError)) { - LOGW_SYNCPAL_WARN(_logger, - L"Error in Utility::getRights: " << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disable_recursion_pending(); - continue; - } - - if (ioError == IoError::NoSuchFileOrDirectory) { - LOGW_SYNCPAL_DEBUG(_logger, L"Directory entry does not exist anymore: " - << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disable_recursion_pending(); - continue; - } - - if (!readPermission || !writePermission || (dirIt->is_directory() && !execPermission)) { - LOGW_SYNCPAL_INFO(_logger, L"Item: " << Utility::formatSyncPath(absolutePath).c_str() - << L" rejected because access is denied"); - sendAccessDeniedError(absolutePath); - toExclude = true; - } } + nodeId = std::to_string(fileStat.inode); } if (toExclude) { @@ -807,7 +763,7 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen } } - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; } @@ -828,12 +784,13 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_SYNCPAL_DEBUG(_logger, L"Directory doesn't exist anymore: " << Utility::formatSyncPath(absolutePath.parent_path()).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; } else if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_DEBUG(_logger, L"Directory misses search permission: " << Utility::formatSyncPath(absolutePath.parent_path()).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); + sendAccessDeniedError(absolutePath); continue; } @@ -843,9 +800,15 @@ ExitCode LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (!IoHelper::getItemType(absolutePath, itemType)) { LOGW_SYNCPAL_DEBUG(_logger, L"Error in IoHelper::getItemType: " << Utility::formatIoError(absolutePath, itemType.ioError).c_str()); - dirIt.disable_recursion_pending(); + dirIt.disableRecursionPending(); continue; } + if (itemType.ioError == IoError::AccessDenied) { + LOGW_SYNCPAL_DEBUG(_logger, L"getItemType failed for item: " << Utility::formatSyncPath(absolutePath).c_str() + << L" with ioError: " << itemType.ioError + << L". Blacklisting it temporarily"); + sendAccessDeniedError(absolutePath); + } SnapshotItem item(nodeId, parentNodeId, absolutePath.filename().native(), fileStat.creationTime, fileStat.modtime, itemType.nodeType, fileStat.size, isLink, true, true); diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h index 3f2787b5a..836e3d37f 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h @@ -38,7 +38,7 @@ class LocalFileSystemObserverWorker : public FileSystemObserverWorker { protected: virtual void execute() override; - ExitCode exploreDir(const SyncPath &absoluteParentDirPath); + ExitInfo exploreDir(const SyncPath &absoluteParentDirPath); SyncPath _rootFolder; // std::unique_ptr _checksumWorker = nullptr; From 8319cd41c7e7cb81e048c4cda153d9ec17fb0a39 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:20:57 +0200 Subject: [PATCH 09/56] We don't skip edits on folder on Windows anymore to allow the detection of rights modifications. --- .../folderwatcher_win.cpp | 31 ++----------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp index e6ac3f9c1..4b30efa80 100644 --- a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp @@ -152,31 +152,6 @@ void FolderWatcher_win::watchChanges() { bool converted = false; OperationType opType = operationFromAction(notifInfo->Action); - if (notifInfo->Action == FILE_ACTION_MODIFIED) { - bool isDirectory = false; - IoError ioError = IoError::Success; - const bool isDirectorySuccess = IoHelper::checkIfIsDirectory(filepath, isDirectory, ioError); - if (!isDirectorySuccess) { - LOGW_WARN(_logger, - L"Error in IoHelper::checkIfIsDirectory: " << Utility::formatIoError(filepath, ioError).c_str()); - } - if (!isDirectory) { - if (ioError == IoError::NoSuchFileOrDirectory) { - LOGW_DEBUG(_logger, L"Convert operation " << opType << L" detected on item " - << Utility::formatSyncPath(filepath).c_str() - << L" to Delete (item doesn't exist)"); - opType = OperationType::Delete; - converted = true; - } - } else { - if (ParametersCache::isExtendedLogEnabled()) { - LOGW_DEBUG(_logger, L"Skip operation " << opType << L" detected on item with " - << Utility::formatSyncPath(filepath).c_str() << L" (directory)"); - } - skip = true; - } - } - // Unless the file was removed or renamed, get its full long name SyncPath longfilepath = filepath; if (!skip && opType != OperationType::Delete && notifInfo->Action != FILE_ACTION_RENAMED_OLD_NAME) { @@ -185,14 +160,14 @@ void FolderWatcher_win::watchChanges() { if (notFound) { // Item doesn't exist anymore LOGW_DEBUG(_logger, L"Convert operation " << opType << L" detected on item " - << Utility::formatSyncPath(longfilepath).c_str() + << Utility::formatSyncPath(longfilepath) << L" to Delete (item doesn't exist)"); opType = OperationType::Delete; converted = true; } else { // Keep original name LOGW_WARN(KDC::Log::instance()->getLogger(), - L"Error in Utility::longpath for path=" << Path2WStr(filepath).c_str()); + L"Error in Utility::longpath for path=" << Path2WStr(filepath)); } } } @@ -200,7 +175,7 @@ void FolderWatcher_win::watchChanges() { if (!skip) { if (ParametersCache::isExtendedLogEnabled()) { LOGW_DEBUG(_logger, L"Operation " << opType << (converted ? L"(converted) " : L"") << L" detected on item with " - << Utility::formatSyncPath(longfilepath).c_str()); + << Utility::formatSyncPath(longfilepath)); } changeDetected(longfilepath, opType); From 7724567be5ad520b6605a6c1c3907a21fe85bba7 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:22:34 +0200 Subject: [PATCH 10/56] Minor change in ioHelper. --- src/libcommonserver/io/iohelper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcommonserver/io/iohelper.cpp b/src/libcommonserver/io/iohelper.cpp index 9f898327b..7b43ec628 100644 --- a/src/libcommonserver/io/iohelper.cpp +++ b/src/libcommonserver/io/iohelper.cpp @@ -913,7 +913,7 @@ bool IoHelper::DirectoryIterator::next(DirectoryEntry &nextEntry, bool &endOfDir nextEntry = *_dirIterator; return true; } catch (std::filesystem::filesystem_error &) { - _dirIterator.disable_recursion_pending(); + disableRecursionPending(); return next(nextEntry, endOfDirectory, ioError); } From 387834dc3ae45b2c828e3f92ce844def65ccc3c2 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:23:45 +0200 Subject: [PATCH 11/56] Add Utility::checkIfDirEntryIsManaged(const DirectoryEntry &dirEntry, ...) --- src/libcommonserver/utility/utility.cpp | 20 ++++++++++++-------- src/libcommonserver/utility/utility.h | 5 +++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/libcommonserver/utility/utility.cpp b/src/libcommonserver/utility/utility.cpp index d7ef61fb9..495b5a908 100644 --- a/src/libcommonserver/utility/utility.cpp +++ b/src/libcommonserver/utility/utility.cpp @@ -682,36 +682,40 @@ SyncPath Utility::normalizedSyncPath(const SyncPath &path) noexcept { return result; } -bool Utility::checkIfDirEntryIsManaged(std::filesystem::recursive_directory_iterator &dirIt, bool &isManaged, bool &isLink, +bool Utility::checkIfDirEntryIsManaged(const std::filesystem::recursive_directory_iterator &dirIt, bool &isManaged, bool &isLink, IoError &ioError) { + return checkIfDirEntryIsManaged(*dirIt, isManaged, isLink, ioError); +} + +bool Utility::checkIfDirEntryIsManaged(const DirectoryEntry &dirEntry, bool &isManaged, bool &isLink, IoError &ioError) { isManaged = true; isLink = false; ioError = IoError::Success; ItemType itemType; - bool result = IoHelper::getItemType(dirIt->path(), itemType); + bool result = IoHelper::getItemType(dirEntry.path(), itemType); ioError = itemType.ioError; if (!result) { - LOGW_WARN(logger(), L"Error in IoHelper::getItemType: " << Utility::formatIoError(dirIt->path(), ioError).c_str()); + LOGW_WARN(logger(), L"Error in IoHelper::getItemType: " << Utility::formatIoError(dirEntry.path(), ioError).c_str()); return false; } if (itemType.ioError == IoError::NoSuchFileOrDirectory || itemType.ioError == IoError::AccessDenied) { - LOGW_DEBUG(logger(), L"Error in IoHelper::getItemType: " << formatIoError(dirIt->path(), ioError).c_str()); + LOGW_DEBUG(logger(), L"Error in IoHelper::getItemType: " << formatIoError(dirEntry.path(), ioError).c_str()); return true; } isLink = itemType.linkType != LinkType::None; - if (!dirIt->is_directory() && !dirIt->is_regular_file() && !isLink) { - LOGW_WARN(logger(), L"Ignore " << formatSyncPath(dirIt->path()).c_str() + if (!dirEntry.is_directory() && !dirEntry.is_regular_file() && !isLink) { + LOGW_WARN(logger(), L"Ignore " << formatSyncPath(dirEntry.path()).c_str() << L" because it's not a directory, a regular file or a symlink"); isManaged = false; return true; } - if (dirIt->path().native().length() > CommonUtility::maxPathLength()) { + if (dirEntry.path().native().length() > CommonUtility::maxPathLength()) { LOGW_WARN(logger(), - L"Ignore " << formatSyncPath(dirIt->path()).c_str() << L" because size > " << CommonUtility::maxPathLength()); + L"Ignore " << formatSyncPath(dirEntry.path()).c_str() << L" because size > " << CommonUtility::maxPathLength()); isManaged = false; return true; } diff --git a/src/libcommonserver/utility/utility.h b/src/libcommonserver/utility/utility.h index 13921c6b8..d4425a60e 100644 --- a/src/libcommonserver/utility/utility.h +++ b/src/libcommonserver/utility/utility.h @@ -162,8 +162,9 @@ struct COMMONSERVER_EXPORT Utility { static bool fileExists(DWORD dwordError) noexcept; static bool longPath(const SyncPath &shortPathIn, SyncPath &longPathOut, bool ¬Found); #endif - static bool checkIfDirEntryIsManaged(std::filesystem::recursive_directory_iterator &dirIt, bool &isManaged, bool &isLink, - IoError &ioError); + static bool checkIfDirEntryIsManaged(const DirectoryEntry &dirEntry, bool &isManaged, bool &isLink, IoError &ioError); + static bool checkIfDirEntryIsManaged(const std::filesystem::recursive_directory_iterator &dirIt, bool &isManaged, + bool &isLink, IoError &ioError); /* Resources analyser */ static bool totalRamAvailable(uint64_t &ram, int &errorCode); From a083f737284e59ee0fd731dc701c6a96c82f3e79 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:24:50 +0200 Subject: [PATCH 12/56] Avoid misleading error log. --- .../platforminconsistencycheckerworker.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp index be8087bac..df4875abb 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp @@ -40,10 +40,10 @@ void PlatformInconsistencyCheckerWorker::execute() { _syncPal->updateTree(ReplicaSide::Remote)->rootNode()->name()); for (const auto &idItem: _idsToBeRemoved) { - if (!_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(idItem.remoteId)) { + if (!idItem.remoteId.empty() && !_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(idItem.remoteId)) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node id=" << Utility::s2ws(idItem.remoteId.c_str())); } - if (!_syncPal->updateTree(ReplicaSide::Local)->deleteNode(idItem.localId)) { + if (!idItem.localId.empty() && !_syncPal->updateTree(ReplicaSide::Local)->deleteNode(idItem.localId)) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node id=" << Utility::s2ws(idItem.localId.c_str())); } } From afd4452213e736f138a5c487fb6f81824cd795a6 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:27:50 +0200 Subject: [PATCH 13/56] Handle file access errors in compute FSO --- .../computefsoperationworker.cpp | 49 +++++++------------ .../computefsoperationworker.h | 2 +- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp index 23ecffae9..d5a44c1ae 100644 --- a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp @@ -155,19 +155,16 @@ ExitCode ComputeFSOperationWorker::inferChangeFromDbNode(const ReplicaSide side, if (!pathInDeletedFolder(dbPath)) { // Check that the file/directory really does not exist on replica bool isExcluded = false; - if (const ExitCode exitCode = checkIfOkToDelete(side, dbPath, nodeId, isExcluded); exitCode != ExitCode::Ok) { - if (exitCode == ExitCode::NoWritePermission) { + if (const ExitInfo exitInfo = checkIfOkToDelete(side, dbPath, nodeId, isExcluded); !exitInfo) { + if (exitInfo.code() == ExitCode::SystemError && exitInfo.cause() == ExitCause::FileAccessError) { // Blacklist node - _syncPal->blacklistTemporarily(nodeId, dbPath, side); - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, dbPath, ConflictType::None, - InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, ExitCause::FileAccessError); - _syncPal->addError(error); + _syncPal->handleAccessDeniedItem(dbPath, nodeId); // Update unsynced list cache updateUnsyncedList(); return ExitCode::Ok; } else { - return exitCode; + return exitInfo; } } @@ -794,7 +791,7 @@ bool ComputeFSOperationWorker::isPathTooLong(const SyncPath &path, const NodeId return false; } -ExitCode ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const SyncPath &relativePath, const NodeId &nodeId, +ExitInfo ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const SyncPath &relativePath, const NodeId &nodeId, bool &isExcluded) { if (side != ReplicaSide::Local) return ExitCode::Ok; @@ -815,29 +812,18 @@ ExitCode ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const Syn if (ioError == IoError::InvalidFileName) { // Observed on MacOSX under special circumstances; see getItemType unit test edge cases. setExitCause(ExitCause::InvalidName); - } else - setExitCause(ExitCause::FileAccessError); + return {ExitCode::SystemError, ExitCause::InvalidName}; - return ExitCode::SystemError; + } else if (ioError == IoError::AccessDenied) { + setExitCause(ExitCause::FileAccessError); + return {ExitCode::SystemError, ExitCause::FileAccessError}; + } + setExitCause(ExitCause::Unknown); + return {ExitCode::SystemError, ExitCause::Unknown}; } if (!existsWithSameId) return ExitCode::Ok; - bool readPermission = false; - bool writePermission = false; - bool execPermission = false; - if (!IoHelper::getRights(absolutePath, readPermission, writePermission, execPermission, ioError)) { - LOGW_WARN(_logger, L"Error in Utility::getRights for " << Utility::formatSyncPath(absolutePath).c_str()); - setExitCause(ExitCause::FileAccessError); - return ExitCode::SystemError; - } - - if (!writePermission) { - LOGW_DEBUG(_logger, - L"Item with " << Utility::formatSyncPath(absolutePath).c_str() << L" doesn't have write permissions!"); - return ExitCode::NoWritePermission; - } - // Check if file is synced bool isWarning = false; ioError = IoError::Success; @@ -846,14 +832,14 @@ ExitCode ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const Syn if (!success) { LOGW_WARN(_logger, L"Error in ExclusionTemplateCache::isExcluded: " << Utility::formatIoError(absolutePath, ioError).c_str()); - setExitCause(ExitCause::FileAccessError); - return ExitCode::SystemError; + setExitCause(ExitCause::Unknown); + return {ExitCode::SystemError, ExitCause::Unknown}; } if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Item with " << Utility::formatSyncPath(absolutePath).c_str() << L" misses search permissions!"); - setExitCause(ExitCause::NoSearchPermission); - return ExitCode::SystemError; + setExitCause(ExitCause::FileAccessError); + return {ExitCode::SystemError, ExitCause::FileAccessError}; } if (isExcluded) return ExitCode::Ok; @@ -871,8 +857,7 @@ ExitCode ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const Syn "Unwanted local delete operation averted"); setExitCause(ExitCause::InvalidSnapshot); - - return ExitCode::DataError; // We need to rebuild the local snapshot from scratch. + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; // We need to rebuild the local snapshot from scratch. } void ComputeFSOperationWorker::deleteChildOpRecursively(const std::shared_ptr remoteSnapshot, diff --git a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.h b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.h index 7f5e4b55b..e3f218ce3 100644 --- a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.h +++ b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.h @@ -68,7 +68,7 @@ class ComputeFSOperationWorker : public ISyncWorker { bool isTooBig(const std::shared_ptr remoteSnapshot, const NodeId &remoteNodeId, int64_t size); bool isPathTooLong(const SyncPath &path, const NodeId &nodeId, NodeType type) const; - ExitCode checkIfOkToDelete(ReplicaSide side, const SyncPath &relativePath, const NodeId &nodeId, bool &isExcluded); + ExitInfo checkIfOkToDelete(ReplicaSide side, const SyncPath &relativePath, const NodeId &nodeId, bool &isExcluded); void deleteChildOpRecursively(const std::shared_ptr remoteSnapshot, const NodeId &remoteNodeId, std::unordered_set &tmpTooBigList); From 72bdee2ba94d0053dfe5a999e9dee87a2b3c978e Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:45:50 +0200 Subject: [PATCH 14/56] Handle file access errors in the executorworker. --- .../propagation/executor/executorworker.cpp | 260 +++++++++++------- .../propagation/executor/executorworker.h | 5 + 2 files changed, 170 insertions(+), 95 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index e61508f62..e127d6781 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -148,14 +148,21 @@ void ExecutorWorker::execute() { } } + if (executorExitInfo.cause() == ExitCause::OperationCanceled){ + setProgressComplete(syncOp, SyncFileStatus::Error); + continue; + } + if (!executorExitInfo) { if (!bypassProgressComplete) { setProgressComplete(syncOp, SyncFileStatus::Error); } - - increaseErrorCount(syncOp); - cancelAllOngoingJobs(); - break; + executorExitInfo = handleOpsExecutionError(syncOp, executorExitInfo); + if (!executorExitInfo) { + increaseErrorCount(syncOp); + cancelAllOngoingJobs(); + break; + } } if (job) { @@ -297,13 +304,124 @@ void ExecutorWorker::setProgressComplete(const SyncOpPtr syncOp, SyncFileStatus _syncPal->setProgressComplete(relativeLocalFilePath, status); } +ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { + assert((syncOp && !opsExitInfo) && "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); + if (!syncOp) { + LOG_WARN(_logger, "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); + return ExitCode::DataError; + } + if (opsExitInfo) { + return opsExitInfo; + } + + // Handle specific errors + switch (static_cast(opsExitInfo)) { + case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)): + case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::MoveToTrashFailed)): { + return handleOpsFileAccessError(syncOp, opsExitInfo); + } + default: { + break; + } + }; + LOG_WARN(_logger, + "Unhandled error in ExecutorWorker::handleOpsExecutionError: " << opsExitInfo.code() << " " << opsExitInfo.cause()); + return opsExitInfo; +} + +ExitInfo ExecutorWorker::handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { + if (syncOp->targetSide() == ReplicaSide::Local && syncOp->type() == OperationType::Create) { + // The item does not exist yet locally, we will only tmpBlacklist the remote item + _syncPal->handleAccessDeniedItem(syncOp->affectedNode()->getPath()); + } else { + // Both local and remote item will be temporarily blacklisted + auto localNode = syncOp->targetSide() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); + if (!localNode) return ExitCode::LogicError; + SyncPath relativeLocalFilePath = localNode->getPath(); + SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; + NodeId localNodeId = localNode->id().has_value() ? *localNode->id() : NodeId(); + _syncPal->handleAccessDeniedItem(absoluteLocalFilePath, localNodeId, opsExitInfo.cause()); + _syncPal->blacklistTemporarily(syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), + syncOp->nodePath(ReplicaSide::Local), otherSide(syncOp->targetSide())); + Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, + _syncPal->localPath() / syncOp->nodePath(ReplicaSide::Local), ConflictType::None, InconsistencyType::None, + CancelType::None, "", opsExitInfo.code(), opsExitInfo.cause()); + _syncPal->addError(error); + + if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + return ExitCode::DataError; + } + + if (syncOp->correspondingNode()) { + if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); + return ExitCode::DataError; + } + } + } + + removeDependentOps(syncOp); + return ExitCode::Ok; +} + +ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { + auto localNode = syncOp->affectedNode()->side() == ReplicaSide::Local ? syncOp->affectedNode() : syncOp->correspondingNode(); + auto remoteNode = + syncOp->affectedNode()->side() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); + + std::list dependentOps; + for (const auto &opId: _opList) { + SyncOpPtr syncOp2 = _syncPal->_syncOps->getOp(opId); + if (!syncOp2) { + LOGW_SYNCPAL_WARN(_logger, L"Operation doesn't exist anymore: id=" << opId); + continue; + } + auto localNode2 = + syncOp2->affectedNode()->side() == ReplicaSide::Local ? syncOp2->affectedNode() : syncOp2->correspondingNode(); + auto remoteNode2 = + syncOp2->affectedNode()->side() == ReplicaSide::Remote ? syncOp2->affectedNode() : syncOp2->correspondingNode(); + SyncName nodeName = localNode2 ? localNode2->name() : SyncName(); + nodeName = (nodeName.empty() && remoteNode2) ? remoteNode2->name() : SyncName(); + if (localNode && localNode2 && (*localNode->id() == *localNode2->id() || localNode->isParentOf(localNode2))) { + LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) + << L" because it depends on " << syncOp->type() << L" operation on " + << SyncName2WStr(localNode->name()) << L" wich failed."); + dependentOps.push_back(opId); + continue; + } + + if (remoteNode && remoteNode2 && (*remoteNode->id() == *remoteNode2->id() || remoteNode->isParentOf(remoteNode2))) { + LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) + << L" because it depends on " << syncOp->type() << L" operation on " + << SyncName2WStr(nodeName) << L"wich failed."); + dependentOps.push_back(opId); + } + } + + for (const auto &opId: dependentOps) { + _opList.remove(opId); + } + for (const auto &opId: dependentOps) { + removeDependentOps(_syncPal->_syncOps->getOp(opId)); + } + if (!dependentOps.empty()) { + LOGW_SYNCPAL_DEBUG(_logger, L"Removed " << dependentOps.size() << L" dependent operations."); + _syncPal->setRestart(true); + } + + return ExitCode::Ok; +} + ExitInfo ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &ignored) { // The execution of the create operation consists of three steps: // 1. If omit-flag is False, propagate the file or directory to target replica, because the object is missing there. // 2. Insert a new entry into the database, to avoid that the object is detected again by compute_ops() on the next sync // iteration. - // 3. Update the update tree structures to ensure that follow-up operations can execute correctly, as they are based on the - // information in these structures. + // 3. Update the update tree structures to ensure that follow-up operations can execute correctly, as they are based on + // the information in these structures. ignored = false; SyncPath relativeLocalFilePath = syncOp->nodePath(ReplicaSide::Local); @@ -552,7 +670,8 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptrnodePath(ReplicaSide::Local); @@ -1090,11 +1209,11 @@ ExitInfo ExecutorWorker::checkLiteSyncInfoForEdit(SyncOpPtr syncOp, const SyncPa ExitInfo ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { // The three execution steps are as follows: - // 1. If omit-flag is False, move the object on replica Y (where it still needs to be moved) from uY to vY, changing the name - // to nameX. + // 1. If omit-flag is False, move the object on replica Y (where it still needs to be moved) from uY to vY, changing the + // name to nameX. // 2. Update the database entry, to avoid detecting the move operation again. - // 3. If the omit flag is False, update the updatetreeY structure to ensure that follow-up operations can execute correctly, - // as they are based on the information in this structure. + // 3. If the omit flag is False, update the updatetreeY structure to ensure that follow-up operations can execute + // correctly, as they are based on the information in this structure. ignored = false; bypassProgressComplete = false; @@ -1147,8 +1266,8 @@ ExitInfo ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &ignored, bool &byp ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { bypassProgressComplete = false; - // 1. If omit-flag is False, move the object on replica Y (where it still needs to be moved) from uY to vY, changing the name - // to nameX. + // 1. If omit-flag is False, move the object on replica Y (where it still needs to be moved) from uY to vY, changing the + // name to nameX. std::shared_ptr job = nullptr; SyncPath relativeDestLocalFilePath; @@ -1259,7 +1378,7 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & // Propagate changes to DB and update trees std::shared_ptr newNode = nullptr; if (ExitInfo exitInfo = propagateChangeToDbAndTree(syncOp, job, newNode); !exitInfo) { - LOG_WARN(_logger, "Failed to propagate changes in DB or update tree for: " << syncOp->affectedNode()->name()); + LOGW_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << syncOp->affectedNode()->name()); return exitInfo; } @@ -1289,10 +1408,10 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & ExitInfo ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { // The three execution steps are as follows: // 1. If omit-flag is False, delete the file or directory on replicaY, because the objects till exists there - // 2. Remove the entry from the database. If nX is a directory node, also remove all entries for each node n ∈ S. This avoids - // that the object(s) are detected again by compute_ops() on the next sync iteration - // 3. Update the update tree structures to ensure that follow-up operations can execute correctly, as they are based on the - // information in these structures + // 2. Remove the entry from the database. If nX is a directory node, also remove all entries for each node n ∈ S. This + // avoids that the object(s) are detected again by compute_ops() on the next sync iteration + // 3. Update the update tree structures to ensure that follow-up operations can execute correctly, as they are based on + // the information in these structures ignored = false; bypassProgressComplete = false; @@ -1403,29 +1522,15 @@ bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { SyncPath relativeLocalFilePath = syncOp->nodePath(ReplicaSide::Local); SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; - bool readPermission = false; - bool writePermission = false; - bool execPermission = false; IoError ioError = IoError::Success; - if (!IoHelper::getRights(absoluteLocalFilePath, readPermission, writePermission, execPermission, ioError)) { - LOGW_WARN(_logger, L"Error in Utility::getRights: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); + if (!IoHelper::checkIfPathExists(absoluteLocalFilePath, exists, ioError)) { + LOGW_WARN(_logger, L"Error in Utility::checkIfPathExists: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); return false; } exists = ioError != IoError::NoSuchFileOrDirectory; if (syncOp->targetSide() == ReplicaSide::Local) { switch (syncOp->type()) { - case OperationType::Create: { - if (exists && !writePermission) { - LOGW_SYNCPAL_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str() - << L" already exists but doesn't have write permissions!"); - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, absoluteLocalFilePath, ConflictType::None, - InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, ExitCause::FileAccessError); - _syncPal->addError(error); - return false; - } - break; - } case OperationType::Edit: case OperationType::Move: case OperationType::Delete: { @@ -1434,17 +1539,9 @@ bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { << L" doesn't exist anymore!"); return false; } - - if (!writePermission) { - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, absoluteLocalFilePath, ConflictType::None, - InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, ExitCause::FileAccessError); - _syncPal->addError(error); - LOGW_SYNCPAL_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str() - << L" doesn't have write permissions!"); - return false; - } break; } + case OperationType::Create: case OperationType::None: default: { break; @@ -1458,13 +1555,6 @@ bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { << L" doesn't exist anymore!"); return false; } - - if (!writePermission) { - LOGW_SYNCPAL_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str() - << L" doesn't have write permissions!"); - return false; - } - std::shared_ptr newCorrespondingParentNode = nullptr; if (affectedUpdateTree(syncOp)->rootNode() == syncOp->affectedNode()->parentNode()) { newCorrespondingParentNode = targetUpdateTree(syncOp)->rootNode(); @@ -1601,29 +1691,30 @@ ExitInfo ExecutorWorker::deleteFinishedAsyncJobs() { SyncPath relativeLocalPath = syncOp->nodePath(ReplicaSide::Local); bool ignored = false; bool bypassProgressComplete = false; - if (ExitInfo handleFinishedJobExitInfo = - handleFinishedJob(job, syncOp, relativeLocalPath, ignored, bypassProgressComplete); - !handleFinishedJobExitInfo) { - increaseErrorCount(syncOp); - exitInfo = handleFinishedJobExitInfo; - } - + exitInfo = handleFinishedJob(job, syncOp, relativeLocalPath, ignored, bypassProgressComplete); if (exitInfo) { - if (ignored) { - setProgressComplete(syncOp, SyncFileStatus::Ignored); + if (!ignored && exitInfo.cause() == ExitCause::OperationCanceled) { + setProgressComplete(syncOp, SyncFileStatus::Error); + exitInfo = ExitCode::Ok; } else { - setProgressComplete(syncOp, SyncFileStatus::Success); - } + if (ignored) { + setProgressComplete(syncOp, SyncFileStatus::Ignored); + } else { + setProgressComplete(syncOp, SyncFileStatus::Success); + } - if (syncOp->affectedNode()->id().has_value()) { - std::unordered_set whiteList; - SyncNodeCache::instance()->syncNodes(_syncPal->syncDbId(), SyncNodeType::WhiteList, whiteList); - if (whiteList.find(syncOp->affectedNode()->id().value()) != whiteList.end()) { - // This item has been synchronized, it can now be removed from white list - whiteList.erase(syncOp->affectedNode()->id().value()); - SyncNodeCache::instance()->update(_syncPal->syncDbId(), SyncNodeType::WhiteList, whiteList); + if (syncOp->affectedNode()->id().has_value()) { + std::unordered_set whiteList; + SyncNodeCache::instance()->syncNodes(_syncPal->syncDbId(), SyncNodeType::WhiteList, whiteList); + if (whiteList.find(syncOp->affectedNode()->id().value()) != whiteList.end()) { + // This item has been synchronized, it can now be removed from white list + whiteList.erase(syncOp->affectedNode()->id().value()); + SyncNodeCache::instance()->update(_syncPal->syncDbId(), SyncNodeType::WhiteList, whiteList); + } } } + } else { + increaseErrorCount(syncOp); } // Delete job @@ -1734,36 +1825,15 @@ ExitInfo ExecutorWorker::handleFinishedJob(std::shared_ptr job, Syn L"Error in handleForbiddenAction for item: " << Utility::formatSyncPath(relativeLocalPath)); return exitInfo; } - } else if (job->exitCode() == ExitCode::SystemError && - (job->exitCause() == ExitCause::FileAccessError || job->exitCause() == ExitCause::MoveToTrashFailed)) { - LOGW_DEBUG(_logger, L"Item: " << Utility::formatSyncPath(relativeLocalPath).c_str() - << L" doesn't have write permissions or is locked!"); - _syncPal->blacklistTemporarily( - syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), relativeLocalPath, - otherSide(syncOp->targetSide())); - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, _syncPal->localPath() / relativeLocalPath, - ConflictType::None, InconsistencyType::None, CancelType::None, "", job->exitCode(), job->exitCause()); - _syncPal->addError(error); - - if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - return ExitCode::DataError; - } - - if (syncOp->correspondingNode()) { - if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); - return ExitCode::DataError; - } - } - } else { + } else if (!handleOpsExecutionError(syncOp, {job->exitCode(), job->exitCause()})) { // Cancel all queued jobs LOGW_SYNCPAL_WARN(_logger, L"Cancelling jobs. exit code: " << job->exitCode() << L" exit cause: " << job->exitCause()); cancelAllOngoingJobs(); return {job->exitCode(), job->exitCause()}; + + } else { // The error is managed and the execution can continue. + return {ExitCode::Ok, ExitCause::OperationCanceled}; } } else { // Propagate changes to DB and update trees diff --git a/src/libsyncengine/propagation/executor/executorworker.h b/src/libsyncengine/propagation/executor/executorworker.h index 1bfe85a17..4200fd470 100644 --- a/src/libsyncengine/propagation/executor/executorworker.h +++ b/src/libsyncengine/propagation/executor/executorworker.h @@ -141,6 +141,11 @@ class ExecutorWorker : public OperationProcessor { void setProgressComplete(const SyncOpPtr syncOp, SyncFileStatus status); + ExitInfo handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo); + ExitInfo handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo); + + ExitInfo removeDependentOps(SyncOpPtr syncOp); + std::unordered_map> _ongoingJobs; TerminatedJobsQueue _terminatedJobs; std::unordered_map _jobToSyncOpMap; From 12d0da53b7696de82a08237524a841b76749c85a Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:46:12 +0200 Subject: [PATCH 15/56] Adapt executor test to the new signatures. --- test/server/workers/testworkers.cpp | 13 +++++++++++++ test/server/workers/testworkers.h | 2 ++ 2 files changed, 15 insertions(+) diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index 2345e3e2b..a40b2dcee 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -296,4 +296,17 @@ void TestWorkers::testConvertToPlaceholder() { } } +void TestWorkers::testHandleOpsExecutionError() { + SyncOpPtr opPtr = std::make_shared(); + _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::Unknown)); + _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::InvalidSnapshot)); + _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::DbEntryNotFound)); + _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::Unknown)); + _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)); + _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::DriveNotRenew)); + + +} + + } // namespace KDC diff --git a/test/server/workers/testworkers.h b/test/server/workers/testworkers.h index 34a43a73f..661d47812 100644 --- a/test/server/workers/testworkers.h +++ b/test/server/workers/testworkers.h @@ -35,6 +35,7 @@ class TestWorkers : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(TestWorkers); CPPUNIT_TEST(testCreatePlaceholder); CPPUNIT_TEST(testConvertToPlaceholder); + CPPUNIT_TEST(testHandleOpsExecutionError); CPPUNIT_TEST_SUITE_END(); public: @@ -42,6 +43,7 @@ class TestWorkers : public CppUnit::TestFixture { void tearDown() override; void testCreatePlaceholder(); void testConvertToPlaceholder(); + void testHandleOpsExecutionError(); protected: static bool createPlaceholder(int syncDbId, const SyncPath &relativeLocalPath, const SyncFileItem &item); From 2126213bb5cebddfffcd19a38fa857a31fff9e90 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:46:35 +0200 Subject: [PATCH 16/56] Add SyncPal::handleAccessDeniedItem --- src/libsyncengine/syncpal/syncpal.cpp | 49 ++++++++++++++++++++++++++- src/libsyncengine/syncpal/syncpal.h | 4 +++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 5567b0773..8fd343c9e 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1138,7 +1138,7 @@ ExitCode SyncPal::fixCorruptedFile(const std::unordered_map &l DbNodeId dbId = -1; bool found = false; if (!_syncDb->dbId(ReplicaSide::Local, localFileInfo.first, dbId, found)) { - LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::dbId for nodeId=" << localFileInfo.first.c_str()); + LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::dbId for LocalNodeId=" << localFileInfo.first.c_str()); return ExitCode::DbError; } if (found) { @@ -1415,6 +1415,53 @@ void SyncPal::removeItemFromTmpBlacklist(const NodeId &nodeId, ReplicaSide side) _tmpBlacklistManager->removeItemFromTmpBlacklist(nodeId, side); } +void SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, const NodeId &LocalNodeId, ExitCause cause) { + + Error error(syncDbId(), "", "", NodeType::Unknown, relativePath, ConflictType::None, InconsistencyType::None, + CancelType::None, "", ExitCode::SystemError, cause); + addError(error); + LOGW_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(LocalNodeId) + << L" is blacklisted temporarily because of access denied"); + + std::optional remoteNodeId; + SyncPath remotePath; + + // Try to found the corresponding node in Db + NodeId correspondingNodeId; + if (bool found = false; !_syncDb->correspondingNodeId(ReplicaSide::Local, LocalNodeId, correspondingNodeId, found)) { + LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::correspondingNodeId"); + } + if (bool found = false; !_syncDb->path(ReplicaSide::Remote, correspondingNodeId, remotePath, found)) { + LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::path"); + } + remoteNodeId = correspondingNodeId; + + // If the corresponding node is not found, try to found the node in the snapshot + if (remoteNodeId->empty()) { + remoteNodeId = snapshotCopy(ReplicaSide::Remote)->itemId(relativePath); + if (remoteNodeId->empty()) { + LOGW_WARN(_logger, L"SyncPal::handleAccessDeniedItem Item " << Utility::formatSyncPath(relativePath) + << L" does not exist on the remote side."); + return; + } + remotePath = relativePath; + } + + if (!LocalNodeId.empty()) { // If the file/dir already exist on local FS, it will be blacklisted. + if (_tmpBlacklistManager->isTmpBlacklisted(LocalNodeId, ReplicaSide::Local)) { + LOGW_INFO(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(LocalNodeId) + << L" or one of its parent is already blacklisted temporarily."); + return; + } + + _tmpBlacklistManager->blacklistItem(LocalNodeId, relativePath, ReplicaSide::Local); + } + + if (remoteNodeId.has_value() && !remoteNodeId->empty() && !remotePath.empty()) { + _tmpBlacklistManager->blacklistItem(*remoteNodeId, remotePath, ReplicaSide::Remote); + } +} + void SyncPal::copySnapshots() { *_localSnapshotCopy = *_localSnapshot; *_remoteSnapshotCopy = *_remoteSnapshot; diff --git a/src/libsyncengine/syncpal/syncpal.h b/src/libsyncengine/syncpal/syncpal.h index 338e0434d..1640f6086 100644 --- a/src/libsyncengine/syncpal/syncpal.h +++ b/src/libsyncengine/syncpal/syncpal.h @@ -254,6 +254,10 @@ class SYNCENGINE_EXPORT SyncPal : public std::enable_shared_from_this { virtual void blacklistTemporarily(const NodeId &nodeId, const SyncPath &relativePath, ReplicaSide side); virtual void refreshTmpBlacklist(); virtual void removeItemFromTmpBlacklist(const NodeId &nodeId, ReplicaSide side); + + // If the item already exists on the local side, path and nodeId are required. Else, only the remote path is required. + void handleAccessDeniedItem(const SyncPath &relativePath, const NodeId &nodeId = NodeId(), + ExitCause cause = ExitCause::FileAccessError); //! Makes copies of real-time snapshots to be used by synchronization workers. void copySnapshots(); From 689cece2b6045b40889b1f4e3771d4452213d5d4 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 10:46:58 +0200 Subject: [PATCH 17/56] Handle displaying of file access errors. --- src/libparms/db/error.cpp | 8 ++++++++ src/server/appserver.cpp | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libparms/db/error.cpp b/src/libparms/db/error.cpp index 9d126c335..9f204a383 100644 --- a/src/libparms/db/error.cpp +++ b/src/libparms/db/error.cpp @@ -17,6 +17,7 @@ */ #include "error.h" +#include "libcommonserver/utility/utility.h" #include #include @@ -93,6 +94,13 @@ bool Error::isSimilarTo(const Error &other) const { return (_exitCode == other.exitCode()) && (_exitCause == other.exitCause()) && (_workerName == other.workerName()); } case ErrorLevel::Node: { + if (_exitCode == ExitCode::SystemError && _exitCause == ExitCause::FileAccessError && + other.exitCode() == ExitCode::SystemError && other.exitCause() == ExitCause::FileAccessError && + (Utility::startsWith(_path.lexically_normal(), other.path().lexically_normal()) || + Utility::startsWith(other.path().lexically_normal(), _path.lexically_normal()))) { + return true; + } + return (_conflictType == other.conflictType()) && (_inconsistencyType == other.inconsistencyType()) && (_cancelType == other.cancelType()) && (_path == other.path() && _destinationPath == other.destinationPath()); } diff --git a/src/server/appserver.cpp b/src/server/appserver.cpp index 1d0f3986c..86397c649 100644 --- a/src/server/appserver.cpp +++ b/src/server/appserver.cpp @@ -3883,7 +3883,9 @@ void AppServer::addError(const Error &error) { if (!existingError.isSimilarTo(error)) continue; // Update existing error time existingError.setTime(error.time()); - + auto shorterPath = + (existingError.path().string().size() > error.path().string().size()) ? error.path() : existingError.path(); + existingError.setPath(shorterPath); bool found = false; if (!ParmsDb::instance()->updateError(existingError, found)) { LOG_WARN(Log::instance()->getLogger(), "Error in ParmsDb::updateError"); @@ -3951,7 +3953,7 @@ void AppServer::addError(const Error &error) { JobManager::instance()->decreasePoolCapacity(); } - if (!ServerRequests::isAutoResolvedError(error)) { + if (!ServerRequests::isAutoResolvedError(error) && errorAlreadyExists) { // Send error to sentry only for technical errors SentryUser sentryUser(user.email().c_str(), user.name().c_str(), std::to_string(user.userId()).c_str()); SentryHandler::instance()->captureMessage(SentryLevel::Warning, "AppServer::addError", error.errorString().c_str(), From 8f1aff6a0c15d836435f8227f52745dff0a5f6ad Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 12:10:01 +0200 Subject: [PATCH 18/56] Move handleOpsFileAccessError down to avoid difficult to review git diff. --- .../propagation/executor/executorworker.cpp | 224 +++++++++--------- 1 file changed, 111 insertions(+), 113 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 09f0ca185..81676af51 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -277,8 +277,6 @@ void ExecutorWorker::initSyncFileItem(SyncOpPtr syncOp, SyncFileItem &syncItem) syncItem.setInstruction(SyncFileInstruction::Put); } } - - return; } void ExecutorWorker::logCorrespondingNodeErrorMsg(const SyncOpPtr syncOp) { @@ -304,117 +302,6 @@ void ExecutorWorker::setProgressComplete(const SyncOpPtr syncOp, SyncFileStatus _syncPal->setProgressComplete(relativeLocalFilePath, status); } -ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { - assert((syncOp && !opsExitInfo) && "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); - if (!syncOp) { - LOG_WARN(_logger, "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); - return ExitCode::DataError; - } - if (opsExitInfo) { - return opsExitInfo; - } - - // Handle specific errors - switch (static_cast(opsExitInfo)) { - case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)): - case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::MoveToTrashFailed)): { - return handleOpsFileAccessError(syncOp, opsExitInfo); - } - default: { - break; - } - }; - LOG_WARN(_logger, - "Unhandled error in ExecutorWorker::handleOpsExecutionError: " << opsExitInfo.code() << " " << opsExitInfo.cause()); - return opsExitInfo; -} - -ExitInfo ExecutorWorker::handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { - if (syncOp->targetSide() == ReplicaSide::Local && syncOp->type() == OperationType::Create) { - // The item does not exist yet locally, we will only tmpBlacklist the remote item - _syncPal->handleAccessDeniedItem(syncOp->affectedNode()->getPath()); - } else { - // Both local and remote item will be temporarily blacklisted - auto localNode = syncOp->targetSide() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); - if (!localNode) return ExitCode::LogicError; - SyncPath relativeLocalFilePath = localNode->getPath(); - SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; - NodeId localNodeId = localNode->id().has_value() ? *localNode->id() : NodeId(); - _syncPal->handleAccessDeniedItem(absoluteLocalFilePath, localNodeId, opsExitInfo.cause()); - _syncPal->blacklistTemporarily(syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), - syncOp->nodePath(ReplicaSide::Local), otherSide(syncOp->targetSide())); - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, - _syncPal->localPath() / syncOp->nodePath(ReplicaSide::Local), ConflictType::None, InconsistencyType::None, - CancelType::None, "", opsExitInfo.code(), opsExitInfo.cause()); - _syncPal->addError(error); - - if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - return ExitCode::DataError; - } - - if (syncOp->correspondingNode()) { - if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); - return ExitCode::DataError; - } - } - } - - removeDependentOps(syncOp); - return ExitCode::Ok; -} - -ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { - auto localNode = syncOp->affectedNode()->side() == ReplicaSide::Local ? syncOp->affectedNode() : syncOp->correspondingNode(); - auto remoteNode = - syncOp->affectedNode()->side() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); - - std::list dependentOps; - for (const auto &opId: _opList) { - SyncOpPtr syncOp2 = _syncPal->_syncOps->getOp(opId); - if (!syncOp2) { - LOGW_SYNCPAL_WARN(_logger, L"Operation doesn't exist anymore: id=" << opId); - continue; - } - auto localNode2 = - syncOp2->affectedNode()->side() == ReplicaSide::Local ? syncOp2->affectedNode() : syncOp2->correspondingNode(); - auto remoteNode2 = - syncOp2->affectedNode()->side() == ReplicaSide::Remote ? syncOp2->affectedNode() : syncOp2->correspondingNode(); - SyncName nodeName = localNode2 ? localNode2->name() : SyncName(); - nodeName = (nodeName.empty() && remoteNode2) ? remoteNode2->name() : SyncName(); - if (localNode && localNode2 && (*localNode->id() == *localNode2->id() || localNode->isParentOf(localNode2))) { - LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) - << L" because it depends on " << syncOp->type() << L" operation on " - << SyncName2WStr(localNode->name()) << L" wich failed."); - dependentOps.push_back(opId); - continue; - } - - if (remoteNode && remoteNode2 && (*remoteNode->id() == *remoteNode2->id() || remoteNode->isParentOf(remoteNode2))) { - LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) - << L" because it depends on " << syncOp->type() << L" operation on " - << SyncName2WStr(nodeName) << L"wich failed."); - dependentOps.push_back(opId); - } - } - - for (const auto &opId: dependentOps) { - _opList.remove(opId); - } - for (const auto &opId: dependentOps) { - removeDependentOps(_syncPal->_syncOps->getOp(opId)); - } - if (!dependentOps.empty()) { - LOGW_SYNCPAL_DEBUG(_logger, L"Removed " << dependentOps.size() << L" dependent operations."); - _syncPal->setRestart(true); - } - - return ExitCode::Ok; -} - ExitInfo ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptr &job, bool &ignored) { // The execution of the create operation consists of three steps: // 1. If omit-flag is False, propagate the file or directory to target replica, because the object is missing there. @@ -2667,4 +2554,115 @@ ExitInfo ExecutorWorker::getFileSize(const SyncPath &path, uint64_t &size) { return ExitCode::Ok; } +ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { + assert((syncOp && !opsExitInfo) && "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); + if (!syncOp) { + LOG_WARN(_logger, "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); + return ExitCode::DataError; + } + if (opsExitInfo) { + return opsExitInfo; + } + + // Handle specific errors + switch (static_cast(opsExitInfo)) { + case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)): + case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::MoveToTrashFailed)): { + return handleOpsFileAccessError(syncOp, opsExitInfo); + } + default: { + break; + } + }; + LOG_WARN(_logger, + "Unhandled error in ExecutorWorker::handleOpsExecutionError: " << opsExitInfo.code() << " " << opsExitInfo.cause()); + return opsExitInfo; +} + +ExitInfo ExecutorWorker::handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { + if (syncOp->targetSide() == ReplicaSide::Local && syncOp->type() == OperationType::Create) { + // The item does not exist yet locally, we will only tmpBlacklist the remote item + _syncPal->handleAccessDeniedItem(syncOp->affectedNode()->getPath()); + } else { + // Both local and remote item will be temporarily blacklisted + auto localNode = syncOp->targetSide() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); + if (!localNode) return ExitCode::LogicError; + SyncPath relativeLocalFilePath = localNode->getPath(); + SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; + NodeId localNodeId = localNode->id().has_value() ? *localNode->id() : NodeId(); + _syncPal->handleAccessDeniedItem(absoluteLocalFilePath, localNodeId, opsExitInfo.cause()); + _syncPal->blacklistTemporarily(syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), + syncOp->nodePath(ReplicaSide::Local), otherSide(syncOp->targetSide())); + Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, + _syncPal->localPath() / syncOp->nodePath(ReplicaSide::Local), ConflictType::None, InconsistencyType::None, + CancelType::None, "", opsExitInfo.code(), opsExitInfo.cause()); + _syncPal->addError(error); + + if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + return ExitCode::DataError; + } + + if (syncOp->correspondingNode()) { + if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); + return ExitCode::DataError; + } + } + } + + removeDependentOps(syncOp); + return ExitCode::Ok; +} + +ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { + auto localNode = syncOp->affectedNode()->side() == ReplicaSide::Local ? syncOp->affectedNode() : syncOp->correspondingNode(); + auto remoteNode = + syncOp->affectedNode()->side() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); + + std::list dependentOps; + for (const auto &opId: _opList) { + SyncOpPtr syncOp2 = _syncPal->_syncOps->getOp(opId); + if (!syncOp2) { + LOGW_SYNCPAL_WARN(_logger, L"Operation doesn't exist anymore: id=" << opId); + continue; + } + auto localNode2 = + syncOp2->affectedNode()->side() == ReplicaSide::Local ? syncOp2->affectedNode() : syncOp2->correspondingNode(); + auto remoteNode2 = + syncOp2->affectedNode()->side() == ReplicaSide::Remote ? syncOp2->affectedNode() : syncOp2->correspondingNode(); + SyncName nodeName = localNode2 ? localNode2->name() : SyncName(); + nodeName = (nodeName.empty() && remoteNode2) ? remoteNode2->name() : SyncName(); + if (localNode && localNode2 && (*localNode->id() == *localNode2->id() || localNode->isParentOf(localNode2))) { + LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) + << L" because it depends on " << syncOp->type() << L" operation on " + << SyncName2WStr(localNode->name()) << L" wich failed."); + dependentOps.push_back(opId); + continue; + } + + if (remoteNode && remoteNode2 && (*remoteNode->id() == *remoteNode2->id() || remoteNode->isParentOf(remoteNode2))) { + LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) + << L" because it depends on " << syncOp->type() << L" operation on " + << SyncName2WStr(nodeName) << L"wich failed."); + dependentOps.push_back(opId); + } + } + + for (const auto &opId: dependentOps) { + _opList.remove(opId); + } + for (const auto &opId: dependentOps) { + removeDependentOps(_syncPal->_syncOps->getOp(opId)); + } + if (!dependentOps.empty()) { + LOGW_SYNCPAL_DEBUG(_logger, L"Removed " << dependentOps.size() << L" dependent operations."); + _syncPal->setRestart(true); + } + + return ExitCode::Ok; +} + } // namespace KDC From bfeff4529474069f5425c5976273f9ce42e4378a Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 13:00:21 +0200 Subject: [PATCH 19/56] Fix executorWorker merge. --- src/libsyncengine/propagation/executor/executorworker.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 81676af51..94c0ac78f 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -2142,10 +2142,10 @@ ExitInfo ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId if (!_syncPal->_syncDb->node(*syncOp->correspondingNode()->idb(), dbNode, found)) { LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::node"); return {ExitCode::DbError, ExitCause::DbAccessError}; - if (!found) { - LOG_SYNCPAL_DEBUG(_logger, "Failed to retrieve node for dbId=" << *syncOp->correspondingNode()->idb()); - return {ExitCode::DataError, ExitCause::DbEntryNotFound}; - } + } + if (!found) { + LOG_SYNCPAL_DEBUG(_logger, "Failed to retrieve node for dbId=" << *syncOp->correspondingNode()->idb()); + return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } // 2. Update the database entry, to avoid detecting the edit operation again. From 3eba13d4423c15993893d121095e2865ed3abe60 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 13:30:15 +0200 Subject: [PATCH 20/56] Merge develop --- .github/workflows/windows.yml | 3 + .../translate_release_notes.py | 54 +++--- src/libcommon/utility/utility.cpp | 7 +- src/libsyncengine/db/syncdb.cpp | 21 ++- src/libsyncengine/progress/progressinfo.cpp | 33 ++-- src/libsyncengine/progress/progressinfo.h | 33 ++-- src/libsyncengine/progress/syncfileitem.h | 2 + .../conflictresolverworker.cpp | 6 +- .../platforminconsistencycheckerutility.cpp | 27 +-- .../platforminconsistencycheckerutility.h | 15 +- .../platforminconsistencycheckerworker.cpp | 177 +++++++++++++----- .../platforminconsistencycheckerworker.h | 9 +- src/libsyncengine/syncpal/syncpal.cpp | 5 +- .../blacklistpropagator.cpp | 2 +- .../computefsoperationworker.cpp | 4 +- .../snapshot/snapshot.cpp | 9 +- .../update_detection/update_detector/node.cpp | 6 + .../update_detection/update_detector/node.h | 5 + src/server/vfs/mac/vfs_mac.cpp | 26 +-- test/libsyncengine/db/testsyncdb.cpp | 9 + test/libsyncengine/db/testsyncdb.h | 2 + .../testoperationsorterworker.cpp | 10 +- .../testconflictfinderworker.cpp | 3 +- .../testconflictresolverworker.cpp | 4 +- ...testplatforminconsistencycheckerworker.cpp | 146 ++++++++++++--- .../testplatforminconsistencycheckerworker.h | 3 + test/libsyncengine/syncpal/testsyncpal.cpp | 47 +++++ test/libsyncengine/syncpal/testsyncpal.h | 3 + .../update_detector/testupdatetree.cpp | 40 ++-- test/server/workers/testworkers.cpp | 10 +- 30 files changed, 482 insertions(+), 239 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f03c3b5ad..09907c7ea 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -65,6 +65,7 @@ jobs: ./infomaniak-build-tools/run-tests.ps1 - name: Compute code coverage + if: false run : | rm -Force -ErrorAction SilentlyContinue BullseyeCoverageExclusions echo "exclude folder 3rdparty/\nexclude folder ../" >> BullseyeCoverageExclusions @@ -74,11 +75,13 @@ jobs: & "covxml.exe" "-f" "$env:COVFILE" "-o" "coverage.xml" - name: Install sonar-scanner + if: false uses: SonarSource/sonarcloud-github-c-cpp@v3 env: SONAR_HOST_URL: https://sonarcloud.io - name: Run sonar-scanner + if: false env: GITHUB_TOKEN: ${{ github.token }} SONAR_TOKEN: ${{ secrets.SONAR_DESKTOP_KDRIVE_TOKEN }} diff --git a/infomaniak-build-tools/translate_release_notes.py b/infomaniak-build-tools/translate_release_notes.py index 3d0c58bfb..6021e98c7 100755 --- a/infomaniak-build-tools/translate_release_notes.py +++ b/infomaniak-build-tools/translate_release_notes.py @@ -22,6 +22,7 @@ import deepl import errno import os +from pathlib import Path import re import shutil import subprocess @@ -54,18 +55,23 @@ def version_regex(arg_value, pattern=re.compile(r'^(\d+\.)?(\d+\.)?(\*|\d+)')): parser.add_argument('-d', '--date', metavar="", type=int, help='The planned release date (defaults to today)', default=datetime.date.today().strftime('%Y%m%d')) args = parser.parse_args() -if not os.path.isfile(f"kDrive-template.html"): - sys.exit("Unable to find kDrive-template.html."); +if not os.path.isfile("kDrive-template.html"): + sys.exit("Unable to find 'kDrive-template.html'.") fullName = f"kDrive-{args.version}.{args.date}" -dirPath = f"../release_notes/{fullName}" +dirPath = Path(__file__).parent.parent / "release_notes" + +if not dirPath.exists(): + sys.exit(f"Release notes path does not exist: '{dirPath}'. \nAborting.") + +dirPath = dirPath / Path(fullName) deepl_key = os.getenv("DEEPL_AUTH_KEY") if not deepl_key: - sys.exit("error: The DeepL API key is not set in env"); + sys.exit("error: The DeepL API key is not set in env") translator = deepl.Translator(deepl_key) -target_lang = [ +target_languages = [ 'FR', 'DE', 'ES', @@ -78,16 +84,15 @@ def version_regex(arg_value, pattern=re.compile(r'^(\d+\.)?(\d+\.)?(\*|\d+)')): 'macOS' ] -def split_os(lang): - lang_ext = f"-{lang.lower()}" if lang != 'en' else "" +def split_os(lang, fullName): + lang_ext = lang.lower() for os_name in os_list: - os_ext = f"-{os_name.lower()}" if os_name != 'macOS' else "" - if os_name != 'macOS': - shutil.copyfile(f"{fullName}{lang_ext}.html", f"{fullName}{os_ext}{lang_ext}.html") - with open(f"{fullName}{os_ext}{lang_ext}.html", "r") as f: + os_ext = os_name.lower() + shutil.copyfile(f"{fullName}-{lang_ext}.html", f"{fullName}-{os_ext}-{lang_ext}.html") + with open(f"{fullName}-{os_ext}-{lang_ext}.html", "r") as f: lines = f.readlines() - with open(f"{fullName}{os_ext}{lang_ext}.html", "w") as f: + with open(f"{fullName}-{os_ext}-{lang_ext}.html", "w") as f: for line in lines: if any(os_note in line for os_note in os_list): if (f"
  • {os_name}" in line): @@ -96,29 +101,26 @@ def split_os(lang): f.write(line) print(f"Generating Release Notes for kDrive-{args.version}.{args.date}") +Path(dirPath).mkdir(mode=0o755, exist_ok=True) -try: - os.mkdir(dirPath, mode = 0o755) -except OSError as exc: - if exc.errno != errno.EEXIST: - raise - pass - -shutil.copyfile("kDrive-template.html", f"{dirPath}/{fullName}.html") +shutil.copyfile("kDrive-template.html", f"{dirPath}/{fullName}-en.html") os.chdir(dirPath) try: - for lang in target_lang: + for target_lang in target_languages: + print(f" - Notes for language {target_lang}") translator.translate_document_from_filepath( - f"{fullName}.html", - f"{fullName}-{lang.lower()}.html", - target_lang=lang, + f"{fullName}-en.html", + f"{fullName}-{target_lang.lower()}.html", + target_lang=target_lang, ) - split_os(lang) + split_os(target_lang, fullName) + Path(f"{fullName}-{target_lang.lower()}.html").unlink() except Exception as e: print(e) -split_os('en') +split_os('en', fullName) +Path(f"{fullName}-en.html").unlink() # Remove english template subprocess.run(['../../infomaniak-build-tools/encode.sh'], shell=True) diff --git a/src/libcommon/utility/utility.cpp b/src/libcommon/utility/utility.cpp index 3a537a24b..f99c4513d 100644 --- a/src/libcommon/utility/utility.cpp +++ b/src/libcommon/utility/utility.cpp @@ -655,8 +655,8 @@ size_t CommonUtility::maxPathLength() { // For folders in short path mode, it is MAX_PATH - 12 // (https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry) // We decided to apply this rule for files too. We could else encounter issues. - // If the path length of a folder is > MAX_PATH - 12 and the path length of a file in this folder is between MAX_PATH - 12 and MAX_PATH. - // It would lead to a synced file in a folder that is not synced (hence excluded because of its path length). + // If the path length of a folder is > MAX_PATH - 12 and the path length of a file in this folder is between MAX_PATH - 12 and + // MAX_PATH. It would lead to a synced file in a folder that is not synced (hence excluded because of its path length). return (_maxPathWin == MAX_PATH_LENGTH_WIN_LONG) ? MAX_PATH_LENGTH_WIN_LONG : MAX_PATH_LENGTH_WIN_SHORT - 12; #elif defined(__APPLE__) return MAX_PATH_LENGTH_MAC; @@ -728,7 +728,8 @@ bool CommonUtility::isVersionLower(const std::string ¤tVersion, const std: return false; // Should not happen } - return std::ranges::lexicographical_compare(currTabVersion, targetTabVersion); + return std::lexicographical_compare(currTabVersion.begin(), currTabVersion.end(), targetTabVersion.begin(), + targetTabVersion.end()); } static std::string tmpDirName = "kdrive_" + CommonUtility::generateRandomStringAlphaNum(); diff --git a/src/libsyncengine/db/syncdb.cpp b/src/libsyncengine/db/syncdb.cpp index 161f81c1e..47dd7268d 100644 --- a/src/libsyncengine/db/syncdb.cpp +++ b/src/libsyncengine/db/syncdb.cpp @@ -2393,8 +2393,6 @@ bool SyncDb::reinstateEncodingOfLocalNames(const std::string &dbFromVersionNumbe LOG_DEBUG(_logger, "Upgrade < 3.6.7 Sync DB"); - normalizeRemoteNames(); - Sync sync; bool found = false; ParmsDb::instance()->selectSync(_dbPath, sync, found); @@ -2405,6 +2403,21 @@ bool SyncDb::reinstateEncodingOfLocalNames(const std::string &dbFromVersionNumbe const SyncPath &localDrivePath = sync.localPath(); + bool exists = false; + IoError existenceCheckError = IoError::Success; + if (!IoHelper::checkIfPathExists(localDrivePath, exists, existenceCheckError)) { + LOGW_WARN(_logger, + L"Error in IoHelper::checkIfPathExists" << Utility::formatIoError(localDrivePath, existenceCheckError)); + return false; + } + if (!exists) { + LOGW_INFO(_logger, L"The synchronisation folder " << Utility::formatSyncPath(localDrivePath) + << L" does not exist anymore. No Sync DB upgrade to do."); + return true; + } + + if (!normalizeRemoteNames()) return false; + NamedNodeMap namedNodeMap; if (!selectNamesWithDistinctEncodings(namedNodeMap)) return false; @@ -2412,8 +2425,8 @@ bool SyncDb::reinstateEncodingOfLocalNames(const std::string &dbFromVersionNumbe IoError ioError = IoError::Success; IoHelper::getDirectoryIterator(localDrivePath, true, ioError, dir); if (ioError != IoError::Success) { - LOGW_WARN(_logger, L"Error in DirectoryIterator: " << Utility::formatIoError(localDrivePath, ioError).c_str()); - return false; + LOGW_WARN(_logger, L"Error in DirectoryIterator: " << Utility::formatIoError(localDrivePath, ioError)); + return (ioError == IoError::NoSuchFileOrDirectory) || (ioError == IoError::AccessDenied); } SyncNameMap localNames; diff --git a/src/libsyncengine/progress/progressinfo.cpp b/src/libsyncengine/progress/progressinfo.cpp index 7fdd08ede..4359637f7 100644 --- a/src/libsyncengine/progress/progressinfo.cpp +++ b/src/libsyncengine/progress/progressinfo.cpp @@ -21,8 +21,7 @@ namespace KDC { -ProgressInfo::ProgressInfo(std::shared_ptr syncPal) : - _syncPal(syncPal), _totalSizeOfCompletedJobs(0), _maxFilesPerSecond(0), _maxBytesPerSecond(0), _update(false) { +ProgressInfo::ProgressInfo(std::shared_ptr syncPal) : _syncPal(syncPal) { reset(); } @@ -77,35 +76,34 @@ void ProgressInfo::updateEstimates() { } void ProgressInfo::initProgress(const SyncFileItem &item) { - SyncPath path = item.newPath().has_value() ? item.newPath().value() : item.path(); + const SyncPath path = item.newPath().has_value() ? item.newPath().value() : item.path(); ProgressItem progressItem; progressItem.setItem(item); progressItem.progress().setTotal(item.size()); progressItem.progress().setCompleted(0); - _currentItems[path].push(progressItem); + _currentItems[Utility::normalizedSyncPath(path)].push(progressItem); _fileProgress.setTotal(_fileProgress.total() + 1); _sizeProgress.setTotal(_sizeProgress.total() + item.size()); } bool ProgressInfo::getSyncFileItem(const SyncPath &path, SyncFileItem &item) { - auto it = _currentItems.find(path); - if (_currentItems.find(path) == _currentItems.end() || it->second.empty()) { + const auto it = _currentItems.find(Utility::normalizedSyncPath(path)); + if (it == _currentItems.end() || it->second.empty()) { return false; } item = it->second.front().item(); return true; } -void ProgressInfo::setProgress(const SyncPath &path, int64_t completed) { - auto it = _currentItems.find(path); +void ProgressInfo::setProgress(const SyncPath &path, const int64_t completed) { + const auto it = _currentItems.find(Utility::normalizedSyncPath(path)); if (it == _currentItems.end() || it->second.empty()) { return; } - SyncFileItem &item = it->second.front().item(); - if (!shouldCountProgress(item)) { + if (const SyncFileItem &item = it->second.front().item(); !shouldCountProgress(item)) { return; } @@ -113,8 +111,8 @@ void ProgressInfo::setProgress(const SyncPath &path, int64_t completed) { recomputeCompletedSize(); } -void ProgressInfo::setProgressComplete(const SyncPath &path, SyncFileStatus status) { - auto it = _currentItems.find(path); +void ProgressInfo::setProgressComplete(const SyncPath &path, const SyncFileStatus status) { + const auto it = _currentItems.find(Utility::normalizedSyncPath(path)); if (it == _currentItems.end() || it->second.empty()) { return; } @@ -135,7 +133,7 @@ void ProgressInfo::setProgressComplete(const SyncPath &path, SyncFileStatus stat it->second.pop(); if (it->second.empty()) { - _currentItems.erase(path); + _currentItems.erase(Utility::normalizedSyncPath(path)); } recomputeCompletedSize(); } @@ -183,15 +181,6 @@ bool ProgressInfo::trustEta() const { return totalProgress().estimatedEta() < 100 * optimisticEta(); } -Estimates ProgressInfo::fileProgress(const SyncFileItem &item) { - auto it = _currentItems.find(item.path()); - if (it == _currentItems.end() || it->second.empty()) { - return Estimates(); - } - - return it->second.front().progress().estimates(); -} - void ProgressInfo::recomputeCompletedSize() { int64_t r = _totalSizeOfCompletedJobs; for (auto &itemElt: _currentItems) { diff --git a/src/libsyncengine/progress/progressinfo.h b/src/libsyncengine/progress/progressinfo.h index dd488c316..0ee93cdf1 100644 --- a/src/libsyncengine/progress/progressinfo.h +++ b/src/libsyncengine/progress/progressinfo.h @@ -34,24 +34,24 @@ class SyncPal; class ProgressInfo { public: - ProgressInfo(std::shared_ptr syncPal); + explicit ProgressInfo(std::shared_ptr syncPal); ~ProgressInfo(); void reset(); - inline void setUpdate(bool value) { _update = value; } - inline bool update() const { return _update; } + void setUpdate(bool value) { _update = value; } + [[nodiscard]] bool update() const { return _update; } void updateEstimates(); void initProgress(const SyncFileItem &item); void setProgress(const SyncPath &path, int64_t completed); void setProgressComplete(const SyncPath &path, SyncFileStatus status); bool getSyncFileItem(const SyncPath &path, SyncFileItem &item); - inline int64_t totalFiles() const { return _fileProgress.total(); } - inline int64_t completedFiles() const { return _fileProgress.completed(); } - inline int64_t totalSize() const { return _sizeProgress.total(); } - inline int64_t completedSize() const { return _sizeProgress.completed(); } - inline int64_t currentFile() const { return completedFiles(); } - Estimates totalProgress() const; + [[nodiscard]] int64_t totalFiles() const { return _fileProgress.total(); } + [[nodiscard]] int64_t completedFiles() const { return _fileProgress.completed(); } + [[nodiscard]] int64_t totalSize() const { return _sizeProgress.total(); } + [[nodiscard]] int64_t completedSize() const { return _sizeProgress.completed(); } + [[nodiscard]] int64_t currentFile() const { return completedFiles(); } + [[nodiscard]] Estimates totalProgress() const; private: std::shared_ptr _syncPal; @@ -60,16 +60,15 @@ class ProgressInfo { // DELETE a file and CREATE a directory with exact same name) Progress _sizeProgress; Progress _fileProgress; - int64_t _totalSizeOfCompletedJobs; - double _maxFilesPerSecond; - double _maxBytesPerSecond; - bool _update; + int64_t _totalSizeOfCompletedJobs{0}; + double _maxFilesPerSecond{0.0}; + double _maxBytesPerSecond{0.0}; + bool _update{false}; - int64_t optimisticEta() const; - bool trustEta() const; - Estimates fileProgress(const SyncFileItem &item); + [[nodiscard]] int64_t optimisticEta() const; + [[nodiscard]] bool trustEta() const; void recomputeCompletedSize(); - bool isSizeDependent(const SyncFileItem &item) const; + [[nodiscard]] bool isSizeDependent(const SyncFileItem &item) const; }; } // namespace KDC diff --git a/src/libsyncengine/progress/syncfileitem.h b/src/libsyncengine/progress/syncfileitem.h index c00372c67..a55e18430 100644 --- a/src/libsyncengine/progress/syncfileitem.h +++ b/src/libsyncengine/progress/syncfileitem.h @@ -74,6 +74,8 @@ class SyncFileItem { inline bool isDirectory() const { return _type == NodeType::Directory; } + bool operator==(const SyncFileItem &) const = default; + private: NodeType _type{NodeType::Unknown}; SyncPath _path; // Sync folder relative filesystem path diff --git a/src/libsyncengine/reconciliation/conflict_resolver/conflictresolverworker.cpp b/src/libsyncengine/reconciliation/conflict_resolver/conflictresolverworker.cpp index 28dd37f1d..13dbb6ecd 100644 --- a/src/libsyncengine/reconciliation/conflict_resolver/conflictresolverworker.cpp +++ b/src/libsyncengine/reconciliation/conflict_resolver/conflictresolverworker.cpp @@ -342,12 +342,12 @@ bool ConflictResolverWorker::generateConflictedName(const std::shared_ptr bool isOrphanNode /*= false*/) const { SyncPath absoluteLocalFilePath = _syncPal->localPath() / node->getPath(); newName = PlatformInconsistencyCheckerUtility::instance()->generateNewValidName( - absoluteLocalFilePath, isOrphanNode ? PlatformInconsistencyCheckerUtility::SuffixTypeOrphan - : PlatformInconsistencyCheckerUtility::SuffixTypeConflict); + absoluteLocalFilePath, isOrphanNode ? PlatformInconsistencyCheckerUtility::SuffixType::Orphan + : PlatformInconsistencyCheckerUtility::SuffixType::Conflict); // Check path size size_t pathSize = absoluteLocalFilePath.parent_path().native().size() + 1 + newName.size(); - if (PlatformInconsistencyCheckerUtility::instance()->checkPathLength(pathSize)) { + if (PlatformInconsistencyCheckerUtility::instance()->isPathTooLong(pathSize)) { // Path is now too long, file needs to be moved to root directory return false; } diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp index c2fb028c0..af9985b97 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp @@ -40,7 +40,7 @@ static const char forbiddenFilenameChars[] = {'/', '\0'}; #endif #endif -#define MAX_NAME_LENGTH 255 +static const int maxNameLengh = 255; // Max filename length is uniformized to 255 characters for all platforms and backends namespace KDC { @@ -62,7 +62,7 @@ std::shared_ptr PlatformInconsistencyChecke SyncName PlatformInconsistencyCheckerUtility::generateNewValidName(const SyncPath &name, SuffixType suffixType) { SyncName suffix = generateSuffix(suffixType); - SyncName sub = name.stem().native().substr(0, MAX_NAME_LENGTH - suffix.size() - name.extension().native().size()); + const SyncName sub = name.stem().native().substr(0, maxNameLengh - suffix.size() - name.extension().native().size()); #ifdef _WIN32 SyncName nameStr(name.native()); @@ -90,7 +90,7 @@ ExitCode PlatformInconsistencyCheckerUtility::renameLocalFile(const SyncPath &ab return moveJob.exitCode(); } -bool PlatformInconsistencyCheckerUtility::checkNameForbiddenChars(const SyncPath &name) { +bool PlatformInconsistencyCheckerUtility::nameHasForbiddenChars(const SyncPath &name) { for (auto c: forbiddenFilenameChars) { if (name.native().find(c) != std::string::npos) { return true; @@ -131,12 +131,8 @@ bool PlatformInconsistencyCheckerUtility::fixNameWithBackslash(const SyncName &n } #endif -bool PlatformInconsistencyCheckerUtility::checkNameSize(const SyncName &name) { - if (name.size() > MAX_NAME_LENGTH) { - return true; - } - - return false; +bool PlatformInconsistencyCheckerUtility::isNameTooLong(const SyncName &name) const { + return name.size() > maxNameLengh; } // return false if the file name is ok @@ -171,7 +167,7 @@ bool PlatformInconsistencyCheckerUtility::checkReservedNames(const SyncName &nam return false; } -bool PlatformInconsistencyCheckerUtility::checkPathLength(size_t pathSize) { +bool PlatformInconsistencyCheckerUtility::isPathTooLong(size_t pathSize) { return pathSize > _maxPathLength; } @@ -189,7 +185,7 @@ void PlatformInconsistencyCheckerUtility::setMaxPath() { _maxPathLength = CommonUtility::maxPathLength(); } -SyncName PlatformInconsistencyCheckerUtility::generateSuffix(SuffixType suffixType /*= SuffixTypeRename*/) { +SyncName PlatformInconsistencyCheckerUtility::generateSuffix(SuffixType suffixType) { std::time_t now = std::time(nullptr); std::tm tm = *std::localtime(&now); @@ -198,16 +194,13 @@ SyncName PlatformInconsistencyCheckerUtility::generateSuffix(SuffixType suffixTy ss << std::put_time(&tm, Str("%Y%m%d_%H%M%S")); switch (suffixType) { - case SuffixTypeRename: - suffix = Str("_renamed_"); - break; - case SuffixTypeConflict: + case SuffixType::Conflict: suffix = Str("_conflict_"); break; - case SuffixTypeOrphan: + case SuffixType::Orphan: suffix = Str("_orphan_"); break; - case SuffixTypeBlacklisted: + case SuffixType::Blacklisted: suffix = Str("_blacklisted_"); break; } diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h index 70914d787..89ae3bdc6 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h @@ -28,28 +28,27 @@ namespace KDC { class PlatformInconsistencyCheckerUtility { public: - typedef enum { SuffixTypeRename, SuffixTypeConflict, SuffixTypeOrphan, SuffixTypeBlacklisted } SuffixType; + enum class SuffixType { Conflict, Orphan, Blacklisted }; public: static std::shared_ptr instance(); - SyncName generateNewValidName(const SyncPath &name, SuffixType suffixType); - static ExitCode renameLocalFile(const SyncPath &absoluteLocalPath, SuffixType suffixType, SyncPath *newPathPtr = nullptr); + bool isNameTooLong(const SyncName &name) const; + bool isPathTooLong(size_t pathSize); + bool nameHasForbiddenChars(const SyncPath &name); - bool checkNameForbiddenChars(const SyncPath &name); #ifdef _WIN32 bool fixNameWithBackslash(const SyncName &name, SyncName &newName); #endif - bool checkNameSize(const SyncName &name); bool checkReservedNames(const SyncName &name); - bool checkPathLength(size_t pathSize); - + SyncName generateNewValidName(const SyncPath &name, SuffixType suffixType); + static ExitCode renameLocalFile(const SyncPath &absoluteLocalPath, SuffixType suffixType, SyncPath *newPathPtr = nullptr); private: PlatformInconsistencyCheckerUtility(); SyncName charToHex(unsigned int c); void setMaxPath(); - SyncName generateSuffix(SuffixType suffixType = SuffixTypeRename); + SyncName generateSuffix(SuffixType suffixType); static std::shared_ptr _instance; static size_t _maxPathLength; diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp index df4875abb..b04efac8c 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp @@ -36,8 +36,8 @@ void PlatformInconsistencyCheckerWorker::execute() { _idsToBeRemoved.clear(); - ExitCode exitCode = checkTree(_syncPal->updateTree(ReplicaSide::Remote)->rootNode(), - _syncPal->updateTree(ReplicaSide::Remote)->rootNode()->name()); + checkTree(ReplicaSide::Remote); + checkTree(ReplicaSide::Local); for (const auto &idItem: _idsToBeRemoved) { if (!idItem.remoteId.empty() && !_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(idItem.remoteId)) { @@ -53,20 +53,39 @@ void PlatformInconsistencyCheckerWorker::execute() { _syncPal->updateTree(ReplicaSide::Remote)->setInconsistencyCheckDone(); - setDone(exitCode); + setDone(ExitCode::Ok); LOG_SYNCPAL_DEBUG(_logger, "Worker stopped: name=" << name().c_str()); } -ExitCode PlatformInconsistencyCheckerWorker::checkTree(std::shared_ptr remoteNode, const SyncPath &parentPath) { +ExitCode PlatformInconsistencyCheckerWorker::checkTree(ReplicaSide side) { + std::shared_ptr node = _syncPal->updateTree(side)->rootNode(); + const SyncPath &parentPath = node->name(); + assert(side == ReplicaSide::Remote || + side == ReplicaSide::Local && "Invalid side in PlatformInconsistencyCheckerWorker::checkTree"); + + ExitCode exitCode = ExitCode::Unknown; + if (side == ReplicaSide::Remote) { + exitCode = checkRemoteTree(node, parentPath); + } else if (side == ReplicaSide::Local) { + exitCode = checkLocalTree(node, parentPath); + } + + if (exitCode != ExitCode::Ok) { + LOG_SYNCPAL_WARN(_logger, + "PlatformInconsistencyCheckerWorker::check" << side << "Tree partially failed, ExitCode=" << exitCode); + } + + return exitCode; +} + +ExitCode PlatformInconsistencyCheckerWorker::checkRemoteTree(std::shared_ptr remoteNode, const SyncPath &parentPath) { if (remoteNode->hasChangeEvent(OperationType::Delete)) { return ExitCode::Ok; } - if (remoteNode->hasChangeEvent(OperationType::Create) || remoteNode->hasChangeEvent(OperationType::Move)) { - if (!checkPathAndName(remoteNode)) { - // Item has been blacklisted - return ExitCode::Ok; - } + if (pathChanged(remoteNode) && !checkPathAndName(remoteNode)) { + // Item has been blacklisted + return ExitCode::Ok; } bool checkAgainstSiblings = false; @@ -87,11 +106,11 @@ ExitCode PlatformInconsistencyCheckerWorker::checkTree(std::shared_ptr rem std::shared_ptr currentChildNode = it->second; - if (currentChildNode->hasChangeEvent(OperationType::Create) || currentChildNode->hasChangeEvent(OperationType::Move)) { + if (pathChanged(currentChildNode)) { checkAgainstSiblings = true; } - ExitCode exitCode = checkTree(currentChildNode, parentPath / remoteNode->name()); + ExitCode exitCode = checkRemoteTree(currentChildNode, parentPath / remoteNode->name()); if (exitCode != ExitCode::Ok) { return exitCode; } @@ -104,66 +123,92 @@ ExitCode PlatformInconsistencyCheckerWorker::checkTree(std::shared_ptr rem return ExitCode::Ok; } -void PlatformInconsistencyCheckerWorker::blacklistNode(const std::shared_ptr remoteNode, const SyncPath &relativePath, +ExitCode PlatformInconsistencyCheckerWorker::checkLocalTree(std::shared_ptr localNode, const SyncPath &parentPath) { + if (localNode->hasChangeEvent(OperationType::Delete)) { + return ExitCode::Ok; + } + if (pathChanged(localNode) && PlatformInconsistencyCheckerUtility::instance()->isNameTooLong(localNode->name())) { + blacklistNode(localNode, InconsistencyType::NameLength); + return ExitCode::Ok; + } + + auto childIt = localNode->children().begin(); + for (; childIt != localNode->children().end(); childIt++) { + if (stopAsked()) { + return ExitCode::Ok; + } + + while (pauseAsked() || isPaused()) { + if (!isPaused()) { + setPauseDone(); + } + + Utility::msleep(LOOP_PAUSE_SLEEP_PERIOD); + } + + const ExitCode exitCode = checkLocalTree(childIt->second, parentPath / localNode->name()); + if (exitCode != ExitCode::Ok) { + return exitCode; + } + } + return ExitCode::Ok; +} + +void PlatformInconsistencyCheckerWorker::blacklistNode(const std::shared_ptr node, const InconsistencyType inconsistencyType) { // Local node needs to be excluded before call to blacklistTemporarily because // we need the DB entry to retrieve the corresponding node NodeIdPair nodeIDs; + const std::shared_ptr localNode = node->side() == ReplicaSide::Local ? node : correspondingNodeDirect(node); + const std::shared_ptr remoteNode = node->side() == ReplicaSide::Remote ? node : correspondingNodeDirect(node); - if (const auto localNode = correspondingNodeDirect(remoteNode); localNode) { - // Also exclude local node by adding "conflict" suffix + if (localNode) { const SyncPath absoluteLocalPath = _syncPal->localPath() / localNode->getPath(); - LOGW_SYNCPAL_INFO(_logger, - L"Excluding also local item with " << Utility::formatSyncPath(absoluteLocalPath).c_str() << L"."); - PlatformInconsistencyCheckerUtility::renameLocalFile(absoluteLocalPath, - PlatformInconsistencyCheckerUtility::SuffixTypeConflict); + LOGW_SYNCPAL_INFO(_logger, L"Excluding local item with " << Utility::formatSyncPath(absoluteLocalPath) << L"."); + PlatformInconsistencyCheckerUtility::renameLocalFile( + absoluteLocalPath, node->side() == ReplicaSide::Remote + ? PlatformInconsistencyCheckerUtility::SuffixType::Conflict + : PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); + + if (node->side() == ReplicaSide::Local) { + removeLocalNodeFromDb(localNode); + } + } - if (localNode->id().has_value()) nodeIDs.localId = *localNode->id(); + if (node->side() == ReplicaSide::Remote) { + _syncPal->blacklistTemporarily(remoteNode->id().value(), remoteNode->getPath(), remoteNode->side()); } - _syncPal->blacklistTemporarily(remoteNode->id().value(), relativePath, remoteNode->side()); - Error error(_syncPal->syncDbId(), "", remoteNode->id().value(), remoteNode->type(), relativePath, ConflictType::None, + Error error(_syncPal->syncDbId(), "", node->id().value(), node->type(), node->getPath(), ConflictType::None, inconsistencyType); _syncPal->addError(error); - std::wstring causeStr; - switch (inconsistencyType) { - case InconsistencyType::Case: - causeStr = L"of a name clash"; - break; - case InconsistencyType::ForbiddenChar: - causeStr = L"of a forbidden character"; - break; - case InconsistencyType::ReservedName: - causeStr = L"the name is reserved on this OS"; - break; - case InconsistencyType::NameLength: - causeStr = L"the name is too long"; - break; - default: - break; - } - LOGW_SYNCPAL_INFO(_logger, L"Blacklisting remote item with " << Utility::formatSyncPath(relativePath).c_str() << L" because " - << causeStr.c_str() << L"."); - - nodeIDs.remoteId = *remoteNode->id(); + LOGW_SYNCPAL_INFO(_logger, L"Blacklisting " << node->side() << L" item with " << Utility::formatSyncPath(node->getPath()) + << L" because " << inconsistencyType << L"."); + + auto safeNodeId = [](const std::shared_ptr &unsafeNodePtr) { + return (unsafeNodePtr && unsafeNodePtr->id().has_value()) ? *unsafeNodePtr->id() : NodeId(); + }; + nodeIDs.remoteId = safeNodeId(remoteNode); + nodeIDs.localId = safeNodeId(localNode); + _idsToBeRemoved.emplace_back(nodeIDs); } bool PlatformInconsistencyCheckerWorker::checkPathAndName(std::shared_ptr remoteNode) { const SyncPath relativePath = remoteNode->getPath(); - if (PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(remoteNode->name())) { - blacklistNode(remoteNode, relativePath, InconsistencyType::ForbiddenChar); + if (PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(remoteNode->name())) { + blacklistNode(remoteNode, InconsistencyType::ForbiddenChar); return false; } if (PlatformInconsistencyCheckerUtility::instance()->checkReservedNames(remoteNode->name())) { - blacklistNode(remoteNode, relativePath, InconsistencyType::ReservedName); + blacklistNode(remoteNode, InconsistencyType::ReservedName); return false; } - if (PlatformInconsistencyCheckerUtility::instance()->checkNameSize(remoteNode->name())) { - blacklistNode(remoteNode, relativePath, InconsistencyType::NameLength); + if (PlatformInconsistencyCheckerUtility::instance()->isNameTooLong(remoteNode->name())) { + blacklistNode(remoteNode, InconsistencyType::NameLength); return false; } @@ -201,9 +246,9 @@ void PlatformInconsistencyCheckerWorker::checkNameClashAgainstSiblings(const std // Some software save their files by deleting and re-creating them (Delete-Create), or by deleting the original file // and renaming a temporary file that contains the latest version (Delete-Move) In those cases, we should not check // for name clash, it is ok to have 2 children with the same name - const auto oneNodeIsDeleted = [](const std::shared_ptr &node, const std::shared_ptr &prevNode) -> bool { - return node->hasChangeEvent(OperationType::Move | OperationType::Create) && - prevNode->hasChangeEvent(OperationType::Delete); + const auto oneNodeIsDeleted = [this](const std::shared_ptr &node, + const std::shared_ptr &prevNode) -> bool { + return pathChanged(node) && prevNode->hasChangeEvent(OperationType::Delete); }; if (oneNodeIsDeleted(currentChildNode, prevChildNode) || oneNodeIsDeleted(prevChildNode, currentChildNode)) { @@ -212,11 +257,11 @@ void PlatformInconsistencyCheckerWorker::checkNameClashAgainstSiblings(const std if (currentChildNode->hasChangeEvent() && !isSpecialFolder) { // Blacklist the new one - blacklistNode(currentChildNode, currentChildNode->getPath(), InconsistencyType::Case); + blacklistNode(currentChildNode, InconsistencyType::Case); continue; } else { // Blacklist the previously discovered child - blacklistNode(prevChildNode, prevChildNode->getPath(), InconsistencyType::Case); + blacklistNode(prevChildNode, InconsistencyType::Case); continue; } } @@ -226,4 +271,32 @@ void PlatformInconsistencyCheckerWorker::checkNameClashAgainstSiblings(const std #endif } +bool PlatformInconsistencyCheckerWorker::pathChanged(std::shared_ptr node) const { + return node->hasChangeEvent(OperationType::Create) || node->hasChangeEvent(OperationType::Move); +} + +void PlatformInconsistencyCheckerWorker::removeLocalNodeFromDb(std::shared_ptr localNode) { + if (localNode && localNode->side() == ReplicaSide::Local) { + bool found = false; + DbNodeId dbId = -1; + const SyncPath absoluteLocalPath = _syncPal->localPath() / localNode->getPath(); + if (!_syncPal->syncDb()->dbId(ReplicaSide::Local, *localNode->id(), dbId, found)) { + LOGW_WARN(_logger, L"Failed to retrieve dbId for local node: " << Utility::formatSyncPath(absoluteLocalPath)); + } + if (found && !_syncPal->syncDb()->deleteNode(dbId, found)) { + // Remove node (and childs by cascade) from DB if it exists (else ignore as it is already not in DB) + LOGW_WARN(_logger, L"Failed to delete node from DB: " << Utility::formatSyncPath(absoluteLocalPath)); + } + + if (!_syncPal->vfsFileStatusChanged(absoluteLocalPath, SyncFileStatus::Error)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in SyncPal::vfsFileStatusChanged: " << Utility::formatSyncPath(absoluteLocalPath)); + } + } else { + LOG_WARN(_logger, localNode + ? "Invalid side in PlatformInconsistencyCheckerWorker::removeLocalNodeFromDb" + : "localNode should not be null in PlatformInconsistencyCheckerWorker::removeLocalNodeFromDb"); + assert(false); + } +} + } // namespace KDC diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.h b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.h index 6be741b57..d9080e70a 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.h +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.h @@ -31,13 +31,16 @@ class PlatformInconsistencyCheckerWorker : public OperationProcessor { void execute() override; private: - ExitCode checkTree(std::shared_ptr remoteNode, const SyncPath &parentPath); + ExitCode checkTree(ReplicaSide side); + ExitCode checkRemoteTree(std::shared_ptr remoteNode, const SyncPath &parentPath); + ExitCode checkLocalTree(std::shared_ptr localNode, const SyncPath &parentPath); - void blacklistNode(const std::shared_ptr remoteNode, const SyncPath &relativePath, - const InconsistencyType inconsistencyType); + void blacklistNode(const std::shared_ptr node, const InconsistencyType inconsistencyType); bool checkPathAndName(std::shared_ptr remoteNode); void checkNameClashAgainstSiblings(const std::shared_ptr &remoteParentNode); + bool pathChanged(std::shared_ptr node) const; + void removeLocalNodeFromDb(std::shared_ptr localNode); struct NodeIdPair { NodeId remoteId; NodeId localId; // Optional, only required if the file is already synchronized. diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 8fd343c9e..1d6e0b16b 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1036,8 +1036,7 @@ bool SyncPal::existOnServer(const SyncPath &path) const { bool SyncPal::canShareItem(const SyncPath &path) const { // Path is normalized on server side const SyncPath normalizedPath = Utility::normalizedSyncPath(path); - const NodeId nodeId = _remoteSnapshot->itemId(path); - if (!nodeId.empty()) { + if (const NodeId nodeId = _remoteSnapshot->itemId(normalizedPath); !nodeId.empty()) { return _remoteSnapshot->canShare(nodeId); } return false; @@ -1127,7 +1126,7 @@ ExitCode SyncPal::fixCorruptedFile(const std::unordered_map &l for (const auto &localFileInfo: localFileMap) { SyncPath destPath; if (ExitCode exitCode = PlatformInconsistencyCheckerUtility::renameLocalFile( - localFileInfo.second, PlatformInconsistencyCheckerUtility::SuffixTypeConflict, &destPath); + localFileInfo.second, PlatformInconsistencyCheckerUtility::SuffixType::Conflict, &destPath); exitCode != ExitCode::Ok) { LOGW_SYNCPAL_WARN(_logger, L"Fail to rename " << Path2WStr(localFileInfo.second).c_str() << L" into " << Path2WStr(destPath).c_str()); diff --git a/src/libsyncengine/update_detection/blacklist_changes_propagator/blacklistpropagator.cpp b/src/libsyncengine/update_detection/blacklist_changes_propagator/blacklistpropagator.cpp index 7f5009d87..5b6fb2a7a 100644 --- a/src/libsyncengine/update_detection/blacklist_changes_propagator/blacklistpropagator.cpp +++ b/src/libsyncengine/update_detection/blacklist_changes_propagator/blacklistpropagator.cpp @@ -246,7 +246,7 @@ ExitCode BlacklistPropagator::removeItem(const NodeId &localNodeId, const NodeId SyncPath destPath; PlatformInconsistencyCheckerUtility::renameLocalFile( - absolutePath, PlatformInconsistencyCheckerUtility::SuffixTypeBlacklisted, &destPath); + absolutePath, PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted, &destPath); Error err(_syncPal->syncDbId(), "", "", NodeType::Directory, absolutePath, ConflictType::None, InconsistencyType::None, CancelType::MoveToBinFailed, destPath); diff --git a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp index d5a44c1ae..52ce8171d 100644 --- a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp @@ -182,7 +182,7 @@ ExitCode ComputeFSOperationWorker::inferChangeFromDbNode(const ReplicaSide side, // Do not propagate delete if the path is too long. const size_t pathSize = localPath.native().size(); - if (PlatformInconsistencyCheckerUtility::instance()->checkPathLength(pathSize)) { + if (PlatformInconsistencyCheckerUtility::instance()->isPathTooLong(pathSize)) { LOGW_SYNCPAL_WARN(_logger, L"Path length too long (" << pathSize << L" characters) for item with " << Utility::formatSyncPath(localPath).c_str() << L". Item is ignored."); @@ -778,7 +778,7 @@ bool ComputeFSOperationWorker::isTooBig(const std::shared_ptr re bool ComputeFSOperationWorker::isPathTooLong(const SyncPath &path, const NodeId &nodeId, NodeType type) const { const SyncPath absolutePath = _syncPal->localPath() / path; const size_t pathSize = absolutePath.native().size(); - if (PlatformInconsistencyCheckerUtility::instance()->checkPathLength(pathSize)) { + if (PlatformInconsistencyCheckerUtility::instance()->isPathTooLong(pathSize)) { LOGW_SYNCPAL_WARN(_logger, L"Path length too long (" << pathSize << L" characters) for item with " << Utility::formatSyncPath(absolutePath).c_str() << L". Item is ignored."); diff --git a/src/libsyncengine/update_detection/file_system_observer/snapshot/snapshot.cpp b/src/libsyncengine/update_detection/file_system_observer/snapshot/snapshot.cpp index 4c6d08264..23509e9e2 100644 --- a/src/libsyncengine/update_detection/file_system_observer/snapshot/snapshot.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/snapshot/snapshot.cpp @@ -497,10 +497,11 @@ bool Snapshot::checkIntegrityRecursively(const NodeId &parentId) { auto result = names.insert(_items[*childId].name()); if (!result.second) { - LOG_ERROR(Log::instance()->getLogger(), - "Snapshot integrity check failed, the folder named: \"" - << SyncName2Str(parentItem.name()).c_str() << "\"(" << parentItem.id().c_str() << ") contains: \"" - << SyncName2Str(_items[*childId].name()).c_str() << "\" twice with two differents NodeId"); + LOGW_WARN(Log::instance()->getLogger(), L"Snapshot integrity check failed, the folder named: \"" + << SyncName2WStr(parentItem.name()).c_str() << L"\"(" + << Utility::s2ws(parentItem.id()).c_str() << L") contains: \"" + << SyncName2WStr(_items[*childId].name()).c_str() + << L"\" twice with two differents NodeId"); return false; } } diff --git a/src/libsyncengine/update_detection/update_detector/node.cpp b/src/libsyncengine/update_detection/update_detector/node.cpp index 80c967016..18cc1d69f 100644 --- a/src/libsyncengine/update_detection/update_detector/node.cpp +++ b/src/libsyncengine/update_detection/update_detector/node.cpp @@ -35,6 +35,12 @@ Node::Node(const std::optional &idb, const ReplicaSide &side, const Sy setParentNode(parentNode); } +Node::Node(const ReplicaSide &side, const SyncName &name, NodeType type, OperationType changeEvents, + const std::optional &id, std::optional createdAt, std::optional lastmodified, int64_t size, + std::shared_ptr parentNode, std::optional moveOrigin, std::optional moveOriginParentDbId) : + Node(std::nullopt, side, name, type, changeEvents, id, createdAt, lastmodified, size, parentNode, moveOrigin, + moveOriginParentDbId) {} + Node::Node(const ReplicaSide &side, const SyncName &name, NodeType type, std::shared_ptr parentNode) : _side(side), _name(name), _type(type), _isTmp(true) { _id = "tmp_" + CommonUtility::generateRandomStringAlphaNum(); diff --git a/src/libsyncengine/update_detection/update_detector/node.h b/src/libsyncengine/update_detection/update_detector/node.h index eb0786f06..d88687914 100644 --- a/src/libsyncengine/update_detection/update_detector/node.h +++ b/src/libsyncengine/update_detection/update_detector/node.h @@ -37,6 +37,11 @@ class Node { std::optional lastmodified, int64_t size, std::shared_ptr parentNode, std::optional moveOrigin = std::nullopt, std::optional moveOriginParentDbId = std::nullopt); + Node(const ReplicaSide &side, const SyncName &name, NodeType type, + OperationType changeEvents, const std::optional &id, std::optional createdAt, + std::optional lastmodified, int64_t size, std::shared_ptr parentNode, + std::optional moveOrigin = std::nullopt, std::optional moveOriginParentDbId = std::nullopt); + /** * @brief Node * diff --git a/src/server/vfs/mac/vfs_mac.cpp b/src/server/vfs/mac/vfs_mac.cpp index 1161dfbd9..40b705f92 100644 --- a/src/server/vfs/mac/vfs_mac.cpp +++ b/src/server/vfs/mac/vfs_mac.cpp @@ -53,17 +53,21 @@ VfsMac::VfsMac(KDC::VfsSetupParams &vfsSetupParams, QObject *parent) : throw std::runtime_error("Error getting LiteSyncExtConnector instance!"); } - // Start worker threads - for (int i = 0; i < NB_WORKERS; i++) { - for (int j = 0; j < s_nb_threads[i]; j++) { - QThread *workerThread = new QThread(); - _workerInfo[i]._threadList.append(workerThread); - Worker *worker = new Worker(this, i, j, logger()); - worker->moveToThread(workerThread); - connect(workerThread, &QThread::started, worker, &Worker::start); - connect(workerThread, &QThread::finished, worker, &QObject::deleteLater); - connect(workerThread, &QThread::finished, workerThread, &QObject::deleteLater); - workerThread->start(); + // Start hydration/dehydration workers + // !!! Disabled for testing because no QEventLoop !!! + if (qApp) { + // Start worker threads + for (int i = 0; i < NB_WORKERS; i++) { + for (int j = 0; j < s_nb_threads[i]; j++) { + QThread *workerThread = new QThread(); + _workerInfo[i]._threadList.append(workerThread); + Worker *worker = new Worker(this, i, j, logger()); + worker->moveToThread(workerThread); + connect(workerThread, &QThread::started, worker, &Worker::start); + connect(workerThread, &QThread::finished, worker, &QObject::deleteLater); + connect(workerThread, &QThread::finished, workerThread, &QObject::deleteLater); + workerThread->start(); + } } } } diff --git a/test/libsyncengine/db/testsyncdb.cpp b/test/libsyncengine/db/testsyncdb.cpp index ffa656cc5..db221883f 100644 --- a/test/libsyncengine/db/testsyncdb.cpp +++ b/test/libsyncengine/db/testsyncdb.cpp @@ -277,6 +277,15 @@ void TestSyncDb::testUpgradeTo3_6_5() { ParmsDb::reset(); } +void TestSyncDb::testUpgradeTo3_6_7() { + createParmsDb(_testObj->dbPath(), SyncPath("local_sync_dir_does_not_exist")); + + CPPUNIT_ASSERT(_testObj->upgrade("3.6.4", "3.6.7")); + + ParmsDb::instance()->close(); + ParmsDb::reset(); +} + void TestSyncDb::testInit3_6_4() { SyncDbMock testDb(_testObj->dbPath().string(), "3.6.4"); const LocalTemporaryDirectory localTmpDir("testUpgradeTo3_6_5"); diff --git a/test/libsyncengine/db/testsyncdb.h b/test/libsyncengine/db/testsyncdb.h index 83ac05f74..9bc7dc7eb 100644 --- a/test/libsyncengine/db/testsyncdb.h +++ b/test/libsyncengine/db/testsyncdb.h @@ -40,6 +40,7 @@ class TestSyncDb : public CppUnit::TestFixture { CPPUNIT_TEST(testSyncNodes); CPPUNIT_TEST(testCorrespondingNodeId); CPPUNIT_TEST(testUpdateLocalName); + CPPUNIT_TEST(testUpgradeTo3_6_7); CPPUNIT_TEST(testUpgradeTo3_6_5CheckNodeMap); CPPUNIT_TEST(testUpgradeTo3_6_5); CPPUNIT_TEST(testInit3_6_4); @@ -54,6 +55,7 @@ class TestSyncDb : public CppUnit::TestFixture { void testSyncNodes(); void testCorrespondingNodeId(); void testUpdateLocalName(); + void testUpgradeTo3_6_7(); void testUpgradeTo3_6_5(); void testUpgradeTo3_6_5CheckNodeMap(); void testInit3_6_4(); diff --git a/test/libsyncengine/propagation/operation_sorter/testoperationsorterworker.cpp b/test/libsyncengine/propagation/operation_sorter/testoperationsorterworker.cpp index 74b3fe99a..19aca9f4d 100644 --- a/test/libsyncengine/propagation/operation_sorter/testoperationsorterworker.cpp +++ b/test/libsyncengine/propagation/operation_sorter/testoperationsorterworker.cpp @@ -53,15 +53,15 @@ void TestOperationSorterWorker::tearDown() { } void TestOperationSorterWorker::testMoveFirstAfterSecond() { - const auto node1 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("1"), NodeType::Directory, + const auto node1 = std::make_shared(ReplicaSide::Local, Str("1"), NodeType::Directory, OperationType::None, std::nullopt, 0, 0, 12345, nullptr); - const auto node2 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("2"), NodeType::Directory, + const auto node2 = std::make_shared(ReplicaSide::Local, Str("2"), NodeType::Directory, OperationType::None, std::nullopt, 0, 0, 12345, nullptr); - const auto node3 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("3"), NodeType::Directory, + const auto node3 = std::make_shared(ReplicaSide::Local, Str("3"), NodeType::Directory, OperationType::None, std::nullopt, 0, 0, 12345, nullptr); - const auto node4 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("4"), NodeType::Directory, + const auto node4 = std::make_shared(ReplicaSide::Local, Str("4"), NodeType::Directory, OperationType::None, std::nullopt, 0, 0, 12345, nullptr); - const auto node5 = std::make_shared(std::nullopt, ReplicaSide::Local, Str("5"), NodeType::Directory, + const auto node5 = std::make_shared(ReplicaSide::Local, Str("5"), NodeType::Directory, OperationType::None, std::nullopt, 0, 0, 12345, nullptr); const auto op1 = std::make_shared(); const auto op2 = std::make_shared(); diff --git a/test/libsyncengine/reconciliation/conflict_finder/testconflictfinderworker.cpp b/test/libsyncengine/reconciliation/conflict_finder/testconflictfinderworker.cpp index 1e62745b2..eb8192db1 100644 --- a/test/libsyncengine/reconciliation/conflict_finder/testconflictfinderworker.cpp +++ b/test/libsyncengine/reconciliation/conflict_finder/testconflictfinderworker.cpp @@ -521,7 +521,8 @@ void TestConflictFinderWorker::testCase55b() { _syncPal->updateTree(ReplicaSide::Remote)->insertNode(rNodeA); // Conflict Situation - nodeA->setChangeEvents(OperationType::Move | OperationType::Edit); + nodeA->setChangeEvents( + OperationType::Edit); nodeA->setMoveOrigin("A"); nodeA->setName(Str("B")); rNodeA->setChangeEvents(OperationType::Edit); diff --git a/test/libsyncengine/reconciliation/conflict_resolver/testconflictresolverworker.cpp b/test/libsyncengine/reconciliation/conflict_resolver/testconflictresolverworker.cpp index 0237a4970..e7183a0cd 100644 --- a/test/libsyncengine/reconciliation/conflict_resolver/testconflictresolverworker.cpp +++ b/test/libsyncengine/reconciliation/conflict_resolver/testconflictresolverworker.cpp @@ -138,7 +138,7 @@ void TestConflictResolverWorker::testCreateCreate() { std::shared_ptr rNodeAA = _syncPal->updateTree(ReplicaSide::Remote)->getNodeById("rAA"); std::shared_ptr rNodeAAB = - std::make_shared(std::nullopt, _syncPal->updateTree(ReplicaSide::Remote)->side(), Str("AAB.txt"), + std::make_shared(_syncPal->updateTree(ReplicaSide::Remote)->side(), Str("AAB.txt"), NodeType::File, OperationType::Create, "rAAB", testhelpers::defaultTime, testhelpers::defaultTime, testhelpers::defaultFileSize, rNodeAA); CPPUNIT_ASSERT(rNodeAA->insertChildren(rNodeAAB)); @@ -610,7 +610,7 @@ void TestConflictResolverWorker::testMoveMoveSource() { void TestConflictResolverWorker::testMoveMoveSourceWithOrphanNodes() { // Initial state : Node AAA is orphan. const SyncName orphanName = PlatformInconsistencyCheckerUtility::instance()->generateNewValidName( - "AAA", PlatformInconsistencyCheckerUtility::SuffixTypeOrphan); + "AAA", PlatformInconsistencyCheckerUtility::SuffixType::Orphan); std::shared_ptr lNodeAAA = _syncPal->updateTree(ReplicaSide::Local)->getNodeById("lAAA"); lNodeAAA->setName(orphanName); diff --git a/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.cpp b/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.cpp index 238e462e6..cbc98beea 100644 --- a/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.cpp +++ b/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.cpp @@ -81,47 +81,47 @@ void TestPlatformInconsistencyCheckerWorker::tearDown() { void TestPlatformInconsistencyCheckerWorker::testFixNameSize() { SyncName shortName = Str("1234567890"); - CPPUNIT_ASSERT(!PlatformInconsistencyCheckerUtility::instance()->checkNameSize(shortName)); + CPPUNIT_ASSERT(!PlatformInconsistencyCheckerUtility::instance()->isNameTooLong(shortName)); SyncName longName = Str( "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456" "78901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012" "3456789012345678901234567890"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameSize(longName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->isNameTooLong(longName)); } void TestPlatformInconsistencyCheckerWorker::testCheckNameForbiddenChars() { SyncName allowedName = Str("test-test"); - CPPUNIT_ASSERT(!PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(allowedName)); + CPPUNIT_ASSERT(!PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(allowedName)); SyncName forbiddenName = Str("test/test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); #if defined(WIN32) forbiddenName = Str("test\\test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test:test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test*test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test?test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test\"test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("testcheckNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test>test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test|test"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test\ntest"); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); forbiddenName = Str("test "); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); #elif defined(__unix__) && !defined(__APPLE__) forbiddenName = std::string("test"); forbiddenName.append(1, '\0'); - CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->checkNameForbiddenChars(forbiddenName)); + CPPUNIT_ASSERT(PlatformInconsistencyCheckerUtility::instance()->nameHasForbiddenChars(forbiddenName)); #endif } @@ -145,14 +145,14 @@ void TestPlatformInconsistencyCheckerWorker::testCheckReservedNames() { } void TestPlatformInconsistencyCheckerWorker::testNameClash() { - const auto parentNode = std::make_shared(std::nullopt, _syncPal->updateTree(ReplicaSide::Remote)->side(), - Str("parentNode"), NodeType::Directory, OperationType::Create, "parentID", 0, - 0, 12345, _syncPal->updateTree(ReplicaSide::Remote)->rootNode()); + const auto parentNode = std::make_shared(_syncPal->updateTree(ReplicaSide::Remote)->side(), Str("parentNode"), + NodeType::Directory, OperationType::Create, "parentID", 0, 0, 12345, + _syncPal->updateTree(ReplicaSide::Remote)->rootNode()); - const auto nodeLower = std::make_shared(std::nullopt, _syncPal->updateTree(ReplicaSide::Remote)->side(), Str("a"), - NodeType::File, OperationType::Create, "a", 0, 0, 12345, parentNode); - const auto nodeUpper = std::make_shared(std::nullopt, _syncPal->updateTree(ReplicaSide::Remote)->side(), Str("A"), - NodeType::File, OperationType::Create, "A", 0, 0, 12345, parentNode); + const auto nodeLower = std::make_shared(_syncPal->updateTree(ReplicaSide::Remote)->side(), Str("a"), NodeType::File, + OperationType::Create, "a", 0, 0, 12345, parentNode); + const auto nodeUpper = std::make_shared(_syncPal->updateTree(ReplicaSide::Remote)->side(), Str("A"), NodeType::File, + OperationType::Create, "A", 0, 0, 12345, parentNode); CPPUNIT_ASSERT(parentNode->insertChildren(nodeLower)); CPPUNIT_ASSERT(parentNode->insertChildren(nodeUpper)); @@ -242,14 +242,14 @@ void TestPlatformInconsistencyCheckerWorker::testNameClashAfterRename() { } void TestPlatformInconsistencyCheckerWorker::testExecute() { - const auto parentNode = std::make_shared(std::nullopt, _syncPal->updateTree(ReplicaSide::Remote)->side(), - Str("parentNode"), NodeType::Directory, OperationType::Create, "parentID", 0, - 0, 12345, _syncPal->updateTree(ReplicaSide::Remote)->rootNode()); + const auto parentNode = std::make_shared(_syncPal->updateTree(ReplicaSide::Remote)->side(), Str("parentNode"), + NodeType::Directory, OperationType::Create, "parentID", 0, 0, 12345, + _syncPal->updateTree(ReplicaSide::Remote)->rootNode()); - const auto nodeLower = std::make_shared(std::nullopt, ReplicaSide::Remote, Str("a"), NodeType::File, - OperationType::Create, "a", 0, 0, 12345, parentNode); - const auto nodeUpper = std::make_shared(std::nullopt, ReplicaSide::Remote, Str("A"), NodeType::File, - OperationType::Create, "A", 0, 0, 12345, parentNode); + const auto nodeLower = std::make_shared(ReplicaSide::Remote, Str("a"), NodeType::File, OperationType::Create, "a", 0, 0, + 12345, parentNode); + const auto nodeUpper = std::make_shared(ReplicaSide::Remote, Str("A"), NodeType::File, OperationType::Create, "A", 0, 0, + 12345, parentNode); CPPUNIT_ASSERT(_syncPal->updateTree(ReplicaSide::Remote)->rootNode()->insertChildren(parentNode)); CPPUNIT_ASSERT(parentNode->insertChildren(nodeLower)); @@ -265,7 +265,14 @@ void TestPlatformInconsistencyCheckerWorker::testExecute() { !_syncPal->updateTree(ReplicaSide::Remote)->exists(*nodeLower->id())) || (!_syncPal->updateTree(ReplicaSide::Remote)->exists(*nodeUpper->id()) && _syncPal->updateTree(ReplicaSide::Remote)->exists(*nodeLower->id())); - + LOG_DEBUG(Log::instance()->getLogger(), + "TestPlatformInconsistencyCheckerWorker::testExecute()" + "Upper Node exists: " + << _syncPal->updateTree(ReplicaSide::Remote)->exists(*nodeUpper->id())); + LOG_DEBUG(Log::instance()->getLogger(), + "TestPlatformInconsistencyCheckerWorker::testExecute()" + "Lower Node exists: " + << _syncPal->updateTree(ReplicaSide::Remote)->exists(*nodeLower->id())); CPPUNIT_ASSERT(_syncPal->updateTree(ReplicaSide::Remote)->exists(*parentNode->id())); #if defined(WIN32) || defined(__APPLE__) @@ -276,4 +283,83 @@ void TestPlatformInconsistencyCheckerWorker::testExecute() { #endif } +void TestPlatformInconsistencyCheckerWorker::testNameSizeLocalTree() { + initUpdateTree(ReplicaSide::Local); + + _syncPal->_platformInconsistencyCheckerWorker->execute(); + + CPPUNIT_ASSERT(_syncPal->updateTree(ReplicaSide::Local)->exists("testNode")); + CPPUNIT_ASSERT(_syncPal->updateTree(ReplicaSide::Local)->exists("aNode")); + CPPUNIT_ASSERT(_syncPal->updateTree(ReplicaSide::Local)->exists("ANode")); + CPPUNIT_ASSERT(!_syncPal->updateTree(ReplicaSide::Local)->exists("longNameANode")); + CPPUNIT_ASSERT(!_syncPal->updateTree(ReplicaSide::Local)->exists("testNode2")); + CPPUNIT_ASSERT(!_syncPal->updateTree(ReplicaSide::Local)->exists("bNode")); + CPPUNIT_ASSERT(!_syncPal->updateTree(ReplicaSide::Local)->exists("BNode")); +} + +void TestPlatformInconsistencyCheckerWorker::initUpdateTree(ReplicaSide side) { + /* Initial tree structure: + * | + * +-- /test (dir) CREATE + * | | + * | +-- a.txt (file) CREATE + * | +-- A.txt (file) CREATE + * | +-- aaaaaaaaaaaaaaaaaaaa...aaaaaaaaaaaaaaaaaa.txt (file) [maxNameLengh +1] CREATE + * | + * +-- /testDiraaaaaaaaaaaaaaa...aaaaaaaaaaaaaaaaa (dir) [maxNameLengh +1] MOVE + * | | + * | +-- b.txt (file) NONE + * | +-- B.txt (file) NONE + */ + + const auto testNode = + std::make_shared(_syncPal->updateTree(side)->side(), Str2SyncName("test"), NodeType::Directory, + OperationType::Create, "testNode", 0, 0, 12345, _syncPal->updateTree(side)->rootNode()); + + const auto aNode = std::make_shared(_syncPal->updateTree(side)->side(), Str2SyncName("a.txt"), NodeType::File, + OperationType::Create, "aNode", 0, 0, 12345, testNode); + + const auto ANode = std::make_shared(_syncPal->updateTree(side)->side(), Str2SyncName("A.txt"), NodeType::File, + OperationType::Create, "ANode", 0, 0, 12345, testNode); + + const auto longNameANode = std::make_shared( + std::nullopt, _syncPal->updateTree(side)->side(), + Str2SyncName("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.txt"), + NodeType::File, OperationType::Create, "longNameANode", 0, 0, 12345, testNode); + + const auto testNode2 = std::make_shared( + std::nullopt, _syncPal->updateTree(side)->side(), + Str2SyncName("testDiraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + NodeType::Directory, OperationType::Move, "testNode2", 0, 0, 12345, _syncPal->updateTree(side)->rootNode()); + + const auto bNode = std::make_shared(_syncPal->updateTree(side)->side(), Str2SyncName("b.txt"), NodeType::File, + OperationType::None, "bNode", 0, 0, 12345, testNode2); + + const auto BNode = std::make_shared(_syncPal->updateTree(side)->side(), Str2SyncName("B.txt"), NodeType::File, + OperationType::None, "BNode", 0, 0, 12345, testNode2); + + + CPPUNIT_ASSERT(_syncPal->updateTree(side)->rootNode()->insertChildren(testNode)); + CPPUNIT_ASSERT(testNode->insertChildren(aNode)); + CPPUNIT_ASSERT(testNode->insertChildren(ANode)); + CPPUNIT_ASSERT(testNode->insertChildren(longNameANode)); + CPPUNIT_ASSERT(_syncPal->updateTree(side)->rootNode()->insertChildren(testNode2)); + CPPUNIT_ASSERT(testNode2->insertChildren(bNode)); + CPPUNIT_ASSERT(testNode2->insertChildren(BNode)); + + _syncPal->updateTree(side)->insertNode(testNode); + _syncPal->updateTree(side)->insertNode(aNode); + _syncPal->updateTree(side)->insertNode(ANode); + _syncPal->updateTree(side)->insertNode(longNameANode); + _syncPal->updateTree(side)->insertNode(testNode2); + _syncPal->updateTree(side)->insertNode(bNode); + _syncPal->updateTree(side)->insertNode(BNode); +} + } // namespace KDC diff --git a/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.h b/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.h index dc2bda874..8402ddc87 100644 --- a/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.h +++ b/test/libsyncengine/reconciliation/platform_inconsistency_checker/testplatforminconsistencycheckerworker.h @@ -34,6 +34,7 @@ class TestPlatformInconsistencyCheckerWorker : public CppUnit::TestFixture { CPPUNIT_TEST(testNameClash); CPPUNIT_TEST(testNameClashAfterRename); CPPUNIT_TEST(testExecute); + CPPUNIT_TEST(testNameSizeLocalTree); CPPUNIT_TEST_SUITE_END(); public: @@ -47,10 +48,12 @@ class TestPlatformInconsistencyCheckerWorker : public CppUnit::TestFixture { void testNameClash(); void testNameClashAfterRename(); void testExecute(); + void testNameSizeLocalTree(); private: std::shared_ptr _syncPal{nullptr}; LocalTemporaryDirectory _tempDir{"testNameClashAfterRename"}; + void initUpdateTree(ReplicaSide side); }; } // namespace KDC diff --git a/test/libsyncengine/syncpal/testsyncpal.cpp b/test/libsyncengine/syncpal/testsyncpal.cpp index a0d3ad3a4..2bd94831b 100644 --- a/test/libsyncengine/syncpal/testsyncpal.cpp +++ b/test/libsyncengine/syncpal/testsyncpal.cpp @@ -152,6 +152,53 @@ void TestSyncPal::testCopySnapshots() { CPPUNIT_ASSERT(_syncPal->snapshot(ReplicaSide::Local, true)->isValid() != _syncPal->snapshot(ReplicaSide::Local, false)->isValid()); } +void TestSyncPal::testSyncFileItem() { + _syncPal->_progressInfo = std::make_shared(_syncPal); + + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->completedSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->completedFiles()); + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->totalSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->totalFiles()); + + const SyncPath nfcPath = SyncPath(Str("/") + testhelpers::makeNfcSyncName() + Str("/test.txt")).native(); + const SyncPath nfdPath = SyncPath(Str("/") + testhelpers::makeNfdSyncName() + Str("/test.txt")).native(); + SyncFileItem initItem; + initItem.setType(NodeType::File); + initItem.setPath(nfcPath); + initItem.setLocalNodeId("l123"); + initItem.setDirection(SyncDirection::Up); + initItem.setInstruction(SyncFileInstruction::Put); + initItem.setSize(testhelpers::defaultFileSize); + initItem.setModTime(testhelpers::defaultTime); + _syncPal->initProgress(initItem); + + SyncFileItem testItem; + CPPUNIT_ASSERT(_syncPal->getSyncFileItem(nfcPath, testItem)); + CPPUNIT_ASSERT(testItem == initItem); + CPPUNIT_ASSERT(_syncPal->getSyncFileItem(nfdPath, testItem)); + CPPUNIT_ASSERT(testItem == initItem); + + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->completedSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->completedFiles()); + CPPUNIT_ASSERT_EQUAL(testhelpers::defaultFileSize, _syncPal->_progressInfo->totalSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), _syncPal->_progressInfo->totalFiles()); + + constexpr int64_t progress = 15; + _syncPal->setProgress(nfdPath, progress); + CPPUNIT_ASSERT(_syncPal->getSyncFileItem(nfdPath, testItem)); + + CPPUNIT_ASSERT_EQUAL(progress, _syncPal->_progressInfo->completedSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(0), _syncPal->_progressInfo->completedFiles()); + CPPUNIT_ASSERT_EQUAL(testhelpers::defaultFileSize, _syncPal->_progressInfo->totalSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), _syncPal->_progressInfo->totalFiles()); + + _syncPal->setProgressComplete(nfdPath, SyncFileStatus::Success); + + CPPUNIT_ASSERT_EQUAL(testhelpers::defaultFileSize, _syncPal->_progressInfo->completedSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), _syncPal->_progressInfo->completedFiles()); + CPPUNIT_ASSERT_EQUAL(testhelpers::defaultFileSize, _syncPal->_progressInfo->totalSize()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), _syncPal->_progressInfo->totalFiles()); +} void TestSyncPal::testAll() { // Start sync diff --git a/test/libsyncengine/syncpal/testsyncpal.h b/test/libsyncengine/syncpal/testsyncpal.h index 33a6cb05a..3cf550aa2 100644 --- a/test/libsyncengine/syncpal/testsyncpal.h +++ b/test/libsyncengine/syncpal/testsyncpal.h @@ -30,6 +30,7 @@ class TestSyncPal : public CppUnit::TestFixture { CPPUNIT_TEST(testSnapshot); CPPUNIT_TEST(testCopySnapshots); CPPUNIT_TEST(testOperationSet); + CPPUNIT_TEST(testSyncFileItem); CPPUNIT_TEST_SUITE_END(); public: @@ -50,6 +51,8 @@ class TestSyncPal : public CppUnit::TestFixture { void testOperationSet(); void testCopySnapshots(); + void testSyncFileItem(); + void testAll(); void testConflictQueue(); bool exec_case_6_4(); diff --git a/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp b/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp index 80a11b9bf..5a4d57a58 100644 --- a/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp +++ b/test/libsyncengine/update_detection/update_detector/testupdatetree.cpp @@ -69,12 +69,12 @@ void TestUpdateTree::testConstructors() { void TestUpdateTree::testInsertionOfFileNamesWithDifferentEncodings() { CPPUNIT_ASSERT(_myTree->_nodes.empty()); - auto node1 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::None, + auto node1 = std::make_shared(_myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::None, "l1", 0, 0, 12345, _myTree->rootNode()); - const auto node2 = std::make_shared(std::nullopt, _myTree->side(), testhelpers::makeNfcSyncName(), NodeType::Directory, + const auto node2 = std::make_shared(_myTree->side(), testhelpers::makeNfcSyncName(), NodeType::Directory, OperationType::None, "l2", 0, 0, 12345, node1); - const auto node3 = std::make_shared(std::nullopt, _myTree->side(), testhelpers::makeNfdSyncName(), NodeType::Directory, + const auto node3 = std::make_shared(_myTree->side(), testhelpers::makeNfdSyncName(), NodeType::Directory, OperationType::None, "l3", 0, 0, 12345, node1); CPPUNIT_ASSERT_EQUAL(SyncPath("Dir 1") / testhelpers::makeNfcSyncName(), node2->getPath()); @@ -83,31 +83,31 @@ void TestUpdateTree::testInsertionOfFileNamesWithDifferentEncodings() { void TestUpdateTree::testAll() { CPPUNIT_ASSERT(_myTree->_nodes.empty()); - auto node1 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::None, + auto node1 = std::make_shared(_myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::None, "l1", 0, 0, 12345, _myTree->rootNode()); _myTree->insertNode(node1); CPPUNIT_ASSERT(_myTree->_nodes.size() == 1); - auto node2 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 2"), NodeType::Directory, OperationType::None, + auto node2 = std::make_shared(_myTree->side(), Str("Dir 2"), NodeType::Directory, OperationType::None, "l2", 0, 0, 12345, _myTree->rootNode()); - auto node3 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 3"), NodeType::Directory, OperationType::None, + auto node3 = std::make_shared(_myTree->side(), Str("Dir 3"), NodeType::Directory, OperationType::None, "l3", 0, 0, 12345, _myTree->rootNode()); - auto node4 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 4"), NodeType::Directory, OperationType::None, + auto node4 = std::make_shared(_myTree->side(), Str("Dir 4"), NodeType::Directory, OperationType::None, "l4", 0, 0, 12345, _myTree->rootNode()); - auto node11 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1.1"), NodeType::Directory, OperationType::None, + auto node11 = std::make_shared(_myTree->side(), Str("Dir 1.1"), NodeType::Directory, OperationType::None, "l11", 0, 0, 12345, node1); - auto node111 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1.1.1"), NodeType::Directory, + auto node111 = std::make_shared(_myTree->side(), Str("Dir 1.1.1"), NodeType::Directory, OperationType::None, "l111", 0, 0, 12345, node11); - auto node1111 = std::make_shared(std::nullopt, _myTree->side(), Str("File 1.1.1.1"), NodeType::File, + auto node1111 = std::make_shared(_myTree->side(), Str("File 1.1.1.1"), NodeType::File, OperationType::None, "l1111", 0, 0, 12345, node111); - auto node31 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 3.1"), NodeType::Directory, OperationType::None, + auto node31 = std::make_shared(_myTree->side(), Str("Dir 3.1"), NodeType::Directory, OperationType::None, "l31", 0, 0, 12345, node3); - auto node41 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 4.1"), NodeType::Directory, OperationType::None, + auto node41 = std::make_shared(_myTree->side(), Str("Dir 4.1"), NodeType::Directory, OperationType::None, "l41", 0, 0, 12345, node4); - auto node411 = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 4.1.1"), NodeType::Directory, + auto node411 = std::make_shared(_myTree->side(), Str("Dir 4.1.1"), NodeType::Directory, OperationType::None, "l411", 0, 0, 12345, node41); - auto node4111 = std::make_shared(std::nullopt, _myTree->side(), Str("File 4.1.1.1"), NodeType::File, + auto node4111 = std::make_shared(_myTree->side(), Str("File 4.1.1.1"), NodeType::File, OperationType::None, "l4111", 0, 0, 12345, node411); CPPUNIT_ASSERT(_myTree->rootNode()->insertChildren(node1)); @@ -142,22 +142,22 @@ void TestUpdateTree::testAll() { } void TestUpdateTree::testChangeEvents() { - std::shared_ptr node = std::make_shared(std::nullopt, _myTree->side(), Str("Dir 0"), NodeType::Directory, + std::shared_ptr node = std::make_shared(_myTree->side(), Str("Dir 0"), NodeType::Directory, OperationType::None, "0", 0, 0, 12345, _myTree->rootNode()); const std::shared_ptr nodeCreate = - std::make_shared(std::nullopt, _myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::Create, "l1", + std::make_shared(_myTree->side(), Str("Dir 1"), NodeType::Directory, OperationType::Create, "l1", 0, 0, 12345, _myTree->rootNode()); const std::shared_ptr nodeEdit = - std::make_shared(std::nullopt, _myTree->side(), Str("Dir 2"), NodeType::Directory, OperationType::Edit, "l2", 0, + std::make_shared(_myTree->side(), Str("Dir 2"), NodeType::Directory, OperationType::Edit, "l2", 0, 0, 12345, _myTree->rootNode()); const std::shared_ptr nodeMove = - std::make_shared(std::nullopt, _myTree->side(), Str("Dir 3"), NodeType::Directory, OperationType::Move, "l3", 0, + std::make_shared(_myTree->side(), Str("Dir 3"), NodeType::Directory, OperationType::Move, "l3", 0, 0, 12345, _myTree->rootNode()); const std::shared_ptr nodeDelete = - std::make_shared(std::nullopt, _myTree->side(), Str("Dir 4"), NodeType::Directory, OperationType::Delete, "l4", + std::make_shared(_myTree->side(), Str("Dir 4"), NodeType::Directory, OperationType::Delete, "l4", 0, 0, 12345, _myTree->rootNode()); const std::shared_ptr nodeMoveEdit = - std::make_shared(std::nullopt, _myTree->side(), Str("Dir 5"), NodeType::Directory, + std::make_shared(_myTree->side(), Str("Dir 5"), NodeType::Directory, OperationType::Move | OperationType::Edit, "l5", 0, 0, 12345, _myTree->rootNode()); CPPUNIT_ASSERT(!node->hasChangeEvent()); diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index a40b2dcee..e7bb3dc85 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -174,7 +174,7 @@ void TestWorkers::testCreatePlaceholder() { CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); -#if defined(__APPLE__) && defined(__WIN32) +#if defined(__APPLE__) || defined(__WIN32) // Folder already exists exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath); CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); @@ -191,7 +191,7 @@ void TestWorkers::testCreatePlaceholder() { syncItem.setDirection(SyncDirection::Down); _syncPal->initProgress(syncItem); -#if defined(__APPLE__) && defined(__WIN32) +#if defined(__APPLE__) || defined(__WIN32) // Folder access denied IoError ioError{IoError::Unknown}; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, false, false, false, ioError) && @@ -211,7 +211,7 @@ void TestWorkers::testCreatePlaceholder() { CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); -#if defined(__APPLE__) && defined(__WIN32) +#if defined(__APPLE__) || defined(__WIN32) // File already exists exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); @@ -239,7 +239,7 @@ void TestWorkers::testConvertToPlaceholder() { syncItem.setDirection(SyncDirection::Down); _syncPal->initProgress(syncItem); -#if defined(__APPLE__) && defined(__WIN32) +#if defined(__APPLE__) || defined(__WIN32) // Folder doesn't exist exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true); CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); @@ -264,7 +264,7 @@ void TestWorkers::testConvertToPlaceholder() { syncItem.setDirection(SyncDirection::Down); _syncPal->initProgress(syncItem); -#if defined(__APPLE__) && defined(__WIN32) +#if defined(__APPLE__) || defined(__WIN32) // Folder access denied IoError ioError{IoError::Unknown}; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, false, false, false, ioError) && From a1ce45c952fe3e849da20f68f18eae865db8616d Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 21 Oct 2024 13:44:33 +0200 Subject: [PATCH 21/56] Minor docstring change --- src/libcommon/utility/types.h | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 654026723..80dc3c8df 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -#pragma once +#pragma onceExitInfo #include #include @@ -238,7 +238,7 @@ struct ExitInfo { const ExitCause &cause() const { return _cause; } operator ExitCode() const { return _code; } operator ExitCause() const { return _cause; } - operator bool() const { return _code == ExitCode::Ok; } + constexpr operator bool() const { return _code == ExitCode::Ok; } constexpr explicit operator int() const { return toInt(_code) * 100 + toInt(_cause); } constexpr bool operator==(const ExitInfo &other) const { return _code == other._code && _cause == other._cause; } @@ -420,8 +420,8 @@ struct ItemType { SyncPath targetPath; // The value of the data member `ioError` is `IoError::NoSuchFileOrDirectory` if // - the file or directory indicated by `path` doesn't exist - // - the file or directory indicated by `path` is a symlink or an alias (in which case `linkType` is different - // from `LinkType::Unknown`) and its target doesn't exist. + // - the file or directory indicated by `path` is a symlink or an alias (in which case `linkType` is different from + // `LinkType::Unknown`) and its target doesn't exist. IoError ioError{IoError::Success}; }; @@ -460,8 +460,7 @@ struct VersionInfo { std::string tag; // Version number. Example: 3.6.4 std::string changeLog; // List of changes in this version std::uint64_t buildVersion = 0; // Example: 20240816 - std::string buildMinOsVersion; // Optionnal. Minimum supported version of the OS. Examples: 10.15, 11, server - // 2005, ... + std::string buildMinOsVersion; // Optionnal. Minimum supported version of the OS. Examples: 10.15, 11, server 2005, ... std::string downloadUrl; // URL to download the version [[nodiscard]] bool isValid() const { @@ -484,8 +483,7 @@ enum class SentryConfidentialityLevel { }; std::string toString(SentryConfidentialityLevel e); -// Adding a new types here requires to add it in stringToAppStateValue and appStateValueToString in -// libcommon/utility/utility.cpp +// Adding a new types here requires to add it in stringToAppStateValue and appStateValueToString in libcommon/utility/utility.cpp using AppStateValue = std::variant; /* @@ -544,8 +542,8 @@ inline bool bitWiseEnumToBool(const C a) { } namespace typesUtility { -std::wstring stringToWideString(const std::string &str); // Convert string to wstring (We can't use the s2ws of Utility - // because it's in libCommonServer and it includes types.h) +std::wstring stringToWideString(const std::string &str); // Convert string to wstring (We can't use the s2ws of Utility because + // it's in libCommonServer and it includes types.h) } // namespace typesUtility // Stream Operator (toString) From fffd6b029207a703d6c94246464bce3ceb72919d Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 11:55:15 +0200 Subject: [PATCH 22/56] Simplification of error handling for folder access by replacing with `SyncDirAccesError`. Added `Utility::isSameOrChildPath` for path comparison. Standardized error returns with `ExitInfo`. Improved error handling in `ExecutorWorker`. --- src/gui/parametersdialog.cpp | 11 +- src/libcommon/utility/types.cpp | 6 +- src/libcommon/utility/types.h | 5 +- src/libcommonserver/io/iohelper.h | 2 +- src/libcommonserver/io/iohelper_win.cpp | 9 +- src/libcommonserver/utility/utility.cpp | 13 ++ src/libcommonserver/utility/utility.h | 1 + src/libparms/db/error.cpp | 4 +- src/libparms/db/parmsdb.cpp | 7 +- .../API_v2/continuefilelistwithcursorjob.h | 2 +- .../network/API_v2/copytodirectoryjob.cpp | 4 +- .../jobs/network/API_v2/copytodirectoryjob.h | 2 +- .../jobs/network/API_v2/createdirjob.cpp | 4 +- .../jobs/network/API_v2/createdirjob.h | 2 +- .../API_v2/csvfullfilelistwithcursorjob.h | 2 +- .../jobs/network/API_v2/deletejob.h | 2 +- .../jobs/network/API_v2/downloadjob.h | 2 +- .../jobs/network/API_v2/duplicatejob.cpp | 4 +- .../jobs/network/API_v2/duplicatejob.h | 2 +- .../jobs/network/API_v2/getdriveslistjob.h | 2 +- .../jobs/network/API_v2/getfileinfojob.h | 2 +- .../jobs/network/API_v2/getfilelinkjob.h | 2 +- .../jobs/network/API_v2/getfilelistjob.h | 2 +- .../jobs/network/API_v2/getinfodrivejob.h | 2 +- .../jobs/network/API_v2/getinfodriveuserjob.h | 2 +- .../jobs/network/API_v2/getinfouserjob.h | 2 +- .../jobs/network/API_v2/getrootfilelistjob.h | 2 +- .../jobs/network/API_v2/getsizejob.h | 2 +- .../jobs/network/API_v2/getthumbnailjob.h | 2 +- .../API_v2/initfilelistwithcursorjob.h | 2 +- .../API_v2/jsonfullfilelistwithcursorjob.h | 2 +- .../jobs/network/API_v2/longpolljob.h | 2 +- .../jobs/network/API_v2/movejob.cpp | 4 +- .../jobs/network/API_v2/movejob.h | 2 +- .../jobs/network/API_v2/postfilelinkjob.h | 2 +- .../jobs/network/API_v2/renamejob.cpp | 4 +- .../jobs/network/API_v2/renamejob.h | 2 +- .../upload_session/uploadsessioncanceljob.h | 2 +- .../upload_session/uploadsessionchunkjob.h | 2 +- .../upload_session/uploadsessionfinishjob.cpp | 4 +- .../upload_session/uploadsessionfinishjob.h | 2 +- .../upload_session/uploadsessionstartjob.cpp | 4 +- .../upload_session/uploadsessionstartjob.h | 2 +- .../jobs/network/API_v2/uploadjob.cpp | 88 ++++------ .../jobs/network/API_v2/uploadjob.h | 6 +- .../jobs/network/abstractnetworkjob.cpp | 6 +- .../jobs/network/abstractnetworkjob.h | 2 +- .../jobs/network/getappversionjob.h | 3 +- src/libsyncengine/jobs/network/getavatarjob.h | 2 +- .../login/gettokenfromapppasswordjob.cpp | 4 +- .../login/gettokenfromapppasswordjob.h | 2 +- .../jobs/network/login/gettokenjob.cpp | 4 +- .../jobs/network/login/gettokenjob.h | 2 +- .../jobs/network/login/refreshtokenjob.cpp | 4 +- .../jobs/network/login/refreshtokenjob.h | 2 +- .../propagation/executor/executorworker.cpp | 151 +++++++++++------- .../propagation/executor/executorworker.h | 1 + .../operationsorterworker.cpp | 10 +- .../conflict_finder/conflictfinderworker.cpp | 4 +- .../platforminconsistencycheckerutility.cpp | 4 +- .../platforminconsistencycheckerutility.h | 2 +- .../platforminconsistencycheckerworker.cpp | 7 + src/libsyncengine/syncpal/syncpal.cpp | 72 +++++---- src/libsyncengine/syncpal/syncpal.h | 6 +- src/libsyncengine/syncpal/syncpalworker.cpp | 6 +- .../syncpal/tmpblacklistmanager.cpp | 41 ++++- .../syncpal/tmpblacklistmanager.h | 2 + .../computefsoperationworker.cpp | 4 +- .../file_system_observer/folderwatcher.h | 5 +- .../folderwatcher_linux.cpp | 9 +- .../folderwatcher_win.cpp | 2 + .../localfilesystemobserverworker.cpp | 57 +++---- .../localfilesystemobserverworker.h | 2 +- test/libcommonserver/utility/testutility.cpp | 16 ++ test/libcommonserver/utility/testutility.h | 2 + 75 files changed, 369 insertions(+), 299 deletions(-) diff --git a/src/gui/parametersdialog.cpp b/src/gui/parametersdialog.cpp index 66d7aa085..c29140833 100644 --- a/src/gui/parametersdialog.cpp +++ b/src/gui/parametersdialog.cpp @@ -424,14 +424,9 @@ QString ParametersDialog::getSyncPalSystemErrorText(const QString &err, ExitCaus "Synchronization will resume as soon as the folder is accessible.") .arg(err); - case ExitCause::SyncDirReadError: + case ExitCause::SyncDirAccesError: return tr("The synchronization folder is inaccessible (error %1).
    " - "Please check that you have read access to this folder.") - .arg(err); - - case ExitCause::SyncDirWriteError: - return tr("The synchronization folder is inaccessible (error %1).
    " - "Please check that you have write access to this folder.") + "Please check that you have read and write access to this folder.") .arg(err); case ExitCause::NotEnoughDiskSpace: @@ -832,7 +827,7 @@ QString ParametersDialog::getErrorLevelNodeText(const ErrorInfo &errorInfo) cons if (errorInfo.exitCause() == ExitCause::FileAccessError) { return tr( "Can't access item.
    " - "Please fix the write permissions and restart the synchronization."); + "Please fix the read and write permissions."); } if (errorInfo.exitCause() == ExitCause::MoveToTrashFailed) { diff --git a/src/libcommon/utility/types.cpp b/src/libcommon/utility/types.cpp index a3232f713..bb611fa0d 100644 --- a/src/libcommon/utility/types.cpp +++ b/src/libcommon/utility/types.cpp @@ -132,10 +132,8 @@ std::string toString(const ExitCause e) { return "InvalidSnapshot"; case ExitCause::SyncDirDoesntExist: return "SyncDirDoesntExist"; - case ExitCause::SyncDirReadError: - return "SyncDirReadError"; - case ExitCause::SyncDirWriteError: - return "SyncDirWriteError"; + case ExitCause::SyncDirAccesError: + return "SyncDirAccesError"; case ExitCause::HttpErr: return "HttpErr"; case ExitCause::HttpErrForbidden: diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 80dc3c8df..22cb68bf6 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -#pragma onceExitInfo +#pragma once #include #include @@ -195,8 +195,7 @@ enum class ExitCause { DbEntryNotFound, InvalidSnapshot, SyncDirDoesntExist, - SyncDirReadError, - SyncDirWriteError, + SyncDirAccesError, HttpErr, HttpErrForbidden, RedirectionError, diff --git a/src/libcommonserver/io/iohelper.h b/src/libcommonserver/io/iohelper.h index 5dabcad4f..21d7cbd31 100644 --- a/src/libcommonserver/io/iohelper.h +++ b/src/libcommonserver/io/iohelper.h @@ -428,7 +428,7 @@ struct IoHelper { static std::unique_ptr _psid; static TRUSTEE _trustee; static std::mutex _initRightsWindowsApiMutex; - static void initRightsWindowsApi(); + [[deprecated]] static void initRightsWindowsApi(); #endif }; diff --git a/src/libcommonserver/io/iohelper_win.cpp b/src/libcommonserver/io/iohelper_win.cpp index 5dda7f832..b1da9f212 100644 --- a/src/libcommonserver/io/iohelper_win.cpp +++ b/src/libcommonserver/io/iohelper_win.cpp @@ -250,7 +250,6 @@ bool IoHelper::getFileStat(const SyncPath &path, FileStat *filestat, IoError &io if ((isNtfs && !NT_SUCCESS(status)) || (!isNtfs && dwError != 0)) { // On FAT32 file system, NT_SUCCESS will return false even if it is a success, therefore we // also check GetLastError - LOGW_DEBUG(logger(), L"Error in zwQueryDirectoryFile: " << Utility::formatSyncPath(path.parent_path()).c_str()); CloseHandle(hParent); if (!NT_SUCCESS(status)) { @@ -258,6 +257,8 @@ bool IoHelper::getFileStat(const SyncPath &path, FileStat *filestat, IoError &io } else if (dwError != 0) { ioError = dWordError2ioError(dwError, logger()); } + LOGW_DEBUG(logger(), L"Error in zwQueryDirectoryFile: " << Utility::formatIoError(path, ioError)); + return isExpectedError(ioError); } @@ -469,7 +470,7 @@ void IoHelper::initRightsWindowsApi() { } // Always return false if ioError != IoError::Success, caller should call _isExpectedError -static bool setRightsWindowsApi(const SyncPath &path, DWORD permission, ACCESS_MODE accessMode, IoError &ioError, +[[deprecated]] static bool setRightsWindowsApi(const SyncPath &path, DWORD permission, ACCESS_MODE accessMode, IoError &ioError, log4cplus::Logger logger, bool inherite = false) noexcept { PACL pACLold = nullptr; // Current ACL PACL pACLnew = nullptr; // New ACL @@ -552,7 +553,7 @@ static bool setRightsWindowsApi(const SyncPath &path, DWORD permission, ACCESS_M return true; } -static bool getRightsWindowsApi(const SyncPath &path, bool &read, bool &write, bool &exec, IoError &ioError, +[[deprecated]] static bool getRightsWindowsApi(const SyncPath &path, bool &read, bool &write, bool &exec, IoError &ioError, log4cplus::Logger logger) noexcept { ioError = IoError::Success; read = false; @@ -721,7 +722,7 @@ bool IoHelper::checkIfIsJunction(const SyncPath &path, bool &isJunction, IoError ioError = IoError::Success; HANDLE hFile = - CreateFileW(Path2WStr(path).c_str(), FILE_WRITE_ATTRIBUTES, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + CreateFileW(Path2WStr(path).c_str(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL); if (hFile == INVALID_HANDLE_VALUE) { ioError = dWordError2ioError(GetLastError(), logger()); diff --git a/src/libcommonserver/utility/utility.cpp b/src/libcommonserver/utility/utility.cpp index 495b5a908..c13777972 100644 --- a/src/libcommonserver/utility/utility.cpp +++ b/src/libcommonserver/utility/utility.cpp @@ -409,6 +409,19 @@ bool Utility::isEqualNormalized(const SyncPath &a, const SyncPath &b) { return aNormalized == bNormalized; } +bool Utility::isSameOrChildPath(const SyncPath &potentialChild, const SyncPath &path) { + if (path == potentialChild) return true; + for (auto it = potentialChild.begin(), it2 = path.begin(); it != potentialChild.end(); ++it, ++it2) { + if (it2 == path.end()) { + return true; + } + if (*it != *it2) { + return false; + } + } + return false; +} + bool Utility::moveItemToTrash(const SyncPath &itemPath) { return moveItemToTrash_private(itemPath); } diff --git a/src/libcommonserver/utility/utility.h b/src/libcommonserver/utility/utility.h index d4425a60e..0cd85f936 100644 --- a/src/libcommonserver/utility/utility.h +++ b/src/libcommonserver/utility/utility.h @@ -102,6 +102,7 @@ struct COMMONSERVER_EXPORT Utility { static bool endsWithInsensitive(const SyncName &str, const SyncName &suffix); static bool isEqualInsensitive(const SyncName &a, const SyncName &b); #endif + static bool isSameOrChildPath(const SyncPath &potentialChild, const SyncPath &path); /** * Normalize the SyncName parameters before comparing them. * @param a SyncName value to be compared. diff --git a/src/libparms/db/error.cpp b/src/libparms/db/error.cpp index 9f204a383..6d5770782 100644 --- a/src/libparms/db/error.cpp +++ b/src/libparms/db/error.cpp @@ -96,8 +96,8 @@ bool Error::isSimilarTo(const Error &other) const { case ErrorLevel::Node: { if (_exitCode == ExitCode::SystemError && _exitCause == ExitCause::FileAccessError && other.exitCode() == ExitCode::SystemError && other.exitCause() == ExitCause::FileAccessError && - (Utility::startsWith(_path.lexically_normal(), other.path().lexically_normal()) || - Utility::startsWith(other.path().lexically_normal(), _path.lexically_normal()))) { + (Utility::isSameOrChildPath(_path.lexically_normal(), other.path().lexically_normal()) || + Utility::isSameOrChildPath(other.path().lexically_normal(), _path.lexically_normal()))) { return true; } diff --git a/src/libparms/db/parmsdb.cpp b/src/libparms/db/parmsdb.cpp index b57bcbb79..657c2d558 100644 --- a/src/libparms/db/parmsdb.cpp +++ b/src/libparms/db/parmsdb.cpp @@ -432,8 +432,8 @@ #define UPDATE_ERROR_REQUEST_ID "update_error" #define UPDATE_ERROR_REQUEST \ - "UPDATE error SET time=?1 " \ - "WHERE dbId=?2;" + "UPDATE error SET time=?1, path=?2 " \ + "WHERE dbId=?3;" #define DELETE_ALL_ERROR_BY_EXITCODE_REQUEST_ID "delete_error_by_exitcode" #define DELETE_ALL_ERROR_BY_EXITCODE_REQUEST \ @@ -2735,7 +2735,8 @@ bool ParmsDb::updateError(const Error &err, bool &found) { ASSERT(queryResetAndClearBindings(UPDATE_ERROR_REQUEST_ID)); ASSERT(queryBindValue(UPDATE_ERROR_REQUEST_ID, 1, err.time())); - ASSERT(queryBindValue(UPDATE_ERROR_REQUEST_ID, 2, err.dbId())); + ASSERT(queryBindValue(UPDATE_ERROR_REQUEST_ID, 2, err.path())); + ASSERT(queryBindValue(UPDATE_ERROR_REQUEST_ID, 3, err.dbId())); if (!queryExec(UPDATE_ERROR_REQUEST_ID, errId, error)) { LOG_WARN(_logger, "Error running query: " << UPDATE_ERROR_REQUEST_ID); return false; diff --git a/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h b/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h index ed8e1cc5e..e26aceff2 100644 --- a/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h +++ b/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h @@ -30,7 +30,7 @@ class ContinueFileListWithCursorJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } std::string _cursor; }; diff --git a/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.cpp b/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.cpp index c24f52e07..f3804a2c7 100644 --- a/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.cpp @@ -58,14 +58,14 @@ std::string CopyToDirectoryJob::getSpecificUrl() { return str; } -void CopyToDirectoryJob::setData(bool &canceled) { +ExitInfo CopyToDirectoryJob::setData() { Poco::JSON::Object json; json.set("name", _newName); std::stringstream ss; json.stringify(ss); _data = ss.str(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.h b/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.h index 5f6ded6b8..75b57571b 100644 --- a/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.h +++ b/src/libsyncengine/jobs/network/API_v2/copytodirectoryjob.h @@ -35,7 +35,7 @@ class CopyToDirectoryJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; NodeId _remoteFileId; NodeId _remoteDestId; diff --git a/src/libsyncengine/jobs/network/API_v2/createdirjob.cpp b/src/libsyncengine/jobs/network/API_v2/createdirjob.cpp index 672afca92..d19af8224 100644 --- a/src/libsyncengine/jobs/network/API_v2/createdirjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/createdirjob.cpp @@ -51,7 +51,7 @@ std::string CreateDirJob::getSpecificUrl() { return str; } -void CreateDirJob::setData(bool &canceled) { +ExitInfo CreateDirJob::setData() { Poco::JSON::Object json; json.set("name", _name); if (!_color.empty()) { @@ -61,7 +61,7 @@ void CreateDirJob::setData(bool &canceled) { std::stringstream ss; json.stringify(ss); _data = ss.str(); - canceled = false; + return ExitCode::Ok; } bool CreateDirJob::handleResponse(std::istream &is) { diff --git a/src/libsyncengine/jobs/network/API_v2/createdirjob.h b/src/libsyncengine/jobs/network/API_v2/createdirjob.h index ffd0491bb..bc3e4808c 100644 --- a/src/libsyncengine/jobs/network/API_v2/createdirjob.h +++ b/src/libsyncengine/jobs/network/API_v2/createdirjob.h @@ -41,7 +41,7 @@ class CreateDirJob : public AbstractTokenNetworkJob { std::string getSpecificUrl() override; void setQueryParameters(Poco::URI &, bool &) override { /* Query parameters are not mandatory */ } - void setData(bool &canceled) override; + ExitInfo setData() override; SyncPath _filePath; NodeId _parentDirId; diff --git a/src/libsyncengine/jobs/network/API_v2/csvfullfilelistwithcursorjob.h b/src/libsyncengine/jobs/network/API_v2/csvfullfilelistwithcursorjob.h index d376754d8..25da6c6be 100644 --- a/src/libsyncengine/jobs/network/API_v2/csvfullfilelistwithcursorjob.h +++ b/src/libsyncengine/jobs/network/API_v2/csvfullfilelistwithcursorjob.h @@ -89,7 +89,7 @@ class CsvFullFileListWithCursorJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &) override; - inline virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } virtual bool handleResponse(std::istream &is) override; diff --git a/src/libsyncengine/jobs/network/API_v2/deletejob.h b/src/libsyncengine/jobs/network/API_v2/deletejob.h index 5d62a7839..bf7e04f06 100644 --- a/src/libsyncengine/jobs/network/API_v2/deletejob.h +++ b/src/libsyncengine/jobs/network/API_v2/deletejob.h @@ -32,7 +32,7 @@ class DeleteJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } const NodeId _remoteItemId; const NodeId _localItemId; diff --git a/src/libsyncengine/jobs/network/API_v2/downloadjob.h b/src/libsyncengine/jobs/network/API_v2/downloadjob.h index fee6e84aa..071f72c4c 100644 --- a/src/libsyncengine/jobs/network/API_v2/downloadjob.h +++ b/src/libsyncengine/jobs/network/API_v2/downloadjob.h @@ -39,7 +39,7 @@ class DownloadJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } virtual bool canRun() override; virtual void runJob() noexcept override; diff --git a/src/libsyncengine/jobs/network/API_v2/duplicatejob.cpp b/src/libsyncengine/jobs/network/API_v2/duplicatejob.cpp index 75f4a3ed0..699d9de5b 100644 --- a/src/libsyncengine/jobs/network/API_v2/duplicatejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/duplicatejob.cpp @@ -73,7 +73,7 @@ std::string DuplicateJob::getSpecificUrl() { return str; } -void DuplicateJob::setData(bool &canceled) { +ExitInfo DuplicateJob::setData() { Poco::JSON::Object json; SyncName name = _absoluteFinalPath.filename().native(); json.set("name", name); @@ -81,7 +81,7 @@ void DuplicateJob::setData(bool &canceled) { std::stringstream ss; json.stringify(ss); _data = ss.str(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/duplicatejob.h b/src/libsyncengine/jobs/network/API_v2/duplicatejob.h index f78dc0c03..2baa13e6d 100644 --- a/src/libsyncengine/jobs/network/API_v2/duplicatejob.h +++ b/src/libsyncengine/jobs/network/API_v2/duplicatejob.h @@ -36,7 +36,7 @@ class DuplicateJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; NodeId _remoteFileId; SyncPath _absoluteFinalPath; diff --git a/src/libsyncengine/jobs/network/API_v2/getdriveslistjob.h b/src/libsyncengine/jobs/network/API_v2/getdriveslistjob.h index 09b599ac3..b9ebf7007 100644 --- a/src/libsyncengine/jobs/network/API_v2/getdriveslistjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getdriveslistjob.h @@ -28,7 +28,7 @@ class GetDrivesListJob : public AbstractTokenNetworkJob { private: virtual void setQueryParameters(Poco::URI &uri, bool &) override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } virtual std::string getSpecificUrl() override; }; diff --git a/src/libsyncengine/jobs/network/API_v2/getfileinfojob.h b/src/libsyncengine/jobs/network/API_v2/getfileinfojob.h index 644e8287a..0603a6cf8 100644 --- a/src/libsyncengine/jobs/network/API_v2/getfileinfojob.h +++ b/src/libsyncengine/jobs/network/API_v2/getfileinfojob.h @@ -44,7 +44,7 @@ class GetFileInfoJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } NodeId _nodeId; NodeId _parentNodeId; diff --git a/src/libsyncengine/jobs/network/API_v2/getfilelinkjob.h b/src/libsyncengine/jobs/network/API_v2/getfilelinkjob.h index 07810f566..7c4ddc100 100644 --- a/src/libsyncengine/jobs/network/API_v2/getfilelinkjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getfilelinkjob.h @@ -29,7 +29,7 @@ class GetFileLinkJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } NodeId _nodeId; }; diff --git a/src/libsyncengine/jobs/network/API_v2/getfilelistjob.h b/src/libsyncengine/jobs/network/API_v2/getfilelistjob.h index d6195f99a..d684622a2 100644 --- a/src/libsyncengine/jobs/network/API_v2/getfilelistjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getfilelistjob.h @@ -30,7 +30,7 @@ class GetFileListJob : public GetRootFileListJob { private: virtual std::string getSpecificUrl() override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } std::string _fileId; }; diff --git a/src/libsyncengine/jobs/network/API_v2/getinfodrivejob.h b/src/libsyncengine/jobs/network/API_v2/getinfodrivejob.h index b9b938151..1f5466835 100644 --- a/src/libsyncengine/jobs/network/API_v2/getinfodrivejob.h +++ b/src/libsyncengine/jobs/network/API_v2/getinfodrivejob.h @@ -32,7 +32,7 @@ class GetInfoDriveJob : public AbstractTokenNetworkJob { private: virtual void setQueryParameters(Poco::URI &, bool &canceled) override { canceled = false; } - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } }; } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/getinfodriveuserjob.h b/src/libsyncengine/jobs/network/API_v2/getinfodriveuserjob.h index a9060d7a3..7ec4eb8b5 100644 --- a/src/libsyncengine/jobs/network/API_v2/getinfodriveuserjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getinfodriveuserjob.h @@ -29,7 +29,7 @@ class GetInfoDriveUserJob : public AbstractTokenNetworkJob { private: virtual void setQueryParameters(Poco::URI &, bool &canceled) override { canceled = false; } - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } virtual std::string getSpecificUrl() override; }; diff --git a/src/libsyncengine/jobs/network/API_v2/getinfouserjob.h b/src/libsyncengine/jobs/network/API_v2/getinfouserjob.h index 3ea271587..1efc0daca 100644 --- a/src/libsyncengine/jobs/network/API_v2/getinfouserjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getinfouserjob.h @@ -29,7 +29,7 @@ class GetInfoUserJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &canceled) override { canceled = false; } - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } }; } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/getrootfilelistjob.h b/src/libsyncengine/jobs/network/API_v2/getrootfilelistjob.h index 454c53c4e..7834d3553 100644 --- a/src/libsyncengine/jobs/network/API_v2/getrootfilelistjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getrootfilelistjob.h @@ -33,7 +33,7 @@ class GetRootFileListJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &canceled) override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } uint64_t _page; bool _dirOnly; diff --git a/src/libsyncengine/jobs/network/API_v2/getsizejob.h b/src/libsyncengine/jobs/network/API_v2/getsizejob.h index b76e233f9..24855e1bb 100644 --- a/src/libsyncengine/jobs/network/API_v2/getsizejob.h +++ b/src/libsyncengine/jobs/network/API_v2/getsizejob.h @@ -37,7 +37,7 @@ class GetSizeJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &canceled) override { canceled = false; } - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } NodeId _nodeId; int64_t _size{0}; diff --git a/src/libsyncengine/jobs/network/API_v2/getthumbnailjob.h b/src/libsyncengine/jobs/network/API_v2/getthumbnailjob.h index e62c91711..4d9e0722a 100644 --- a/src/libsyncengine/jobs/network/API_v2/getthumbnailjob.h +++ b/src/libsyncengine/jobs/network/API_v2/getthumbnailjob.h @@ -29,7 +29,7 @@ class GetThumbnailJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &canceled) override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } NodeId _nodeId; unsigned _width; diff --git a/src/libsyncengine/jobs/network/API_v2/initfilelistwithcursorjob.h b/src/libsyncengine/jobs/network/API_v2/initfilelistwithcursorjob.h index a5d4170ef..2d6f4e69a 100644 --- a/src/libsyncengine/jobs/network/API_v2/initfilelistwithcursorjob.h +++ b/src/libsyncengine/jobs/network/API_v2/initfilelistwithcursorjob.h @@ -29,7 +29,7 @@ class InitFileListWithCursorJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &canceled) override; - inline virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } NodeId _dirId; }; diff --git a/src/libsyncengine/jobs/network/API_v2/jsonfullfilelistwithcursorjob.h b/src/libsyncengine/jobs/network/API_v2/jsonfullfilelistwithcursorjob.h index c769d0526..b538a573b 100644 --- a/src/libsyncengine/jobs/network/API_v2/jsonfullfilelistwithcursorjob.h +++ b/src/libsyncengine/jobs/network/API_v2/jsonfullfilelistwithcursorjob.h @@ -30,7 +30,7 @@ class JsonFullFileListWithCursorJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &canceled) override; - inline virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } virtual bool handleResponse(std::istream &is) override; diff --git a/src/libsyncengine/jobs/network/API_v2/longpolljob.h b/src/libsyncengine/jobs/network/API_v2/longpolljob.h index 47d4bf0ee..a4569aeaf 100644 --- a/src/libsyncengine/jobs/network/API_v2/longpolljob.h +++ b/src/libsyncengine/jobs/network/API_v2/longpolljob.h @@ -29,7 +29,7 @@ class LongPollJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &canceled) override; - inline virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } std::string _cursor; }; diff --git a/src/libsyncengine/jobs/network/API_v2/movejob.cpp b/src/libsyncengine/jobs/network/API_v2/movejob.cpp index 1ae64d9bf..cfa503048 100644 --- a/src/libsyncengine/jobs/network/API_v2/movejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/movejob.cpp @@ -89,7 +89,7 @@ std::string MoveJob::getSpecificUrl() { return str; } -void MoveJob::setData(bool &canceled) { +ExitInfo MoveJob::setData() { Poco::JSON::Object json; if (!_name.empty()) { json.set("name", _name); @@ -98,7 +98,7 @@ void MoveJob::setData(bool &canceled) { json.stringify(ss); _data = ss.str(); } - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/movejob.h b/src/libsyncengine/jobs/network/API_v2/movejob.h index 51e14d342..7975c4dae 100644 --- a/src/libsyncengine/jobs/network/API_v2/movejob.h +++ b/src/libsyncengine/jobs/network/API_v2/movejob.h @@ -34,7 +34,7 @@ class MoveJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &canceled) override { canceled = false; } - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; SyncPath _destFilepath; std::string _fileId; diff --git a/src/libsyncengine/jobs/network/API_v2/postfilelinkjob.h b/src/libsyncengine/jobs/network/API_v2/postfilelinkjob.h index b5c68c124..fa204222b 100644 --- a/src/libsyncengine/jobs/network/API_v2/postfilelinkjob.h +++ b/src/libsyncengine/jobs/network/API_v2/postfilelinkjob.h @@ -29,7 +29,7 @@ class PostFileLinkJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &uri, bool &canceled) override; - virtual void setData(bool &canceled) override { canceled = false; } + inline virtual ExitInfo setData() override { return ExitCode::Ok; } NodeId _nodeId; }; diff --git a/src/libsyncengine/jobs/network/API_v2/renamejob.cpp b/src/libsyncengine/jobs/network/API_v2/renamejob.cpp index 9e72e56f7..3cb2007ae 100644 --- a/src/libsyncengine/jobs/network/API_v2/renamejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/renamejob.cpp @@ -51,7 +51,7 @@ std::string RenameJob::getSpecificUrl() { return str; } -void RenameJob::setData(bool &canceled) { +ExitInfo RenameJob::setData() { Poco::JSON::Object json; SyncName name = _absoluteFinalPath.filename().native(); json.set("name", name); @@ -59,7 +59,7 @@ void RenameJob::setData(bool &canceled) { std::stringstream ss; json.stringify(ss); _data = ss.str(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/renamejob.h b/src/libsyncengine/jobs/network/API_v2/renamejob.h index fb84f852a..f42e23718 100644 --- a/src/libsyncengine/jobs/network/API_v2/renamejob.h +++ b/src/libsyncengine/jobs/network/API_v2/renamejob.h @@ -31,7 +31,7 @@ class RenameJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &canceled) override { canceled = false; } - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; std::string _remoteFileId; SyncPath _absoluteFinalPath; diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessioncanceljob.h b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessioncanceljob.h index b1e216b55..0ff11d366 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessioncanceljob.h +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessioncanceljob.h @@ -31,7 +31,7 @@ class UploadSessionCancelJob : public AbstractUploadSessionJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &) override {} + inline virtual ExitInfo setData() override { return ExitCode::Ok; } virtual bool handleError(std::istream &is, const Poco::URI &uri) override; }; diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionchunkjob.h b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionchunkjob.h index 6c8a7853f..adcf9f1fe 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionchunkjob.h +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionchunkjob.h @@ -40,7 +40,7 @@ class UploadSessionChunkJob : public AbstractUploadSessionJob { virtual std::string getSpecificUrl() override; virtual std::string getContentType(bool &canceled) override; virtual void setQueryParameters(Poco::URI &, bool &canceled) override; - virtual void setData(bool &) override {} + inline virtual ExitInfo setData() override { return ExitCode::Ok; } std::string _chunkHash; uint64_t _chunkNb = 0; diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.cpp b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.cpp index 08b2e3a64..a510b7080 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.cpp @@ -74,7 +74,7 @@ std::string UploadSessionFinishJob::getSpecificUrl() { return str; } -void UploadSessionFinishJob::setData(bool &canceled) { +ExitInfo UploadSessionFinishJob::setData() { Poco::JSON::Object json; json.set("total_chunk_hash", "xxh3:" + _totalChunkHash); json.set("total_chunks", _totalChunks); @@ -83,7 +83,7 @@ void UploadSessionFinishJob::setData(bool &canceled) { std::stringstream ss; json.stringify(ss); _data = ss.str(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.h b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.h index 37a81afcb..6bc0ea755 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.h +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionfinishjob.h @@ -42,7 +42,7 @@ class UploadSessionFinishJob : public AbstractUploadSessionJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &) override; + inline virtual ExitInfo setData() override; std::string _totalChunkHash; uint64_t _totalChunks = 0; diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.cpp b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.cpp index 5f460bbe2..39786d408 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.cpp @@ -45,7 +45,7 @@ std::string UploadSessionStartJob::getSpecificUrl() { return str; } -void UploadSessionStartJob::setData(bool &canceled) { +ExitInfo UploadSessionStartJob::setData() { Poco::JSON::Object json; using namespace std::chrono; auto timestamp = duration_cast(time_point_cast(system_clock::now()).time_since_epoch()); @@ -74,7 +74,7 @@ void UploadSessionStartJob::setData(bool &canceled) { std::stringstream ss; json.stringify(ss); _data = ss.str(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.h b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.h index 0874a8ecb..efe6742af 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.h +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/uploadsessionstartjob.h @@ -38,7 +38,7 @@ class UploadSessionStartJob : public AbstractUploadSessionJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override {} - virtual void setData(bool &) override; + inline virtual ExitInfo setData() override; SyncName _filename; NodeId _fileId; diff --git a/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp b/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp index 8274b84b5..6db1bf38b 100644 --- a/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/uploadjob.cpp @@ -79,8 +79,7 @@ bool UploadJob::canRun() { } if (!exists) { - LOGW_DEBUG(_logger, - L"Item does not exist anymore. Aborting current sync and restart - path=" << Path2WStr(_filePath)); + LOGW_DEBUG(_logger, L"Item does not exist anymore. Aborting current sync and restart - path=" << Path2WStr(_filePath)); _exitCode = ExitCode::NeedRestart; _exitCause = ExitCause::UnexpectedFileSystemEvent; return false; @@ -136,23 +135,21 @@ void UploadJob::setQueryParameters(Poco::URI &uri, bool &canceled) { canceled = false; } -void UploadJob::setData(bool &canceled) { - canceled = true; - +ExitInfo UploadJob::setData() { ItemType itemType; if (!IoHelper::getItemType(_filePath, itemType)) { LOGW_WARN(_logger, L"Error in IoHelper::getItemType - " << Utility::formatSyncPath(_filePath).c_str()); - return; + return ExitCode::SystemError; } if (itemType.ioError == IoError::NoSuchFileOrDirectory) { LOGW_DEBUG(_logger, L"Item does not exist anymore - " << Utility::formatSyncPath(_filePath).c_str()); - return; + return {ExitCode::SystemError, ExitCause::UnexpectedFileSystemEvent}; } if (itemType.ioError == IoError::AccessDenied) { LOGW_DEBUG(_logger, L"Item misses search permission - " << Utility::formatSyncPath(_filePath).c_str()); - return; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } _linkType = itemType.linkType; @@ -160,15 +157,14 @@ void UploadJob::setData(bool &canceled) { if (IoHelper::isLink(_linkType)) { LOG_DEBUG(_logger, "Read link data - type=" << _linkType); - if (!readLink()) return; + if (ExitInfo exitInfo = readLink(); !exitInfo) return exitInfo; } else { LOG_DEBUG(_logger, "Read file data"); - if (!readFile()) return; + if (ExitInfo exitInfo = readFile(); !exitInfo) return exitInfo; } _contentHash = Utility::computeXxHash(_data); - - canceled = false; + return ExitCode::Ok; } std::string UploadJob::getContentType(bool &canceled) { @@ -187,13 +183,11 @@ std::string UploadJob::getContentType(bool &canceled) { } } -bool UploadJob::readFile() { +ExitInfo UploadJob::readFile() { std::ifstream file(_filePath, std::ios_base::in | std::ios_base::binary); if (!file.is_open()) { LOGW_WARN(_logger, L"Failed to open file - path=" << Path2WStr(_filePath)); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } std::ostringstream ostrm; @@ -201,42 +195,34 @@ bool UploadJob::readFile() { if (ostrm.bad()) { // Read/writing error or logical error LOGW_WARN(_logger, L"Failed to insert file content into string stream - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } ostrm.flush(); if (ostrm.bad()) { // Read/writing error or logical error LOGW_WARN(_logger, L"Failed to flush string stream - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } file.close(); if (file.bad()) { // Read/writing error or logical error LOGW_WARN(_logger, L"Failed to close file - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } try { _data = ostrm.str(); } catch (const std::bad_alloc &) { LOGW_WARN(_logger, L"Memory allocation error when setting data content - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NotEnoughtMemory; - return false; + return {ExitCode::SystemError, ExitCause::NotEnoughtMemory}; } - return true; + return ExitCode::Ok; } -bool UploadJob::readLink() { +ExitInfo UploadJob::readLink() { if (_linkType == LinkType::Symlink) { std::error_code ec; _linkTarget = std::filesystem::read_symlink(_filePath, ec); @@ -249,24 +235,20 @@ bool UploadJob::readLink() { #endif if (!exists) { LOGW_DEBUG(_logger, L"File doesn't exist - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NotFound; - return false; + return {ExitCode::SystemError, ExitCause::NotFound}; } LOGW_WARN(_logger, L"Failed to read symlink - path=" << Path2WStr(_filePath).c_str() << L": " << Utility::s2ws(ec.message()).c_str() << L" (" << ec.value() << L")"); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::Unknown; - return false; + return ExitCode::SystemError; } _data = Path2Str(_linkTarget); } else if (_linkType == LinkType::Hardlink) { - if (!readFile()) { + if (ExitInfo exitInfo = readFile(); !exitInfo) { LOGW_WARN(_logger, L"Failed to read file - path=" << Path2WStr(_filePath).c_str()); - return false; + return exitInfo; } _linkTarget = _filePath; @@ -275,24 +257,19 @@ bool UploadJob::readLink() { IoError ioError = IoError::Success; if (!IoHelper::readJunction(_filePath, _data, _linkTarget, ioError)) { LOGW_WARN(_logger, L"Failed to read junction - " << Utility::formatIoError(_filePath, ioError).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::Unknown; - - return false; + return ExitCode::SystemError; } if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_DEBUG(_logger, L"File doesn't exist - " << Utility::formatSyncPath(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NotFound; - return false; + return {ExitCode::SystemError, ExitCause::NotFound}; } if (ioError == IoError::AccessDenied) { LOGW_DEBUG(_logger, L"File misses search permissions - " << Utility::formatSyncPath(_filePath).c_str()); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } #endif } else if (_linkType == LinkType::FinderAlias) { @@ -300,36 +277,27 @@ bool UploadJob::readLink() { IoError ioError = IoError::Success; if (!IoHelper::readAlias(_filePath, _data, _linkTarget, ioError)) { LOGW_WARN(_logger, L"Failed to read alias - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::Unknown; - - return false; + return ExitCode::SystemError; } if (ioError == IoError::NoSuchFileOrDirectory) { LOGW_DEBUG(_logger, L"File doesn't exist - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::NotFound; - - return false; + return {ExitCode::SystemError, ExitCause::NotFound}; } if (ioError == IoError::AccessDenied) { LOGW_DEBUG(_logger, L"File with insufficient access rights - path=" << Path2WStr(_filePath).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; - - return false; + return {ExitCode::SystemError, ExitCause::FileAccessError}; } assert(ioError == IoError::Success); // For every other error type, false should have been returned. #endif } else { LOG_WARN(_logger, "Link type not managed - type=" << _linkType); - return false; + return ExitCode::SystemError; } - return true; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/API_v2/uploadjob.h b/src/libsyncengine/jobs/network/API_v2/uploadjob.h index 61836d50f..d4d117cdf 100644 --- a/src/libsyncengine/jobs/network/API_v2/uploadjob.h +++ b/src/libsyncengine/jobs/network/API_v2/uploadjob.h @@ -47,11 +47,11 @@ class UploadJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &canceled) override; - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; virtual std::string getContentType(bool &canceled) override; - bool readFile(); - bool readLink(); + ExitInfo readFile(); + ExitInfo readLink(); SyncPath _filePath; SyncName _filename; diff --git a/src/libsyncengine/jobs/network/abstractnetworkjob.cpp b/src/libsyncengine/jobs/network/abstractnetworkjob.cpp index 0a1ca8c7b..82fd14217 100644 --- a/src/libsyncengine/jobs/network/abstractnetworkjob.cpp +++ b/src/libsyncengine/jobs/network/abstractnetworkjob.cpp @@ -136,10 +136,10 @@ void AbstractNetworkJob::runJob() noexcept { } bool canceled = false; - setData(canceled); // Must be called before setQueryParameters - if (canceled) { + if (ExitInfo exitInfo = setData(); !exitInfo) { // Must be called before setQueryParameters LOG_WARN(_logger, "Job " << jobId() << " is cancelled"); - _exitCode = ExitCode::DataError; + _exitCode = exitInfo.code(); + _exitCause = exitInfo.cause(); break; } diff --git a/src/libsyncengine/jobs/network/abstractnetworkjob.h b/src/libsyncengine/jobs/network/abstractnetworkjob.h index d68631bab..a90d6ecaa 100644 --- a/src/libsyncengine/jobs/network/abstractnetworkjob.h +++ b/src/libsyncengine/jobs/network/abstractnetworkjob.h @@ -117,7 +117,7 @@ class AbstractNetworkJob : public AbstractJob { std::string _octetStreamRes; virtual void setQueryParameters(Poco::URI &, bool &canceled) = 0; - virtual void setData(bool &canceled) = 0; + virtual ExitInfo setData() = 0; virtual std::string getContentType(bool &canceled) = 0; diff --git a/src/libsyncengine/jobs/network/getappversionjob.h b/src/libsyncengine/jobs/network/getappversionjob.h index 466ba6ace..5ee562084 100644 --- a/src/libsyncengine/jobs/network/getappversionjob.h +++ b/src/libsyncengine/jobs/network/getappversionjob.h @@ -39,8 +39,7 @@ class GetAppVersionJob : public AbstractNetworkJob { std::string getContentType(bool &canceled) override; void setQueryParameters(Poco::URI &, bool &canceled) override { /* no query parameters */ } - void setData(bool &canceled) override { /* no body parameters */ - } + ExitInfo setData() override { return ExitCode::Ok; } bool handleError(std::istream &is, const Poco::URI &uri) override; [[nodiscard]] DistributionChannel toDistributionChannel(const std::string &val) const; diff --git a/src/libsyncengine/jobs/network/getavatarjob.h b/src/libsyncengine/jobs/network/getavatarjob.h index d3ee6b6c2..56a0249fe 100644 --- a/src/libsyncengine/jobs/network/getavatarjob.h +++ b/src/libsyncengine/jobs/network/getavatarjob.h @@ -33,7 +33,7 @@ class GetAvatarJob : public AbstractNetworkJob { std::string getSpecificUrl() override { return {}; } std::string getContentType(bool &canceled) override; void setQueryParameters(Poco::URI &, bool &) override {} - void setData(bool &canceled) override { canceled = false; } + ExitInfo setData() override { return ExitCode::Ok; } bool handleError(std::istream &is, const Poco::URI &uri) override; bool handleResponse(std::istream &is) override; diff --git a/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.cpp b/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.cpp index ac8f3f3f2..42423c2c0 100644 --- a/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.cpp +++ b/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.cpp @@ -25,7 +25,7 @@ namespace KDC { GetTokenFromAppPasswordJob::GetTokenFromAppPasswordJob(const std::string &username, const std::string &password) : AbstractLoginJob(), _username(username), _password(password) {} -void GetTokenFromAppPasswordJob::setData(bool &canceled) { +ExitInfo GetTokenFromAppPasswordJob::setData() { Poco::URI uri; uri.addQueryParameter(usernameKey, _username); uri.addQueryParameter(passwordKey, _password); @@ -33,7 +33,7 @@ void GetTokenFromAppPasswordJob::setData(bool &canceled) { uri.addQueryParameter(clientIdKey, CLIENT_ID); _data = uri.getRawQuery(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.h b/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.h index 0da23e5b8..1c4fd6d65 100644 --- a/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.h +++ b/src/libsyncengine/jobs/network/login/gettokenfromapppasswordjob.h @@ -27,7 +27,7 @@ class GetTokenFromAppPasswordJob : public AbstractLoginJob { GetTokenFromAppPasswordJob(const std::string &username, const std::string &password); private: - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; std::string _username; std::string _password; diff --git a/src/libsyncengine/jobs/network/login/gettokenjob.cpp b/src/libsyncengine/jobs/network/login/gettokenjob.cpp index 226cad1ab..ba0ab065d 100644 --- a/src/libsyncengine/jobs/network/login/gettokenjob.cpp +++ b/src/libsyncengine/jobs/network/login/gettokenjob.cpp @@ -40,7 +40,7 @@ GetTokenJob::~GetTokenJob() { #endif } -void GetTokenJob::setData(bool &canceled) { +ExitInfo GetTokenJob::setData() { Poco::URI uri; uri.addQueryParameter(grantTypeKey, grantTypeAuthorization); uri.addQueryParameter(codeKey, _authorizationCode); @@ -49,7 +49,7 @@ void GetTokenJob::setData(bool &canceled) { uri.addQueryParameter(redirectUriKey, REDIRECT_URI); _data = uri.getRawQuery(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/login/gettokenjob.h b/src/libsyncengine/jobs/network/login/gettokenjob.h index 6c18c9f2a..093223c01 100644 --- a/src/libsyncengine/jobs/network/login/gettokenjob.h +++ b/src/libsyncengine/jobs/network/login/gettokenjob.h @@ -28,7 +28,7 @@ class GetTokenJob : public AbstractLoginJob { ~GetTokenJob(); private: - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; std::string _authorizationCode; std::string _codeVerifier; diff --git a/src/libsyncengine/jobs/network/login/refreshtokenjob.cpp b/src/libsyncengine/jobs/network/login/refreshtokenjob.cpp index 6da40c2df..9fc74c11c 100644 --- a/src/libsyncengine/jobs/network/login/refreshtokenjob.cpp +++ b/src/libsyncengine/jobs/network/login/refreshtokenjob.cpp @@ -41,7 +41,7 @@ RefreshTokenJob::~RefreshTokenJob() { #endif } -void RefreshTokenJob::setData(bool &canceled) { +ExitInfo RefreshTokenJob::setData() { Poco::URI uri; uri.addQueryParameter(grantTypeKey, refreshTokenKey); uri.addQueryParameter(refreshTokenKey, _apiToken.refreshToken().c_str()); @@ -49,7 +49,7 @@ void RefreshTokenJob::setData(bool &canceled) { uri.addQueryParameter(durationKey, infiniteKey); _data = uri.getRawQuery(); - canceled = false; + return ExitCode::Ok; } } // namespace KDC diff --git a/src/libsyncengine/jobs/network/login/refreshtokenjob.h b/src/libsyncengine/jobs/network/login/refreshtokenjob.h index 5dfde4bf9..8fa5d31c0 100644 --- a/src/libsyncengine/jobs/network/login/refreshtokenjob.h +++ b/src/libsyncengine/jobs/network/login/refreshtokenjob.h @@ -29,7 +29,7 @@ class RefreshTokenJob : public AbstractLoginJob { ~RefreshTokenJob(); private: - virtual void setData(bool &canceled) override; + virtual ExitInfo setData() override; }; } // namespace KDC diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 94c0ac78f..064f03bf3 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -162,6 +162,9 @@ void ExecutorWorker::execute() { increaseErrorCount(syncOp); cancelAllOngoingJobs(); break; + } else { + setProgressComplete(syncOp, SyncFileStatus::Error); + continue; } } @@ -406,15 +409,7 @@ ExitInfo ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptrtargetSide() == ReplicaSide::Local) { - return job->exitCode() == ExitCode::NeedRestart ? ExitInfo(ExitCode::DataError, ExitCause::FileAlreadyExist) - : ExitInfo(job->exitCode(), job->exitCause()); - } else if (syncOp->targetSide() == ReplicaSide::Remote) { - return job->exitCode() == ExitCode::NeedRestart ? ExitInfo(ExitCode::BackError, ExitCause::FileAlreadyExist) - : ExitInfo(job->exitCode(), job->exitCause()); - } - return ExitCode::LogicError; + return exitInfo; } if (ExitInfo exitInfo = convertToPlaceholder(relativeLocalFilePath, syncOp->targetSide() == ReplicaSide::Remote); @@ -475,18 +470,9 @@ ExitInfo ExecutorWorker::checkAlreadyExcluded(const SyncPath &absolutePath, cons if (!alreadyExist) { return ExitCode::Ok; } - - // The item already exists, exclude it - exitCode = PlatformInconsistencyCheckerUtility::renameLocalFile(absolutePath, - PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); - if (exitCode != ExitCode::Ok) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to rename file: " << Utility::formatSyncPath(absolutePath).c_str()); - return exitCode; - } return {ExitCode::DataError, ExitCause::FileAlreadyExist}; } -// !!! When returning false, _executorExitCode and _executorExitCause must be set !!! ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr &job) noexcept { // 1. If omit-flag is False, propagate the file or directory to replica Y, because the object is missing there. std::shared_ptr newCorrespondingParentNode = nullptr; @@ -905,6 +891,7 @@ ExitInfo ExecutorWorker::handleEditOp(SyncOpPtr syncOp, std::shared_ptr &job) { @@ -1155,6 +1142,7 @@ ExitInfo ExecutorWorker::handleMoveOp(SyncOpPtr syncOp, bool &ignored, bool &byp return exitInfo; } } + return ExitCode::Ok; } ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { @@ -1354,6 +1342,7 @@ ExitInfo ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &ignored, bool &b return exitInfo; } } + return ExitCode::Ok; } ExitInfo ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool &bypassProgressComplete) { @@ -1553,6 +1542,7 @@ ExitInfo ExecutorWorker::waitForAllJobsToFinish() { sendProgress(); } + return ExitCode::Ok; } ExitInfo ExecutorWorker::deleteFinishedAsyncJobs() { @@ -1724,13 +1714,15 @@ ExitInfo ExecutorWorker::handleFinishedJob(std::shared_ptr job, Syn return {job->exitCode(), job->exitCause()}; } else { // The error is managed and the execution can continue. + LOGW_DEBUG(_logger, L"Error successfully managed: " << job->exitCode() << L" " << job->exitCause() << L" on " + << syncOp->type() << L" operation for " + << Utility::formatSyncPath(syncOp->affectedNode()->getPath())); return {ExitCode::Ok, ExitCause::OperationCanceled}; } } else { // Propagate changes to DB and update trees std::shared_ptr newNode; - if (ExitInfo exitInfo = propagateChangeToDbAndTree(syncOp, job, newNode); exitInfo) { - // function + if (ExitInfo exitInfo = propagateChangeToDbAndTree(syncOp, job, newNode); !exitInfo) { cancelAllOngoingJobs(); return exitInfo; } @@ -1771,9 +1763,9 @@ ExitInfo ExecutorWorker::handleForbiddenAction(SyncOpPtr syncOp, const SyncPath case OperationType::Create: { cancelType = CancelType::Create; ignored = true; - removeFromDb = false; - PlatformInconsistencyCheckerUtility::renameLocalFile(absoluteLocalFilePath, - PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); + // If the rename failed (the file doesn't exist), it is a forbiden download, we should keep it in DB. + removeFromDb = PlatformInconsistencyCheckerUtility::renameLocalFile( + absoluteLocalFilePath, PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); break; } case OperationType::Move: { @@ -2407,24 +2399,7 @@ ExitInfo ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptrexitCode() == ExitCode::NeedRestart) { - // Special case: not an error but sync needs to be restarted - _syncPal->setRestart(true); - return ExitCode::NeedRestart; - } else if ((syncOp->targetSide() == ReplicaSide::Local && job->exitCode() == ExitCode::DataError && - job->exitCause() == ExitCause::FileAlreadyExist) || - (syncOp->targetSide() == ReplicaSide::Remote && job->exitCode() == ExitCode::BackError && - job->exitCause() == ExitCause::FileAlreadyExist)) { - auto localCreateDirJob(std::dynamic_pointer_cast(job)); - if (localCreateDirJob) { - LOGW_SYNCPAL_WARN(_logger, L"Item: " << Utility::formatSyncPath(localCreateDirJob->destFilePath()).c_str() - << L" already exist. Blacklisting it on " - L"local replica."); - PlatformInconsistencyCheckerUtility::renameLocalFile(_syncPal->localPath() / localCreateDirJob->destFilePath(), - PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); - } - return ExitCode::NeedRestart; - } else if (job->exitCode() != ExitCode::Ok) { + if (job->exitCode() != ExitCode::Ok) { LOGW_SYNCPAL_WARN(_logger, L"Failed to create directory: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); return {job->exitCode(), job->exitCause()}; } @@ -2570,6 +2545,10 @@ ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsE case static_cast(ExitInfo(ExitCode::SystemError, ExitCause::MoveToTrashFailed)): { return handleOpsFileAccessError(syncOp, opsExitInfo); } + case static_cast(ExitInfo(ExitCode::BackError, ExitCause::FileAlreadyExist)): + case static_cast(ExitInfo(ExitCode::DataError, ExitCause::FileAlreadyExist)): { + return handleOpsAlreadyExistError(syncOp, opsExitInfo); + } default: { break; } @@ -2582,39 +2561,87 @@ ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsE ExitInfo ExecutorWorker::handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { if (syncOp->targetSide() == ReplicaSide::Local && syncOp->type() == OperationType::Create) { // The item does not exist yet locally, we will only tmpBlacklist the remote item - _syncPal->handleAccessDeniedItem(syncOp->affectedNode()->getPath()); + if (ExitInfo exitInfo = _syncPal->handleAccessDeniedItem(syncOp->affectedNode()->getPath()); !exitInfo) { + return exitInfo; + } } else { // Both local and remote item will be temporarily blacklisted auto localNode = syncOp->targetSide() == ReplicaSide::Remote ? syncOp->affectedNode() : syncOp->correspondingNode(); if (!localNode) return ExitCode::LogicError; SyncPath relativeLocalFilePath = localNode->getPath(); - SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; NodeId localNodeId = localNode->id().has_value() ? *localNode->id() : NodeId(); - _syncPal->handleAccessDeniedItem(absoluteLocalFilePath, localNodeId, opsExitInfo.cause()); - _syncPal->blacklistTemporarily(syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), - syncOp->nodePath(ReplicaSide::Local), otherSide(syncOp->targetSide())); - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, - _syncPal->localPath() / syncOp->nodePath(ReplicaSide::Local), ConflictType::None, InconsistencyType::None, - CancelType::None, "", opsExitInfo.code(), opsExitInfo.cause()); - _syncPal->addError(error); - - if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); - return ExitCode::DataError; + if (ExitInfo exitInfo = _syncPal->handleAccessDeniedItem(relativeLocalFilePath, opsExitInfo.cause()); !exitInfo) { + return exitInfo; } + } + _syncPal->setRestart(true); + return removeDependentOps(syncOp); +} - if (syncOp->correspondingNode()) { - if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { +ExitInfo ExecutorWorker::handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { + // If the file/directory already exist either on local or remote side, we blacklist it localy and the remote verson will be + // downloaded again. + + if (syncOp->type() != OperationType::Create && + syncOp->type() != OperationType::Move) { // The above handling is only for create and move/rename operations. + return opsExitInfo; + } + + SyncPath relativeLocalPath; + SyncPath relativeRemotePath; + if (syncOp->targetSide() == ReplicaSide::Local) { + relativeRemotePath = syncOp->affectedNode()->getPath(); + // In case of a create the corresponding node is not set, remote and local relative path are the same + relativeLocalPath = syncOp->correspondingNode() ? syncOp->correspondingNode()->parentNode()->getPath() / syncOp->newName() + : relativeRemotePath; + + if (_syncPal->isTmpBlacklisted(relativeLocalPath, ReplicaSide::Local)) { + LOGW_SYNCPAL_DEBUG(_logger, Utility::formatSyncPath(relativeLocalPath) + << L" is already blacklisted locally, blacklisting remote corresponding node."); + _syncPal->blacklistTemporarily( + syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), relativeLocalPath, + ReplicaSide::Remote); + if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + return ExitCode::DataError; + } + + if (syncOp->correspondingNode() && !targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); return ExitCode::DataError; } + _syncPal->setRestart(true); + return removeDependentOps(syncOp); } + } else if (syncOp->targetSide() == ReplicaSide::Remote) { + relativeLocalPath = syncOp->affectedNode()->parentNode()->getPath() / syncOp->newName(); + } else { + return ExitCode::LogicError; } - removeDependentOps(syncOp); - return ExitCode::Ok; + SyncPath absoluteLocalPath = _syncPal->localPath() / relativeLocalPath; + LOGW_SYNCPAL_WARN(_logger, L"Item: " << Utility::formatSyncPath(absoluteLocalPath) + << L" already exist. Blacklisting it on " + L"local replica."); + + if (ExitInfo exitInfo = PlatformInconsistencyCheckerUtility::renameLocalFile( + absoluteLocalPath, PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); + !exitInfo) { + LOGW_WARN(_logger, "Failed to blacklist file: " << Utility::formatSyncPath(absoluteLocalPath) << " ExitCode::" + << exitInfo.code() << " ExitCause::" << exitInfo.cause()); + return ExitCode::DataError; // The synchronization will be re-started. + } + + + _syncPal->setRestart(true); + if (ExitInfo exitInfo = propagateDeleteToDbAndTree(syncOp); !exitInfo) { + LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " + << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + return exitInfo; + } + return removeDependentOps(syncOp); } ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { @@ -2634,7 +2661,8 @@ ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { auto remoteNode2 = syncOp2->affectedNode()->side() == ReplicaSide::Remote ? syncOp2->affectedNode() : syncOp2->correspondingNode(); SyncName nodeName = localNode2 ? localNode2->name() : SyncName(); - nodeName = (nodeName.empty() && remoteNode2) ? remoteNode2->name() : SyncName(); + if (nodeName.empty()) nodeName = remoteNode2 ? remoteNode2->name() : SyncName(); + if (localNode && localNode2 && (*localNode->id() == *localNode2->id() || localNode->isParentOf(localNode2))) { LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) << L" because it depends on " << syncOp->type() << L" operation on " @@ -2646,7 +2674,7 @@ ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { if (remoteNode && remoteNode2 && (*remoteNode->id() == *remoteNode2->id() || remoteNode->isParentOf(remoteNode2))) { LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) << L" because it depends on " << syncOp->type() << L" operation on " - << SyncName2WStr(nodeName) << L"wich failed."); + << SyncName2WStr(remoteNode->name()) << L"wich failed."); dependentOps.push_back(opId); } } @@ -2659,7 +2687,6 @@ ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { } if (!dependentOps.empty()) { LOGW_SYNCPAL_DEBUG(_logger, L"Removed " << dependentOps.size() << L" dependent operations."); - _syncPal->setRestart(true); } return ExitCode::Ok; diff --git a/src/libsyncengine/propagation/executor/executorworker.h b/src/libsyncengine/propagation/executor/executorworker.h index efb63ced9..1f28f5cbb 100644 --- a/src/libsyncengine/propagation/executor/executorworker.h +++ b/src/libsyncengine/propagation/executor/executorworker.h @@ -145,6 +145,7 @@ class ExecutorWorker : public OperationProcessor { ExitInfo handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo); ExitInfo handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo); + ExitInfo handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo opsExitInfo); ExitInfo removeDependentOps(SyncOpPtr syncOp); diff --git a/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp b/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp index 173d29706..1e5bdcd55 100644 --- a/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp +++ b/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp @@ -287,7 +287,7 @@ void OperationSorterWorker::fixMoveBeforeDelete() { } SyncPath sourcePath = moveOp->affectedNode()->moveOrigin().value(); - if (Utility::startsWith(sourcePath.lexically_normal(), + if (Utility::isSameOrChildPath(sourcePath.lexically_normal(), SyncPath(deleteDirPath.native() + Str("/")).lexically_normal())) { // move only if op is before moveOp moveFirstAfterSecond(deleteOp, moveOp); @@ -595,10 +595,10 @@ void OperationSorterWorker::fixMoveBeforeMoveHierarchyFlip() { } SyncPath ySourcePath = *yNode->moveOrigin(); - bool isXBelowY = - Utility::startsWith(xDestPath.lexically_normal(), SyncPath(yDestPath.native() + Str("/")).lexically_normal()); + bool isXBelowY = Utility::isSameOrChildPath(xDestPath.lexically_normal(), + SyncPath(yDestPath.native() + Str("/")).lexically_normal()); if (isXBelowY) { - bool isYBelowXInDb = Utility::startsWith(ySourcePath.lexically_normal(), + bool isYBelowXInDb = Utility::isSameOrChildPath(ySourcePath.lexically_normal(), SyncPath(xSourcePath.native() + Str("/")).lexically_normal()); if (isYBelowXInDb) { moveFirstAfterSecond(xOp, yOp); @@ -625,7 +625,7 @@ std::optional OperationSorterWorker::fixImpossibleFirstMoveOp // impossible move if dest = source + "/" SyncPath source = (o1->affectedNode()->moveOrigin().has_value() ? o1->affectedNode()->moveOrigin().value() : ""); SyncPath dest = o1->affectedNode()->getPath(); - if (!Utility::startsWith(dest.lexically_normal(), SyncPath(source.native() + Str("/")).lexically_normal())) { + if (!Utility::isSameOrChildPath(dest.lexically_normal(), SyncPath(source.native() + Str("/")).lexically_normal())) { return std::nullopt; } diff --git a/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp b/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp index 472c4fdb2..c0685d51d 100644 --- a/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp +++ b/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp @@ -409,9 +409,9 @@ std::optional> ConflictFinderWorker::determineMoveMoveCycl continue; } - if (Utility::startsWith(SyncPath(localDbPath).lexically_normal(), + if (Utility::isSameOrChildPath(SyncPath(localDbPath).lexically_normal(), SyncPath(remoteDbPath.native() + Str("/")).lexically_normal()) || - Utility::startsWith(SyncPath(remoteDbPath).lexically_normal(), + Utility::isSameOrChildPath(SyncPath(remoteDbPath).lexically_normal(), SyncPath(localDbPath.native() + Str("/")).lexically_normal())) { continue; } diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp index af9985b97..f3d653c25 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.cpp @@ -75,7 +75,7 @@ SyncName PlatformInconsistencyCheckerUtility::generateNewValidName(const SyncPat return sub + suffix + name.extension().native(); } -ExitCode PlatformInconsistencyCheckerUtility::renameLocalFile(const SyncPath &absoluteLocalPath, SuffixType suffixType, +ExitInfo PlatformInconsistencyCheckerUtility::renameLocalFile(const SyncPath &absoluteLocalPath, SuffixType suffixType, SyncPath *newPathPtr /*= nullptr*/) { const auto newName = PlatformInconsistencyCheckerUtility::instance()->generateNewValidName(absoluteLocalPath, suffixType); auto newFullPath = absoluteLocalPath.parent_path() / newName; @@ -87,7 +87,7 @@ ExitCode PlatformInconsistencyCheckerUtility::renameLocalFile(const SyncPath &ab *newPathPtr = std::move(newFullPath); } - return moveJob.exitCode(); + return {moveJob.exitCode(), moveJob.exitCause()}; } bool PlatformInconsistencyCheckerUtility::nameHasForbiddenChars(const SyncPath &name) { diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h index 89ae3bdc6..09a9c6d1e 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerutility.h @@ -42,7 +42,7 @@ class PlatformInconsistencyCheckerUtility { #endif bool checkReservedNames(const SyncName &name); SyncName generateNewValidName(const SyncPath &name, SuffixType suffixType); - static ExitCode renameLocalFile(const SyncPath &absoluteLocalPath, SuffixType suffixType, SyncPath *newPathPtr = nullptr); + static ExitInfo renameLocalFile(const SyncPath &absoluteLocalPath, SuffixType suffixType, SyncPath *newPathPtr = nullptr); private: PlatformInconsistencyCheckerUtility(); diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp index b04efac8c..dcf56d4df 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp @@ -191,6 +191,13 @@ void PlatformInconsistencyCheckerWorker::blacklistNode(const std::shared_ptrupdateTree(ReplicaSide::Local)->deleteNode(nodeIDs.localId)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node " << Utility::formatSyncPath(node->getPath())); + } + + if (!nodeIDs.remoteId.empty() && !_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(nodeIDs.remoteId)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node " << Utility::formatSyncPath(node->getPath())); + } _idsToBeRemoved.emplace_back(nodeIDs); } diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 1d6e0b16b..5559930b7 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1406,6 +1406,10 @@ void SyncPal::blacklistTemporarily(const NodeId &nodeId, const SyncPath &relativ _tmpBlacklistManager->blacklistItem(nodeId, relativePath, side); } +bool SyncPal::isTmpBlacklisted(const SyncPath &relativePath, ReplicaSide side) const { + return _tmpBlacklistManager->isTmpBlacklisted(relativePath, side); +} + void SyncPal::refreshTmpBlacklist() { _tmpBlacklistManager->refreshBlacklist(); } @@ -1414,51 +1418,51 @@ void SyncPal::removeItemFromTmpBlacklist(const NodeId &nodeId, ReplicaSide side) _tmpBlacklistManager->removeItemFromTmpBlacklist(nodeId, side); } -void SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, const NodeId &LocalNodeId, ExitCause cause) { +void SyncPal::removeItemFromTmpBlacklist(const SyncPath &relativePath) { + _tmpBlacklistManager->removeItemFromTmpBlacklist(relativePath); +} - Error error(syncDbId(), "", "", NodeType::Unknown, relativePath, ConflictType::None, InconsistencyType::None, - CancelType::None, "", ExitCode::SystemError, cause); - addError(error); - LOGW_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(LocalNodeId) - << L" is blacklisted temporarily because of access denied"); +ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause cause) { + if (relativePath.empty()) { + LOGW_SYNCPAL_WARN(_logger, L"Access error on root folder"); + return ExitInfo(ExitCode::SystemError, ExitCause::SyncDirAccesError); + } - std::optional remoteNodeId; - SyncPath remotePath; + NodeId localNodeId; + localNodeId = snapshot(ReplicaSide::Local)->itemId(relativePath); - // Try to found the corresponding node in Db - NodeId correspondingNodeId; - if (bool found = false; !_syncDb->correspondingNodeId(ReplicaSide::Local, LocalNodeId, correspondingNodeId, found)) { - LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::correspondingNodeId"); - } - if (bool found = false; !_syncDb->path(ReplicaSide::Remote, correspondingNodeId, remotePath, found)) { - LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::path"); + if (localNodeId.empty()) { + // The file does not exit yet on local FS, we do not have sufficient right on the parrent. + LOGW_DEBUG(_logger, + L"Item " << Utility::formatSyncPath(relativePath) << "isn't present local FS, blacklisting the parrent."); + return handleAccessDeniedItem(relativePath.parent_path(), cause); } - remoteNodeId = correspondingNodeId; + Error error(syncDbId(), "", "", relativePath.extension() == SyncPath() ? NodeType::Directory : NodeType::File, relativePath, + ConflictType::None, InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, cause); + addError(error); + LOGW_SYNCPAL_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(localNodeId) + << L" is blacklisted temporarily because of access denied"); - // If the corresponding node is not found, try to found the node in the snapshot - if (remoteNodeId->empty()) { - remoteNodeId = snapshotCopy(ReplicaSide::Remote)->itemId(relativePath); - if (remoteNodeId->empty()) { - LOGW_WARN(_logger, L"SyncPal::handleAccessDeniedItem Item " << Utility::formatSyncPath(relativePath) - << L" does not exist on the remote side."); - return; - } - remotePath = relativePath; + NodeId correspondingNodeId; + bool found; + correspondingNodeId = snapshot(ReplicaSide::Remote)->itemId(relativePath); + if (correspondingNodeId.empty() && + !_syncDb->correspondingNodeId(ReplicaSide::Local, localNodeId, correspondingNodeId, found)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in SyncDb::correspondingNodeId"); + return {ExitCode::DbError, ExitCause::Unknown}; } - if (!LocalNodeId.empty()) { // If the file/dir already exist on local FS, it will be blacklisted. - if (_tmpBlacklistManager->isTmpBlacklisted(LocalNodeId, ReplicaSide::Local)) { - LOGW_INFO(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(LocalNodeId) - << L" or one of its parent is already blacklisted temporarily."); - return; - } + _tmpBlacklistManager->blacklistItem(localNodeId, relativePath, ReplicaSide::Local); + if (!correspondingNodeId.empty()) _tmpBlacklistManager->blacklistItem(correspondingNodeId, relativePath, ReplicaSide::Remote); - _tmpBlacklistManager->blacklistItem(LocalNodeId, relativePath, ReplicaSide::Local); + if (!updateTree(ReplicaSide::Local)->deleteNode(localNodeId)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: " << Utility::formatSyncPath(relativePath)); } - if (remoteNodeId.has_value() && !remoteNodeId->empty() && !remotePath.empty()) { - _tmpBlacklistManager->blacklistItem(*remoteNodeId, remotePath, ReplicaSide::Remote); + if (!correspondingNodeId.empty() && !updateTree(ReplicaSide::Remote)->deleteNode(correspondingNodeId)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: " << Utility::formatSyncPath(relativePath)); } + return ExitCode::Ok; } void SyncPal::copySnapshots() { diff --git a/src/libsyncengine/syncpal/syncpal.h b/src/libsyncengine/syncpal/syncpal.h index 1640f6086..87374a31f 100644 --- a/src/libsyncengine/syncpal/syncpal.h +++ b/src/libsyncengine/syncpal/syncpal.h @@ -252,12 +252,12 @@ class SYNCENGINE_EXPORT SyncPal : public std::enable_shared_from_this { virtual void increaseErrorCount(const NodeId &nodeId, NodeType type, const SyncPath &relativePath, ReplicaSide side); virtual int getErrorCount(const NodeId &nodeId, ReplicaSide side) const noexcept; virtual void blacklistTemporarily(const NodeId &nodeId, const SyncPath &relativePath, ReplicaSide side); + virtual bool isTmpBlacklisted(const SyncPath &relativePath, ReplicaSide side) const; virtual void refreshTmpBlacklist(); virtual void removeItemFromTmpBlacklist(const NodeId &nodeId, ReplicaSide side); + virtual void removeItemFromTmpBlacklist(const SyncPath &relativePath); - // If the item already exists on the local side, path and nodeId are required. Else, only the remote path is required. - void handleAccessDeniedItem(const SyncPath &relativePath, const NodeId &nodeId = NodeId(), - ExitCause cause = ExitCause::FileAccessError); + ExitInfo handleAccessDeniedItem(const SyncPath &relativePath, ExitCause cause = ExitCause::FileAccessError); //! Makes copies of real-time snapshots to be used by synchronization workers. void copySnapshots(); diff --git a/src/libsyncengine/syncpal/syncpalworker.cpp b/src/libsyncengine/syncpal/syncpalworker.cpp index 8a271dd45..375fe4781 100644 --- a/src/libsyncengine/syncpal/syncpalworker.cpp +++ b/src/libsyncengine/syncpal/syncpalworker.cpp @@ -175,10 +175,12 @@ void SyncPalWorker::execute() { stopAndWaitForExitOfAllWorkers(fsoWorkers, stepWorkers); if ((stepWorkers[0] && workersExitCode[0] == ExitCode::SystemError && (stepWorkers[0]->exitCause() == ExitCause::NotEnoughDiskSpace || - stepWorkers[0]->exitCause() == ExitCause::FileAccessError)) || + stepWorkers[0]->exitCause() == ExitCause::FileAccessError || + stepWorkers[0]->exitCause() == ExitCause::SyncDirAccesError)) || (stepWorkers[1] && workersExitCode[1] == ExitCode::SystemError && (stepWorkers[1]->exitCause() == ExitCause::NotEnoughDiskSpace || - stepWorkers[1]->exitCause() == ExitCause::FileAccessError))) { + stepWorkers[1]->exitCause() == ExitCause::FileAccessError || + stepWorkers[0]->exitCause() == ExitCause::SyncDirAccesError))) { // Exit without error exitCode = ExitCode::Ok; } else if ((stepWorkers[0] && workersExitCode[0] == ExitCode::UpdateRequired) || diff --git a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp index f450f44af..1a5a6a656 100644 --- a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp +++ b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp @@ -89,6 +89,17 @@ void TmpBlacklistManager::blacklistItem(const NodeId &nodeId, const SyncPath &re } insertInBlacklist(nodeId, side); + + std::list toBeRemoved; + for (const auto &errorInfo: errors) { + if (Utility::isSameOrChildPath(errorInfo.second.path, relativePath) && errorInfo.first != nodeId) { + toBeRemoved.push_back(errorInfo.first); + } + } + + for (const auto &id: toBeRemoved) { + removeItemFromTmpBlacklist(id, side); + } } void TmpBlacklistManager::refreshBlacklist() { @@ -131,10 +142,38 @@ void TmpBlacklistManager::removeItemFromTmpBlacklist(const NodeId &nodeId, Repli errors.erase(nodeId); } +void TmpBlacklistManager::removeItemFromTmpBlacklist(const SyncPath &relativePath) { + NodeId removedId; + NodeId localId; + + // Find the node id of the item to be removed + for (const auto &[nodeId, tmpInfo]: _localErrors) { + if (Utility::isSameOrChildPath(tmpInfo.path, relativePath)) { + localId = nodeId; + break; + } + } + + for (const auto &[nodeId, tmpInfo]: _remoteErrors) { + if (Utility::isSameOrChildPath(tmpInfo.path, relativePath)) { + removedId = nodeId; + break; + } + } + + if (!localId.empty()) { + removeItemFromTmpBlacklist(localId, ReplicaSide::Local); + } + + if (!removedId.empty()) { + removeItemFromTmpBlacklist(removedId, ReplicaSide::Remote); + } +} + bool TmpBlacklistManager::isTmpBlacklisted(const SyncPath &path, ReplicaSide side) const { auto &errors = side == ReplicaSide::Local ? _localErrors : _remoteErrors; for (const auto &errorInfo: errors) { - if (Utility::startsWith(path, errorInfo.second.path)) return true; + if (Utility::isSameOrChildPath(path, errorInfo.second.path)) return true; } return false; diff --git a/src/libsyncengine/syncpal/tmpblacklistmanager.h b/src/libsyncengine/syncpal/tmpblacklistmanager.h index 25a405318..c57590a67 100644 --- a/src/libsyncengine/syncpal/tmpblacklistmanager.h +++ b/src/libsyncengine/syncpal/tmpblacklistmanager.h @@ -39,6 +39,8 @@ class TmpBlacklistManager { void blacklistItem(const NodeId &nodeId, const SyncPath &relativePath, ReplicaSide side); void refreshBlacklist(); void removeItemFromTmpBlacklist(const NodeId &nodeId, ReplicaSide side); + // Remove the item from local and/or remote blacklist + void removeItemFromTmpBlacklist(const SyncPath& relativePath); bool isTmpBlacklisted(const SyncPath &path, ReplicaSide side) const; int getErrorCount(const NodeId &nodeId, ReplicaSide side) const noexcept; diff --git a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp index 52ce8171d..3f908c9a0 100644 --- a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp @@ -158,7 +158,9 @@ ExitCode ComputeFSOperationWorker::inferChangeFromDbNode(const ReplicaSide side, if (const ExitInfo exitInfo = checkIfOkToDelete(side, dbPath, nodeId, isExcluded); !exitInfo) { if (exitInfo.code() == ExitCode::SystemError && exitInfo.cause() == ExitCause::FileAccessError) { // Blacklist node - _syncPal->handleAccessDeniedItem(dbPath, nodeId); + if (ExitInfo exitInfo = _syncPal->handleAccessDeniedItem(dbPath); !exitInfo) { + return exitInfo; + } // Update unsynced list cache updateUnsyncedList(); diff --git a/src/libsyncengine/update_detection/file_system_observer/folderwatcher.h b/src/libsyncengine/update_detection/file_system_observer/folderwatcher.h index 8d3df2a77..42589a404 100644 --- a/src/libsyncengine/update_detection/file_system_observer/folderwatcher.h +++ b/src/libsyncengine/update_detection/file_system_observer/folderwatcher.h @@ -40,12 +40,12 @@ class FolderWatcher { // The FolderWatcher can only become unreliable on Linux inline bool isReliable() const { return _isReliable; } - + ExitInfo exitInfo() const { return _exitInfo; } protected: // Implement this method in your subclass with the code you want your thread to run virtual void startWatching() = 0; virtual void stopWatching() = 0; - + void setExitInfo(ExitInfo exitInfo) { _exitInfo = exitInfo; } log4cplus::Logger _logger; LocalFileSystemObserverWorker *_parent; SyncPath _folder; @@ -56,6 +56,7 @@ class FolderWatcher { static void *executeFunc(void *thisWorker); std::unique_ptr _thread = nullptr; + ExitInfo _exitInfo = ExitCode::Ok; }; } // namespace KDC diff --git a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp index 6d70f2def..75350b513 100644 --- a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp @@ -130,6 +130,11 @@ void FolderWatcher_linux::startWatching() { bool FolderWatcher_linux::findSubFolders(const SyncPath &dir, std::list &fullList) { bool ok = true; bool isReadable = access(dir.c_str(), R_OK) == 0; + if (!isReadable) { + LOG4CPLUS_WARN(_logger, "SyncDir is not readable"); + setExitInfo({ExitCode::SystemError, ExitCause::SyncDirAccesError}); + return false; + } std::error_code ec; if (!(std::filesystem::exists(dir, ec) && isReadable)) { if (ec) { @@ -247,11 +252,11 @@ void FolderWatcher_linux::removeFoldersBelow(const SyncPath &dirPath) { // Remove the entry and all subentries while (it != _pathToWatch.end()) { auto itPath = it->first; - if (Utility::startsWith(itPath, dirPath)) { + if (Utility::isSameOrChildPath(itPath, dirPath)) { break; } - if (itPath != dirPath && Utility::startsWith(itPath, pathSlash)) { + if (itPath != dirPath && Utility::isSameOrChildPath(itPath, pathSlash)) { // order is 'foo', 'foo bar', 'foo/bar' ++it; continue; diff --git a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp index 4b30efa80..d5865b224 100644 --- a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_win.cpp @@ -55,6 +55,7 @@ void FolderWatcher_win::startWatching() { _stopEventHandle = CreateEvent(nullptr, true, false, nullptr); while (!_stop) { + setExitInfo(ExitCode::Ok); watchChanges(); if (!_stop) { @@ -86,6 +87,7 @@ void FolderWatcher_win::watchChanges() { DWORD errorCode = GetLastError(); LOGW_WARN(_logger, L"Failed to create handle for " << _folder.wstring().c_str() << L" - error:" << errorCode); _directoryHandle = nullptr; + setExitInfo({ExitCode::SystemError, ExitCause::SyncDirAccesError}); return; } diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index 723ed658a..cb8fda2ab 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -84,9 +84,16 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listlocalPath(), absolutePath); + const SyncPath absolutePath = changedItem.first.native(); + const SyncPath relativePath = CommonUtility::relativePath(_syncPal->localPath(), absolutePath); + if (_syncPal->isTmpBlacklisted(relativePath, ReplicaSide::Local)) { + _syncPal->removeItemFromTmpBlacklist(relativePath); + if (opTypeFromOS == OperationType::Edit) { + NodeId itemId = _snapshot->itemId(relativePath); + if (!itemId.empty()) _snapshot->setLastModified(itemId, 0); + } + } // Check if exists with same nodeId if (opTypeFromOS == OperationType::Delete) { NodeId prevNodeId = _snapshot->itemId(relativePath); @@ -226,7 +233,6 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listsize(nodeId), _snapshot->lastModified(nodeId), changed, ioError); if (!success) { @@ -283,21 +289,7 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listexists(nodeId); if (!itemExistsInSnapshot) { @@ -440,6 +432,12 @@ void LocalFileSystemObserverWorker::execute() { invalidateSnapshot(); break; } + if (!_folderWatcher->exitInfo()) { + exitCode = _folderWatcher->exitInfo().code(); + setExitCause(_folderWatcher->exitInfo().cause()); + invalidateSnapshot(); + break; + } // We never pause this thread if (!isFolderWatcherReliable() && @@ -591,20 +589,7 @@ void LocalFileSystemObserverWorker::sendAccessDeniedError(const SyncPath &absolu if (isExcluded) { return; } - - NodeId nodeId; - if (!IoHelper::getNodeId(absolutePath, nodeId)) { - nodeId = _snapshot->itemId(relativePath); - } - if(!nodeId.empty()){ - _syncPal->handleAccessDeniedItem(absolutePath, nodeId); - } else { - LOGW_SYNCPAL_WARN(_logger, L"Failed to handle access denied error (nodeId not found) for " << Utility::formatSyncPath(absolutePath)); - } - - Error error(_syncPal->syncDbId(), "", "", NodeType::Directory, absolutePath, ConflictType::None, InconsistencyType::None, - CancelType::None, "", ExitCode::SystemError, ExitCause::FileAccessError); - _syncPal->addError(error); + _syncPal->handleAccessDeniedItem(relativePath); } ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParentDirPath) { @@ -629,8 +614,8 @@ ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (itemType.ioError == IoError::AccessDenied) { LOGW_SYNCPAL_WARN(_logger, L"Sync localpath: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() << L" misses read permission"); - setExitCause(ExitCause::SyncDirReadError); - return {ExitCode::SystemError, ExitCause::SyncDirReadError}; + setExitCause(ExitCause::SyncDirAccesError); + return {ExitCode::SystemError, ExitCause::SyncDirAccesError}; } if (itemType.linkType != LinkType::None) { @@ -657,8 +642,8 @@ ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen if (ioError == IoError::AccessDenied) { LOGW_SYNCPAL_WARN(_logger, L"Sync localpath: " << Utility::formatSyncPath(absoluteParentDirPath).c_str() << L" misses read permission"); - setExitCause(ExitCause::SyncDirReadError); - return {ExitCode::SystemError, ExitCause::SyncDirReadError}; + setExitCause(ExitCause::SyncDirAccesError); + return {ExitCode::SystemError, ExitCause::SyncDirAccesError}; } diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h index 836e3d37f..65cf69751 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h @@ -33,7 +33,7 @@ class LocalFileSystemObserverWorker : public FileSystemObserverWorker { void stop() override; void changesDetected(const std::list> &changes); - + void errorDetected(ExitInfo exitInfo); virtual void forceUpdate() override; protected: diff --git a/test/libcommonserver/utility/testutility.cpp b/test/libcommonserver/utility/testutility.cpp index 7844e7357..0eb0ecf69 100644 --- a/test/libcommonserver/utility/testutility.cpp +++ b/test/libcommonserver/utility/testutility.cpp @@ -480,4 +480,20 @@ void TestUtility::testNormalizedSyncPath() { #endif } +void TestUtility::testIsSameOrParentPath() { + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("", "a")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a", "a/b")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a/b/c1")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c1", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("/a/b/c", "a/b/c")); + + CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("", "")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a", "")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a/b")); + CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a")); +} + } // namespace KDC diff --git a/test/libcommonserver/utility/testutility.h b/test/libcommonserver/utility/testutility.h index ec2473854..b53612397 100644 --- a/test/libcommonserver/utility/testutility.h +++ b/test/libcommonserver/utility/testutility.h @@ -60,6 +60,7 @@ class TestUtility : public CppUnit::TestFixture { CPPUNIT_TEST(isSubDir); CPPUNIT_TEST(testNormalizedSyncName); CPPUNIT_TEST(testNormalizedSyncPath); + CPPUNIT_TEST(testIsSameOrParentPath); CPPUNIT_TEST_SUITE_END(); public: @@ -104,6 +105,7 @@ class TestUtility : public CppUnit::TestFixture { void testcheckIfDirEntryIsManaged(); void testNormalizedSyncName(); void testNormalizedSyncPath(); + void testIsSameOrParentPath(); private: std::unique_ptr _testObj; From 434626f6c40f7041ec2f1b53637e1052313abbfb Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 12:03:32 +0200 Subject: [PATCH 23/56] Fix build on macOs and linux. --- src/libsyncengine/jobs/network/API_v2/downloadjob.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp b/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp index d5d43d39e..3c75d2458 100644 --- a/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp @@ -550,7 +550,7 @@ bool DownloadJob::moveTmpFile(const SyncPath &path, bool &restartSync) { } } #endif - else if (ec.value() == ERROR_ACCESS_DENIED) { + else if (IoHelper::stdError2ioError(ec.value()) == IoError::AccessDenied) { _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; From 7b5ff776fc182f4e339b436d6152b0bef067f65f Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 15:17:18 +0200 Subject: [PATCH 24/56] Fix build. --- src/libcommon/utility/types.h | 25 ++++++++----------- .../propagation/executor/executorworker.cpp | 7 +++--- src/libsyncengine/syncpal/syncpal.cpp | 2 +- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 22cb68bf6..5b9541b7e 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -118,12 +118,20 @@ concept EnumClass = std::is_enum_v; template // Any enum class that can be converted to (and from) int concept IntableEnum = EnumClass && std::is_convertible_v, int>; -template -inline constexpr int toInt(C e); - template // Any enum class that can be printed (with enumClassToString) concept PrintableEnum = EnumClass && requires(C e) { toString(e); }; +// Converters +template +inline constexpr int toInt(C e) { + return static_cast(e); +} + +template +inline constexpr C fromInt(int e) { + return static_cast(e); +} + enum class AppType { None, Server, Client }; std::string toString(AppType e); @@ -493,17 +501,6 @@ using AppStateValue = std::variant; template // Any enum class we want to allow bitwise operations (OperationType & InconsistencyType) concept AllowBitWiseOpEnum = IntableEnum && (std::is_same_v || std::is_same_v); -// Converters -template -inline constexpr int toInt(C e) { - return static_cast(e); -} - -template -inline constexpr C fromInt(int e) { - return static_cast(e); -} - // Operators template inline C operator|=(C &a, const C b) { diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 064f03bf3..52ff3f3e8 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -1260,7 +1260,8 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & // Propagate changes to DB and update trees std::shared_ptr newNode = nullptr; if (ExitInfo exitInfo = propagateChangeToDbAndTree(syncOp, job, newNode); !exitInfo) { - LOGW_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << syncOp->affectedNode()->name()); + LOGW_WARN(_logger, + L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } @@ -2629,8 +2630,8 @@ ExitInfo ExecutorWorker::handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo o if (ExitInfo exitInfo = PlatformInconsistencyCheckerUtility::renameLocalFile( absoluteLocalPath, PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); !exitInfo) { - LOGW_WARN(_logger, "Failed to blacklist file: " << Utility::formatSyncPath(absoluteLocalPath) << " ExitCode::" - << exitInfo.code() << " ExitCause::" << exitInfo.cause()); + LOGW_WARN(_logger, L"Failed to blacklist file: " << Utility::formatSyncPath(absoluteLocalPath) << L" ExitCode::" + << exitInfo.code() << L" ExitCause::" << exitInfo.cause()); return ExitCode::DataError; // The synchronization will be re-started. } diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 5559930b7..b02d4ff92 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1434,7 +1434,7 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause if (localNodeId.empty()) { // The file does not exit yet on local FS, we do not have sufficient right on the parrent. LOGW_DEBUG(_logger, - L"Item " << Utility::formatSyncPath(relativePath) << "isn't present local FS, blacklisting the parrent."); + L"Item " << Utility::formatSyncPath(relativePath) << L"isn't present local FS, blacklisting the parrent."); return handleAccessDeniedItem(relativePath.parent_path(), cause); } Error error(syncDbId(), "", "", relativePath.extension() == SyncPath() ? NodeType::Directory : NodeType::File, relativePath, From 5f7f7a14f3c0fc424f1a9141ffe761b5a9f1ed76 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 15:22:24 +0200 Subject: [PATCH 25/56] fix TestWorkers --- test/server/workers/testworkers.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index e7bb3dc85..7ec9a5d4a 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -264,7 +264,7 @@ void TestWorkers::testConvertToPlaceholder() { syncItem.setDirection(SyncDirection::Down); _syncPal->initProgress(syncItem); -#if defined(__APPLE__) || defined(__WIN32) +#if defined(__APPLE__) || defined(_WIN32) // Folder access denied IoError ioError{IoError::Unknown}; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, false, false, false, ioError) && @@ -272,7 +272,7 @@ void TestWorkers::testConvertToPlaceholder() { exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::SystemError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::NoSearchPermission); + CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::FileAccessError); ioError = IoError::Unknown; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, true, true, true, ioError) && From e7e2c45b889dd9b284493032c5183b19f338c59b Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 16:26:06 +0200 Subject: [PATCH 26/56] Fix testWorkers --- .../propagation/executor/executorworker.cpp | 2 +- test/server/workers/testworkers.cpp | 64 +++++++++---------- test/server/workers/testworkers.h | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 52ff3f3e8..376060d57 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -751,7 +751,7 @@ ExitInfo ExecutorWorker::convertToPlaceholder(const SyncPath &relativeLocalPath, #ifdef __APPLE__ // VfsMac::convertToPlaceholder needs only SyncFileItem::_dehydrated syncItem.setDehydrated(!hydrated); -#elif __WIN32 +#elif _WIN32 // VfsWin::convertToPlaceholder needs only SyncFileItem::_localNodeId FileStat fileStat; IoError ioError = IoError::Success; diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index 7ec9a5d4a..26f5fcb4e 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -28,7 +28,7 @@ namespace KDC { #if defined(__APPLE__) std::unique_ptr TestWorkers::_vfsPtr = nullptr; -#elif defined(__WIN32) +#elif defined(_WIN32) std::unique_ptr TestWorkers::_vfsPtr = nullptr; #else std::unique_ptr TestWorkers::_vfsPtr = nullptr; @@ -96,7 +96,7 @@ void TestWorkers::setUp() { _sync = Sync(1, drive.dbId(), localPathStr, testVariables.remotePath); #if defined(__APPLE__) _sync.setVirtualFileMode(VirtualFileMode::Mac); -#elif defined(__WIN32) +#elif defined(_WIN32) _sync.setVirtualFileMode(VirtualFileMode::Win); #else _sync.setVirtualFileMode(VirtualFileMode::Off); @@ -113,7 +113,7 @@ void TestWorkers::setUp() { // Create VFS instance VfsSetupParams vfsSetupParams; vfsSetupParams._syncDbId = _sync.dbId(); -#ifdef __WIN32 +#ifdef _WIN32 vfsSetupParams._driveId = drive.driveId(); vfsSetupParams._userId = user.userId(); #endif @@ -123,7 +123,7 @@ void TestWorkers::setUp() { #if defined(__APPLE__) _vfsPtr = std::unique_ptr(new VfsMac(vfsSetupParams)); -#elif defined(__WIN32) +#elif defined(_WIN32) _vfsPtr = std::unique_ptr(new VfsWin(vfsSetupParams)); #else _vfsPtr = std::unique_ptr(new VfsOff(vfsSetupParams)); @@ -156,8 +156,8 @@ void TestWorkers::testCreatePlaceholder() { // Progress not intialized { exitInfo = _syncPal->_executorWorker->createPlaceholder(SyncPath("dummy")); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); + CPPUNIT_ASSERT_EQUAL(ExitCode::DataError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::InvalidSnapshot, exitInfo.cause()); } // Create folder operation @@ -171,14 +171,14 @@ void TestWorkers::testCreatePlaceholder() { // Folder doesn't exist (normal case) exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); + CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, exitInfo.cause()); -#if defined(__APPLE__) || defined(__WIN32) +#if defined(__APPLE__) || defined(_WIN32) // Folder already exists exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFolderPath); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); + CPPUNIT_ASSERT_EQUAL(ExitCode::DataError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::InvalidSnapshot, exitInfo.cause()); #endif } @@ -191,15 +191,15 @@ void TestWorkers::testCreatePlaceholder() { syncItem.setDirection(SyncDirection::Down); _syncPal->initProgress(syncItem); -#if defined(__APPLE__) || defined(__WIN32) +#if defined(__APPLE__) || defined(_WIN32) // Folder access denied IoError ioError{IoError::Unknown}; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, false, false, false, ioError) && ioError == IoError::Success); exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::SystemError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::NoSearchPermission); + CPPUNIT_ASSERT_EQUAL(ExitCode::SystemError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::FileAccessError, exitInfo.cause()); ioError = IoError::Unknown; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, true, true, true, ioError) && @@ -208,14 +208,14 @@ void TestWorkers::testCreatePlaceholder() { // File doesn't exist (normal case) exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); + CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, exitInfo.cause()); -#if defined(__APPLE__) || defined(__WIN32) +#if defined(__APPLE__) || defined(_WIN32) // File already exists exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); + CPPUNIT_ASSERT_EQUAL(ExitCode::DataError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::InvalidSnapshot, exitInfo.cause()); #endif } } @@ -226,8 +226,8 @@ void TestWorkers::testConvertToPlaceholder() { // Progress not intialized { exitInfo = _syncPal->_executorWorker->convertToPlaceholder(SyncPath("dummy"), true); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); + CPPUNIT_ASSERT_EQUAL(ExitCode::DataError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::InvalidSnapshot, exitInfo.cause()); } // Convert folder operation @@ -239,11 +239,11 @@ void TestWorkers::testConvertToPlaceholder() { syncItem.setDirection(SyncDirection::Down); _syncPal->initProgress(syncItem); -#if defined(__APPLE__) || defined(__WIN32) +#if defined(__APPLE__) || defined(_WIN32) // Folder doesn't exist exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); + CPPUNIT_ASSERT_EQUAL(ExitCode::DataError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::InvalidSnapshot, exitInfo.cause()); #endif // Folder already exists (normal case) @@ -251,8 +251,8 @@ void TestWorkers::testConvertToPlaceholder() { CPPUNIT_ASSERT(std::filesystem::create_directory(_syncPal->localPath() / relativeFolderPath, ec) && ec.value() == 0); exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFolderPath, true); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); + CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, exitInfo.cause()); } // Convert file operation @@ -271,8 +271,8 @@ void TestWorkers::testConvertToPlaceholder() { ioError == IoError::Success); exitInfo = _syncPal->_executorWorker->createPlaceholder(relativeFilePath); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::SystemError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::FileAccessError); + CPPUNIT_ASSERT_EQUAL(ExitCode::SystemError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::FileAccessError, exitInfo.cause()); ioError = IoError::Unknown; CPPUNIT_ASSERT(IoHelper::setRights(_syncPal->localPath() / relativeFolderPath, true, true, true, ioError) && @@ -280,8 +280,8 @@ void TestWorkers::testConvertToPlaceholder() { // File doesn't exist exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFilePath, true); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::DataError); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::InvalidSnapshot); + CPPUNIT_ASSERT_EQUAL(ExitCode::DataError, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::InvalidSnapshot, exitInfo.cause()); #endif // File already exists (normal case) @@ -291,8 +291,8 @@ void TestWorkers::testConvertToPlaceholder() { } exitInfo = _syncPal->_executorWorker->convertToPlaceholder(relativeFilePath, true); - CPPUNIT_ASSERT_EQUAL(exitInfo.code(), ExitCode::Ok); - CPPUNIT_ASSERT_EQUAL(exitInfo.cause(), ExitCause::Unknown); + CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, exitInfo.code()); + CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, exitInfo.cause()); } } diff --git a/test/server/workers/testworkers.h b/test/server/workers/testworkers.h index 661d47812..cea74216e 100644 --- a/test/server/workers/testworkers.h +++ b/test/server/workers/testworkers.h @@ -57,7 +57,7 @@ class TestWorkers : public CppUnit::TestFixture { #if defined(__APPLE__) static std::unique_ptr _vfsPtr; -#elif defined(__WIN32) +#elif defined(_WIN32) static std::unique_ptr _vfsPtr; #else static std::unique_ptr _vfsPtr; From aacbc3282fa3f2ffc5c2745bcb3ec86ce623d8f4 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 16:41:35 +0200 Subject: [PATCH 27/56] Deactivate testworkers. --- test/CMakeLists.txt | 2 +- test/server/workers/testworkers.cpp | 2 +- test/server/workers/testworkers.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index de1a8d327..f823d153e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,4 +28,4 @@ add_subdirectory(libcommonserver) add_subdirectory(libcommon) add_subdirectory(libparms) add_subdirectory(libsyncengine) -add_subdirectory(server) +#add_subdirectory(server) diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index 26f5fcb4e..582fae4ba 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -124,7 +124,7 @@ void TestWorkers::setUp() { #if defined(__APPLE__) _vfsPtr = std::unique_ptr(new VfsMac(vfsSetupParams)); #elif defined(_WIN32) - _vfsPtr = std::unique_ptr(new VfsWin(vfsSetupParams)); + _vfsPtr = std::unique_ptr(new VfsWin(vfsSetupParams, nullptr)); #else _vfsPtr = std::unique_ptr(new VfsOff(vfsSetupParams)); #endif diff --git a/test/server/workers/testworkers.h b/test/server/workers/testworkers.h index cea74216e..3c90fe80e 100644 --- a/test/server/workers/testworkers.h +++ b/test/server/workers/testworkers.h @@ -20,7 +20,7 @@ #if defined(__APPLE__) #include "server/vfs/mac/vfs_mac.h" -#elif defined(__WIN32) +#elif defined(_WIN32) #include "server/vfs/win/vfs_win.h" #else #include "libcommonserver/vfs.h" From 04c2a559054a73dae81a55cdce07f314b6cde216 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 24 Oct 2024 16:44:19 +0200 Subject: [PATCH 28/56] Fix void TestUtility::testIsSameOrParentPath() { --- test/libcommonserver/utility/testutility.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/libcommonserver/utility/testutility.cpp b/test/libcommonserver/utility/testutility.cpp index 0eb0ecf69..a3fec273e 100644 --- a/test/libcommonserver/utility/testutility.cpp +++ b/test/libcommonserver/utility/testutility.cpp @@ -488,12 +488,12 @@ void TestUtility::testIsSameOrParentPath() { CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a/b/c1")); CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c1", "a/b/c")); CPPUNIT_ASSERT(!Utility::isSameOrChildPath("/a/b/c", "a/b/c")); - + + CPPUNIT_ASSERT(Utility::isSameOrChildPath("", "")); CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a/b/c")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("", "")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a", "")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a/b")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a")); + CPPUNIT_ASSERT(Utility::isSameOrChildPath("a", "")); + CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a/b")); + CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a")); } } // namespace KDC From e5de030b35b74677a8842dc40efe31ef6bb61226 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 25 Oct 2024 11:33:24 +0200 Subject: [PATCH 29/56] First PR review. --- src/libcommonserver/io/iohelper.h | 2 +- src/libcommonserver/io/iohelper_win.cpp | 4 +- src/libsyncengine/jobs/local/localcopyjob.cpp | 7 +++ .../jobs/local/localdeletejob.cpp | 11 +++- .../API_v2/continuefilelistwithcursorjob.h | 2 +- .../propagation/executor/executorworker.cpp | 6 +- src/libsyncengine/syncpal/syncpal.cpp | 23 +++---- src/libsyncengine/syncpal/syncpalworker.cpp | 2 +- .../computefsoperationworker.cpp | 8 +-- .../localfilesystemobserverworker.cpp | 60 +++++++++---------- .../localfilesystemobserverworker.h | 1 - src/server/appserver.cpp | 2 +- test/CMakeLists.txt | 2 +- 13 files changed, 69 insertions(+), 61 deletions(-) diff --git a/src/libcommonserver/io/iohelper.h b/src/libcommonserver/io/iohelper.h index 21d7cbd31..5dabcad4f 100644 --- a/src/libcommonserver/io/iohelper.h +++ b/src/libcommonserver/io/iohelper.h @@ -428,7 +428,7 @@ struct IoHelper { static std::unique_ptr _psid; static TRUSTEE _trustee; static std::mutex _initRightsWindowsApiMutex; - [[deprecated]] static void initRightsWindowsApi(); + static void initRightsWindowsApi(); #endif }; diff --git a/src/libcommonserver/io/iohelper_win.cpp b/src/libcommonserver/io/iohelper_win.cpp index b1da9f212..c4cbdeb33 100644 --- a/src/libcommonserver/io/iohelper_win.cpp +++ b/src/libcommonserver/io/iohelper_win.cpp @@ -470,7 +470,7 @@ void IoHelper::initRightsWindowsApi() { } // Always return false if ioError != IoError::Success, caller should call _isExpectedError -[[deprecated]] static bool setRightsWindowsApi(const SyncPath &path, DWORD permission, ACCESS_MODE accessMode, IoError &ioError, +static bool setRightsWindowsApi(const SyncPath &path, DWORD permission, ACCESS_MODE accessMode, IoError &ioError, log4cplus::Logger logger, bool inherite = false) noexcept { PACL pACLold = nullptr; // Current ACL PACL pACLnew = nullptr; // New ACL @@ -553,7 +553,7 @@ void IoHelper::initRightsWindowsApi() { return true; } -[[deprecated]] static bool getRightsWindowsApi(const SyncPath &path, bool &read, bool &write, bool &exec, IoError &ioError, +static bool getRightsWindowsApi(const SyncPath &path, bool &read, bool &write, bool &exec, IoError &ioError, log4cplus::Logger logger) noexcept { ioError = IoError::Success; read = false; diff --git a/src/libsyncengine/jobs/local/localcopyjob.cpp b/src/libsyncengine/jobs/local/localcopyjob.cpp index d1ae8e8af..5aa9cdecc 100644 --- a/src/libsyncengine/jobs/local/localcopyjob.cpp +++ b/src/libsyncengine/jobs/local/localcopyjob.cpp @@ -93,6 +93,13 @@ void LocalCopyJob::runJob() { LOGW_WARN(_logger, L"Failed to copy item " << Path2WStr(_source).c_str() << L" to " << Path2WStr(_dest).c_str() << L": " << Utility::s2ws(fsError.what()).c_str() << L" (" << fsError.code().value() << L")"); + if (IoHelper::stdError2ioError(fsError.code()) == IoError::AccessDenied) { + _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::FileAccessError; + } else { + _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + } _exitCode = ExitCode::SystemError; _exitCause = ExitCause::Unknown; } catch (...) { diff --git a/src/libsyncengine/jobs/local/localdeletejob.cpp b/src/libsyncengine/jobs/local/localdeletejob.cpp index 3e78f7431..073a08051 100644 --- a/src/libsyncengine/jobs/local/localdeletejob.cpp +++ b/src/libsyncengine/jobs/local/localdeletejob.cpp @@ -182,10 +182,15 @@ void LocalDeleteJob::runJob() { LOGW_DEBUG(_logger, L"Delete item with " << Utility::formatSyncPath(_absolutePath).c_str()); std::error_code ec; std::filesystem::remove_all(_absolutePath, ec); - if (ec) { // We consider this as a permission denied error + if (ec) { LOGW_WARN(_logger, L"Failed to delete item with path " << Utility::formatStdError(_absolutePath, ec).c_str()); - _exitCode = ExitCode::SystemError; - _exitCause = ExitCause::FileAccessError; + if (IoHelper::stdError2ioError(ec) == IoError::AccessDenied) { + _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::FileAccessError; + } else { + _exitCode = ExitCode::SystemError; + _exitCause = ExitCause::Unknown; + } return; } diff --git a/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h b/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h index e26aceff2..47c274514 100644 --- a/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h +++ b/src/libsyncengine/jobs/network/API_v2/continuefilelistwithcursorjob.h @@ -30,7 +30,7 @@ class ContinueFileListWithCursorJob : public AbstractTokenNetworkJob { private: virtual std::string getSpecificUrl() override; virtual void setQueryParameters(Poco::URI &, bool &) override; - inline virtual ExitInfo setData() override { return ExitCode::Ok; } + virtual ExitInfo setData() override { return ExitCode::Ok; } std::string _cursor; }; diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 376060d57..9fa6f2a23 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -154,9 +154,6 @@ void ExecutorWorker::execute() { } if (!executorExitInfo) { - if (!bypassProgressComplete) { - setProgressComplete(syncOp, SyncFileStatus::Error); - } executorExitInfo = handleOpsExecutionError(syncOp, executorExitInfo); if (!executorExitInfo) { increaseErrorCount(syncOp); @@ -166,6 +163,9 @@ void ExecutorWorker::execute() { setProgressComplete(syncOp, SyncFileStatus::Error); continue; } + if (!bypassProgressComplete) { + setProgressComplete(syncOp, SyncFileStatus::Error); + } } if (job) { diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index b02d4ff92..7d451e245 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1137,7 +1137,7 @@ ExitCode SyncPal::fixCorruptedFile(const std::unordered_map &l DbNodeId dbId = -1; bool found = false; if (!_syncDb->dbId(ReplicaSide::Local, localFileInfo.first, dbId, found)) { - LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::dbId for LocalNodeId=" << localFileInfo.first.c_str()); + LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::dbId for nodeId=" << localFileInfo.first.c_str()); return ExitCode::DbError; } if (found) { @@ -1429,14 +1429,13 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause } NodeId localNodeId; - localNodeId = snapshot(ReplicaSide::Local)->itemId(relativePath); - - if (localNodeId.empty()) { + if (localNodeId = snapshot(ReplicaSide::Local)->itemId(relativePath); localNodeId.empty()) { // The file does not exit yet on local FS, we do not have sufficient right on the parrent. LOGW_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L"isn't present local FS, blacklisting the parrent."); return handleAccessDeniedItem(relativePath.parent_path(), cause); } + Error error(syncDbId(), "", "", relativePath.extension() == SyncPath() ? NodeType::Directory : NodeType::File, relativePath, ConflictType::None, InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, cause); addError(error); @@ -1444,24 +1443,26 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause << L" is blacklisted temporarily because of access denied"); NodeId correspondingNodeId; - bool found; correspondingNodeId = snapshot(ReplicaSide::Remote)->itemId(relativePath); - if (correspondingNodeId.empty() && - !_syncDb->correspondingNodeId(ReplicaSide::Local, localNodeId, correspondingNodeId, found)) { + if (bool found; correspondingNodeId.empty() && + !_syncDb->correspondingNodeId(ReplicaSide::Local, localNodeId, correspondingNodeId, found)) { LOGW_SYNCPAL_WARN(_logger, L"Error in SyncDb::correspondingNodeId"); return {ExitCode::DbError, ExitCause::Unknown}; } + // Blacklist the item _tmpBlacklistManager->blacklistItem(localNodeId, relativePath, ReplicaSide::Local); - if (!correspondingNodeId.empty()) _tmpBlacklistManager->blacklistItem(correspondingNodeId, relativePath, ReplicaSide::Remote); - if (!updateTree(ReplicaSide::Local)->deleteNode(localNodeId)) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: " << Utility::formatSyncPath(relativePath)); } - if (!correspondingNodeId.empty() && !updateTree(ReplicaSide::Remote)->deleteNode(correspondingNodeId)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: " << Utility::formatSyncPath(relativePath)); + if (!correspondingNodeId.empty()) { + _tmpBlacklistManager->blacklistItem(correspondingNodeId, relativePath, ReplicaSide::Remote); + if (!updateTree(ReplicaSide::Remote)->deleteNode(correspondingNodeId)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: " << Utility::formatSyncPath(relativePath)); + } } + return ExitCode::Ok; } diff --git a/src/libsyncengine/syncpal/syncpalworker.cpp b/src/libsyncengine/syncpal/syncpalworker.cpp index 375fe4781..de1f52626 100644 --- a/src/libsyncengine/syncpal/syncpalworker.cpp +++ b/src/libsyncengine/syncpal/syncpalworker.cpp @@ -180,7 +180,7 @@ void SyncPalWorker::execute() { (stepWorkers[1] && workersExitCode[1] == ExitCode::SystemError && (stepWorkers[1]->exitCause() == ExitCause::NotEnoughDiskSpace || stepWorkers[1]->exitCause() == ExitCause::FileAccessError || - stepWorkers[0]->exitCause() == ExitCause::SyncDirAccesError))) { + stepWorkers[1]->exitCause() == ExitCause::SyncDirAccesError))) { // Exit without error exitCode = ExitCode::Ok; } else if ((stepWorkers[0] && workersExitCode[0] == ExitCode::UpdateRequired) || diff --git a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp index 3f908c9a0..0fa048035 100644 --- a/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/computefsoperationworker.cpp @@ -156,7 +156,7 @@ ExitCode ComputeFSOperationWorker::inferChangeFromDbNode(const ReplicaSide side, // Check that the file/directory really does not exist on replica bool isExcluded = false; if (const ExitInfo exitInfo = checkIfOkToDelete(side, dbPath, nodeId, isExcluded); !exitInfo) { - if (exitInfo.code() == ExitCode::SystemError && exitInfo.cause() == ExitCause::FileAccessError) { + if (exitInfo == ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)) { // Blacklist node if (ExitInfo exitInfo = _syncPal->handleAccessDeniedItem(dbPath); !exitInfo) { return exitInfo; @@ -821,7 +821,7 @@ ExitInfo ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const Syn return {ExitCode::SystemError, ExitCause::FileAccessError}; } setExitCause(ExitCause::Unknown); - return {ExitCode::SystemError, ExitCause::Unknown}; + return ExitCode::SystemError; } if (!existsWithSameId) return ExitCode::Ok; @@ -835,13 +835,13 @@ ExitInfo ComputeFSOperationWorker::checkIfOkToDelete(ReplicaSide side, const Syn LOGW_WARN(_logger, L"Error in ExclusionTemplateCache::isExcluded: " << Utility::formatIoError(absolutePath, ioError).c_str()); setExitCause(ExitCause::Unknown); - return {ExitCode::SystemError, ExitCause::Unknown}; + return ExitCode::SystemError; } if (ioError == IoError::AccessDenied) { LOGW_WARN(_logger, L"Item with " << Utility::formatSyncPath(absolutePath).c_str() << L" misses search permissions!"); setExitCause(ExitCause::FileAccessError); - return {ExitCode::SystemError, ExitCause::FileAccessError}; + return ExitCode::SystemError; } if (isExcluded) return ExitCode::Ok; diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index cb8fda2ab..df39ccbe4 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -289,7 +289,7 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listexists(nodeId); if (!itemExistsInSnapshot) { @@ -601,7 +601,7 @@ ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen LOGW_WARN(_logger, L"Error in IoHelper::getItemType: " << Utility::formatIoError(absoluteParentDirPath, itemType.ioError).c_str()); setExitCause(ExitCause::Unknown); - return {ExitCode::SystemError, ExitCause::Unknown}; + return ExitCode::SystemError; } if (itemType.ioError == IoError::NoSuchFileOrDirectory) { @@ -629,7 +629,7 @@ ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen LOGW_SYNCPAL_WARN(_logger, L"Error in IoHelper::getDirectoryIterator: " << Utility::formatIoError(absoluteParentDirPath, ioError).c_str()); setExitCause(ExitCause::Unknown); - return {ExitCode::SystemError, ExitCause::Unknown}; + return ExitCode::SystemError; } if (ioError == IoError::NoSuchFileOrDirectory) { @@ -646,7 +646,6 @@ ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen return {ExitCode::SystemError, ExitCause::SyncDirAccesError}; } - DirectoryEntry entry; bool endOfDirectory = false; while (dirIt.next(entry, endOfDirectory, ioError) && !endOfDirectory && ioError == IoError::Success) { @@ -664,35 +663,32 @@ ExitInfo LocalFileSystemObserverWorker::exploreDir(const SyncPath &absoluteParen bool toExclude = false; bool denyFullControl = false; bool isLink = false; - if (!toExclude) { - // Check if the directory entry is managed - bool isManaged = false; - IoError ioError = IoError::Success; - if (!Utility::checkIfDirEntryIsManaged(entry, isManaged, isLink, ioError)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in Utility::checkIfDirEntryIsManaged: " - << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disableRecursionPending(); - continue; - } - if (ioError == IoError::NoSuchFileOrDirectory) { - LOGW_SYNCPAL_DEBUG(_logger, L"Directory entry does not exist anymore: " - << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disableRecursionPending(); - continue; - } - if (ioError == IoError::AccessDenied) { - LOGW_SYNCPAL_DEBUG(_logger, - L"Directory misses search permission: " << Utility::formatSyncPath(absolutePath).c_str()); - dirIt.disableRecursionPending(); - sendAccessDeniedError(absolutePath); - continue; - } - if (!isManaged) { - LOGW_SYNCPAL_DEBUG(_logger, - L"Directory entry is not managed: " << Utility::formatSyncPath(absolutePath).c_str()); - toExclude = true; - } + // Check if the directory entry is managed + bool isManaged = false; + if (!Utility::checkIfDirEntryIsManaged(entry, isManaged, isLink, ioError)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in Utility::checkIfDirEntryIsManaged: " + << Utility::formatSyncPath(absolutePath).c_str()); + dirIt.disableRecursionPending(); + continue; + } + if (ioError == IoError::NoSuchFileOrDirectory) { + LOGW_SYNCPAL_DEBUG(_logger, + L"Directory entry does not exist anymore: " << Utility::formatSyncPath(absolutePath).c_str()); + dirIt.disableRecursionPending(); + continue; + } + if (ioError == IoError::AccessDenied) { + LOGW_SYNCPAL_DEBUG(_logger, + L"Directory misses search permission: " << Utility::formatSyncPath(absolutePath).c_str()); + dirIt.disableRecursionPending(); + sendAccessDeniedError(absolutePath); + continue; + } + + if (!isManaged) { + LOGW_SYNCPAL_DEBUG(_logger, L"Directory entry is not managed: " << Utility::formatSyncPath(absolutePath).c_str()); + toExclude = true; } if (!toExclude) { diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h index 65cf69751..0fd2c313a 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.h @@ -33,7 +33,6 @@ class LocalFileSystemObserverWorker : public FileSystemObserverWorker { void stop() override; void changesDetected(const std::list> &changes); - void errorDetected(ExitInfo exitInfo); virtual void forceUpdate() override; protected: diff --git a/src/server/appserver.cpp b/src/server/appserver.cpp index 86397c649..474c5ac50 100644 --- a/src/server/appserver.cpp +++ b/src/server/appserver.cpp @@ -3953,7 +3953,7 @@ void AppServer::addError(const Error &error) { JobManager::instance()->decreasePoolCapacity(); } - if (!ServerRequests::isAutoResolvedError(error) && errorAlreadyExists) { + if (!ServerRequests::isAutoResolvedError(error) && !errorAlreadyExists) { // Send error to sentry only for technical errors SentryUser sentryUser(user.email().c_str(), user.name().c_str(), std::to_string(user.userId()).c_str()); SentryHandler::instance()->captureMessage(SentryLevel::Warning, "AppServer::addError", error.errorString().c_str(), diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f823d153e..fdb0a02be 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,4 +28,4 @@ add_subdirectory(libcommonserver) add_subdirectory(libcommon) add_subdirectory(libparms) add_subdirectory(libsyncengine) -#add_subdirectory(server) +# add_subdirectory(server) From 6a99a627e8a0a000c44b56ecf797dab28e80f8f2 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 25 Oct 2024 15:32:42 +0200 Subject: [PATCH 30/56] Fix LFSO tests. --- .../testlocalfilesystemobserverworker.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libsyncengine/update_detection/file_system_observer/testlocalfilesystemobserverworker.cpp b/test/libsyncengine/update_detection/file_system_observer/testlocalfilesystemobserverworker.cpp index 846d0fed9..78e5fd2b0 100644 --- a/test/libsyncengine/update_detection/file_system_observer/testlocalfilesystemobserverworker.cpp +++ b/test/libsyncengine/update_detection/file_system_observer/testlocalfilesystemobserverworker.cpp @@ -25,7 +25,7 @@ #else #include "update_detection/file_system_observer/localfilesystemobserverworker_unix.h" #endif - +#include "syncpal/tmpblacklistmanager.h" #include "libcommonserver/io/filestat.h" #include "libcommonserver/io/iohelper.h" #include "libcommonserver/utility/utility.h" @@ -82,7 +82,7 @@ void TestLocalFileSystemObserverWorker::setUp() { _syncPal = std::make_shared(syncDbPath, KDRIVE_VERSION_STRING, true); _syncPal->syncDb()->setAutoDelete(true); _syncPal->setLocalPath(_rootFolderPath); - + _syncPal->_tmpBlacklistManager = std::make_shared(_syncPal); #if defined(_WIN32) _syncPal->_localFSObserverWorker = std::shared_ptr( new LocalFileSystemObserverWorker_win(_syncPal, "Local File System Observer", "LFSO")); From 96c40aa48c37420579e16f26ef87c8e866f2479f Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 25 Oct 2024 15:33:32 +0200 Subject: [PATCH 31/56] Auto merge of AcessError with same parent. --- src/libparms/db/error.cpp | 7 ------ src/libsyncengine/syncpal/syncpal.cpp | 7 +++--- src/server/appserver.cpp | 34 ++++++++++++++++++++++++--- src/server/appserver.h | 2 +- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/libparms/db/error.cpp b/src/libparms/db/error.cpp index 6d5770782..e7d29d2d4 100644 --- a/src/libparms/db/error.cpp +++ b/src/libparms/db/error.cpp @@ -94,13 +94,6 @@ bool Error::isSimilarTo(const Error &other) const { return (_exitCode == other.exitCode()) && (_exitCause == other.exitCause()) && (_workerName == other.workerName()); } case ErrorLevel::Node: { - if (_exitCode == ExitCode::SystemError && _exitCause == ExitCause::FileAccessError && - other.exitCode() == ExitCode::SystemError && other.exitCause() == ExitCause::FileAccessError && - (Utility::isSameOrChildPath(_path.lexically_normal(), other.path().lexically_normal()) || - Utility::isSameOrChildPath(other.path().lexically_normal(), _path.lexically_normal()))) { - return true; - } - return (_conflictType == other.conflictType()) && (_inconsistencyType == other.inconsistencyType()) && (_cancelType == other.cancelType()) && (_path == other.path() && _destinationPath == other.destinationPath()); } diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 7d451e245..26ad7ddac 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1427,6 +1427,9 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause LOGW_SYNCPAL_WARN(_logger, L"Access error on root folder"); return ExitInfo(ExitCode::SystemError, ExitCause::SyncDirAccesError); } + Error error(syncDbId(), "", "", relativePath.extension() == SyncPath() ? NodeType::Directory : NodeType::File, relativePath, + ConflictType::None, InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, cause); + addError(error); NodeId localNodeId; if (localNodeId = snapshot(ReplicaSide::Local)->itemId(relativePath); localNodeId.empty()) { @@ -1436,9 +1439,7 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause return handleAccessDeniedItem(relativePath.parent_path(), cause); } - Error error(syncDbId(), "", "", relativePath.extension() == SyncPath() ? NodeType::Directory : NodeType::File, relativePath, - ConflictType::None, InconsistencyType::None, CancelType::None, "", ExitCode::SystemError, cause); - addError(error); + LOGW_SYNCPAL_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(localNodeId) << L" is blacklisted temporarily because of access denied"); diff --git a/src/server/appserver.cpp b/src/server/appserver.cpp index 474c5ac50..742f0b720 100644 --- a/src/server/appserver.cpp +++ b/src/server/appserver.cpp @@ -3883,9 +3883,6 @@ void AppServer::addError(const Error &error) { if (!existingError.isSimilarTo(error)) continue; // Update existing error time existingError.setTime(error.time()); - auto shorterPath = - (existingError.path().string().size() > error.path().string().size()) ? error.path() : existingError.path(); - existingError.setPath(shorterPath); bool found = false; if (!ParmsDb::instance()->updateError(existingError, found)) { LOG_WARN(Log::instance()->getLogger(), "Error in ParmsDb::updateError"); @@ -3904,6 +3901,8 @@ void AppServer::addError(const Error &error) { LOG_WARN(Log::instance()->getLogger(), "Error in ParmsDb::insertError"); return; } + if (!errorAlreadyExists) errorList.push_back(error); + User user; if (error.syncDbId() && ServerRequests::getUserFromSyncDbId(error.syncDbId(), user) != ExitCode::Ok) { @@ -3951,6 +3950,35 @@ void AppServer::addError(const Error &error) { // Decrease JobManager pool capacity JobManager::instance()->decreasePoolCapacity(); + } else if (error.exitCode() == ExitCode::SystemError && error.exitCause() == ExitCause::FileAccessError) { + // Remove child errors + std::unordered_set toBeRemovedErrorIds; + for (const Error &parentError : errorList) { + for (const Error &childError : errorList) { + if (Utility::isSameOrChildPath(childError.path(), parentError.path()) && + childError.dbId() != parentError.dbId()) { + toBeRemovedErrorIds.insert(childError.dbId()); + } + } + } + for (int errorId : toBeRemovedErrorIds) { + bool found = false; + if (!ParmsDb::instance()->deleteError(errorId, found)) { + LOG_WARN(Log::instance()->getLogger(), "Error in ParmsDb::deleteError"); + return; + } + if (!found) { + LOG_WARN(Log::instance()->getLogger(), "Error not found in Error table for dbId=" << errorId); + return; + } + } + if (toBeRemovedErrorIds.size() > 0) + if (ServerRequests::isDisplayableError(error)) { + // Notify the client + sendErrorsCleared(error.syncDbId()); + } + + } if (!ServerRequests::isAutoResolvedError(error) && !errorAlreadyExists) { diff --git a/src/server/appserver.h b/src/server/appserver.h index 28e8c29a7..ae99fad95 100644 --- a/src/server/appserver.h +++ b/src/server/appserver.h @@ -176,7 +176,7 @@ class AppServer : public SharedTools::QtSingleApplication { void sendSyncDeletionFailed(int syncDbId); void sendGetFolderSizeCompleted(const QString &nodeId, qint64 size); void sendNewBigFolder(int syncDbId, const QString &path); - void sendErrorsCleared(int syncDbId); + static void sendErrorsCleared(int syncDbId); void sendQuit(); // Ask client to quit // See types.h -> AppStateKey for the possible values of status From 62d596a4803b637d5086c00c5adf7541444b5ca3 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 25 Oct 2024 15:51:12 +0200 Subject: [PATCH 32/56] Fix PlatformInconsistencyCheckerWorker --- .../platforminconsistencycheckerworker.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp index dcf56d4df..4a299d160 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp @@ -131,9 +131,9 @@ ExitCode PlatformInconsistencyCheckerWorker::checkLocalTree(std::shared_ptrchildren().begin(); - for (; childIt != localNode->children().end(); childIt++) { + auto childCopy = localNode->children(); + auto childIt = childCopy.begin(); + for (; childIt != childCopy.end(); childIt++) { if (stopAsked()) { return ExitCode::Ok; } @@ -225,8 +225,9 @@ bool PlatformInconsistencyCheckerWorker::checkPathAndName(std::shared_ptr void PlatformInconsistencyCheckerWorker::checkNameClashAgainstSiblings(const std::shared_ptr &remoteParentNode) { #if defined(__APPLE__) || defined(_WIN32) std::unordered_map> processedNodesByName; // key: lowercase name - auto it = remoteParentNode->children().begin(); - for (; it != remoteParentNode->children().end(); it++) { + auto childrenCopy = remoteParentNode->children(); + auto it = childrenCopy.begin(); + for (; it != childrenCopy.end(); it++) { if (stopAsked()) { return; } From 62e0c9d0132d35c90b7faec53bf7186e6bf3965a Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 25 Oct 2024 16:07:07 +0200 Subject: [PATCH 33/56] Remove old comments --- src/libsyncengine/propagation/executor/executorworker.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 9fa6f2a23..ac8f5bccd 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -695,7 +695,6 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr &job) { // 1. If omit-flag is False, propagate the file to replicaY, replacing the existing one. if (syncOp->targetSide() == ReplicaSide::Local) { From 10952217f1910605ad922e6548a8e3297a620844 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Fri, 25 Oct 2024 17:16:16 +0200 Subject: [PATCH 34/56] Rename handleExecutorError --- .../propagation/executor/executorworker.cpp | 19 ++++++++++--------- .../propagation/executor/executorworker.h | 4 +++- test/server/workers/testworkers.cpp | 12 ++++++------ 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index ac8f5bccd..4465014fc 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -148,18 +148,19 @@ void ExecutorWorker::execute() { } } + // If an operation fails but is correctly handled by handleExecutorError, execution can proceed. if (executorExitInfo.cause() == ExitCause::OperationCanceled) { setProgressComplete(syncOp, SyncFileStatus::Error); continue; } if (!executorExitInfo) { - executorExitInfo = handleOpsExecutionError(syncOp, executorExitInfo); - if (!executorExitInfo) { + executorExitInfo = handleExecutorError(syncOp, executorExitInfo); + if (!executorExitInfo) { // If the error is not handled, stop the execution increaseErrorCount(syncOp); cancelAllOngoingJobs(); break; - } else { + } else { // If the error is handled, continue the execution setProgressComplete(syncOp, SyncFileStatus::Error); continue; } @@ -1706,7 +1707,7 @@ ExitInfo ExecutorWorker::handleFinishedJob(std::shared_ptr job, Syn L"Error in handleForbiddenAction for item: " << Utility::formatSyncPath(relativeLocalPath)); return exitInfo; } - } else if (!handleOpsExecutionError(syncOp, {job->exitCode(), job->exitCause()})) { + } else if (!handleExecutorError(syncOp, {job->exitCode(), job->exitCause()})) { // Cancel all queued jobs LOGW_SYNCPAL_WARN(_logger, L"Cancelling jobs. exit code: " << job->exitCode() << L" exit cause: " << job->exitCause()); @@ -2529,10 +2530,10 @@ ExitInfo ExecutorWorker::getFileSize(const SyncPath &path, uint64_t &size) { return ExitCode::Ok; } -ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { - assert((syncOp && !opsExitInfo) && "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); +ExitInfo ExecutorWorker::handleExecutorError(SyncOpPtr syncOp, ExitInfo opsExitInfo) { + assert((syncOp && !opsExitInfo) && "syncOp is nullptr in ExecutorWorker::handleExecutorError"); if (!syncOp) { - LOG_WARN(_logger, "syncOp is nullptr in ExecutorWorker::handleOpsExecutionError"); + LOG_WARN(_logger, "syncOp is nullptr in ExecutorWorker::handleExecutorError"); return ExitCode::DataError; } if (opsExitInfo) { @@ -2554,7 +2555,7 @@ ExitInfo ExecutorWorker::handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsE } }; LOG_WARN(_logger, - "Unhandled error in ExecutorWorker::handleOpsExecutionError: " << opsExitInfo.code() << " " << opsExitInfo.cause()); + "Unhandled error in ExecutorWorker::handleExecutorError: " << opsExitInfo.code() << " " << opsExitInfo.cause()); return opsExitInfo; } @@ -2630,7 +2631,7 @@ ExitInfo ExecutorWorker::handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo o absoluteLocalPath, PlatformInconsistencyCheckerUtility::SuffixType::Blacklisted); !exitInfo) { LOGW_WARN(_logger, L"Failed to blacklist file: " << Utility::formatSyncPath(absoluteLocalPath) << L" ExitCode::" - << exitInfo.code() << L" ExitCause::" << exitInfo.cause()); + << exitInfo.code() << L" ExitCause::" << exitInfo.cause()); return ExitCode::DataError; // The synchronization will be re-started. } diff --git a/src/libsyncengine/propagation/executor/executorworker.h b/src/libsyncengine/propagation/executor/executorworker.h index 1f28f5cbb..29f3dab78 100644 --- a/src/libsyncengine/propagation/executor/executorworker.h +++ b/src/libsyncengine/propagation/executor/executorworker.h @@ -143,7 +143,9 @@ class ExecutorWorker : public OperationProcessor { void setProgressComplete(const SyncOpPtr syncOp, SyncFileStatus status); - ExitInfo handleOpsExecutionError(SyncOpPtr syncOp, ExitInfo opsExitInfo); + // This methode will return ExitCode::Ok if the error is safely managed and the executor can continue. Else, it will + // return opsExitInfo. + ExitInfo handleExecutorError(SyncOpPtr syncOp, ExitInfo opsExitInfo); ExitInfo handleOpsFileAccessError(SyncOpPtr syncOp, ExitInfo opsExitInfo); ExitInfo handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo opsExitInfo); diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index 582fae4ba..a0254f8e5 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -298,12 +298,12 @@ void TestWorkers::testConvertToPlaceholder() { void TestWorkers::testHandleOpsExecutionError() { SyncOpPtr opPtr = std::make_shared(); - _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::Unknown)); - _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::InvalidSnapshot)); - _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::DbEntryNotFound)); - _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::Unknown)); - _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)); - _syncPal->_executorWorker->handleOpsExecutionError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::DriveNotRenew)); + _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::Unknown)); + _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::InvalidSnapshot)); + _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::DbEntryNotFound)); + _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::Unknown)); + _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)); + _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::DriveNotRenew)); } From 0b7b187d5fc3ddb129735eeda3a289261c886e7c Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 28 Oct 2024 10:45:54 +0100 Subject: [PATCH 35/56] Correctly handle file access errors when write access is denied. --- .../propagation/executor/executorworker.cpp | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 4465014fc..cce1ded6e 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -729,6 +729,21 @@ ExitInfo ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath) { << Utility::formatSyncPath(relativeLocalPath).c_str()); return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } + + bool exists = false; + IoError ioError = IoError::Success; + SyncPath absoluteLocalPath = _syncPal->localPath() / relativeLocalPath; + if (!IoHelper::checkIfPathExists(absoluteLocalPath, exists, ioError)) { + LOGW_SYNCPAL_WARN(_logger, + L"Error in IoHelper::checkIfPathExists: " << Utility::formatIoError(absoluteLocalPath, ioError)); + return ExitCode::SystemError; + } + + if (exists) { + LOGW_WARN(_logger, L"Item already exists: " << Utility::formatSyncPath(absoluteLocalPath)); + return {ExitCode::DataError, ExitCause::InvalidSnapshot}; + } + if (!_syncPal->vfsCreatePlaceholder(relativeLocalPath, syncItem)) { // TODO: vfs functions should output an ioError parameter // Check if the item already exists on local replica @@ -802,8 +817,10 @@ ExitInfo ExecutorWorker::processCreateOrConvertToPlaceholderError(const SyncPath return {ExitCode::SystemError, ExitCause::FileAccessError}; } - if ((create && exists) || (!create && !exists)) { - return {ExitCode::DataError, ExitCause::InvalidSnapshot}; + if (create && exists){ + return {ExitCode::SystemError, ExitCause::FileAccessError}; + }else if (!create && !exists){ + return {ExitCode::DataError, ExitCause::FileAlreadyExist}; } if (create) { From da844e86f7878e113e9f0e9c0bfec3aed38cb946 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 28 Oct 2024 15:23:04 +0100 Subject: [PATCH 36/56] Fix wrong comment. --- src/libsyncengine/update_detection/update_detector/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyncengine/update_detection/update_detector/node.cpp b/src/libsyncengine/update_detection/update_detector/node.cpp index 18cc1d69f..1bcdf81ba 100644 --- a/src/libsyncengine/update_detection/update_detector/node.cpp +++ b/src/libsyncengine/update_detection/update_detector/node.cpp @@ -183,7 +183,7 @@ SyncPath Node::getPath() const { bool Node::isParentOf(std::shared_ptr potentialChild) const { if (!potentialChild) return false; // `parentNode` is the root node, - if (potentialChild->id().has_value() && potentialChild->id() == _id) return false; // `parentNode` is the root node + if (potentialChild->id().has_value() && potentialChild->id() == _id) return false; // potentialChild cannot be its own parent while (potentialChild) { if (!potentialChild->id().has_value()) { LOG_ERROR(Log::instance()->getLogger(), "Node::isParentOf Node has no id"); From 5da0abe719e90588238ed8a6cbc5c2d9a36dfb5c Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 28 Oct 2024 15:56:30 +0100 Subject: [PATCH 37/56] Remove useless .c_str(), adapt some comment and add Utility::formatIoError. --- src/libcommon/utility/types.h | 8 ++-- .../jobs/network/API_v2/downloadjob.cpp | 2 +- .../jobs/network/API_v2/movejob.cpp | 2 +- .../upload_session/abstractuploadsession.cpp | 2 +- src/libsyncengine/syncpal/syncpal.cpp | 10 ++--- .../localfilesystemobserverworker.cpp | 38 +++++++++---------- 6 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index 5b9541b7e..967542d3c 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -116,18 +116,18 @@ template // Any enum class concept EnumClass = std::is_enum_v; template // Any enum class that can be converted to (and from) int -concept IntableEnum = EnumClass && std::is_convertible_v, int>; +concept IntegralEnum = EnumClass && std::is_convertible_v, int>; template // Any enum class that can be printed (with enumClassToString) concept PrintableEnum = EnumClass && requires(C e) { toString(e); }; // Converters -template +template inline constexpr int toInt(C e) { return static_cast(e); } -template +template inline constexpr C fromInt(int e) { return static_cast(e); } @@ -499,7 +499,7 @@ using AppStateValue = std::variant; // Concepts template // Any enum class we want to allow bitwise operations (OperationType & InconsistencyType) -concept AllowBitWiseOpEnum = IntableEnum && (std::is_same_v || std::is_same_v); +concept AllowBitWiseOpEnum = IntegralEnum && (std::is_same_v || std::is_same_v); // Operators template diff --git a/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp b/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp index 3c75d2458..32edafed0 100644 --- a/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/downloadjob.cpp @@ -565,7 +565,7 @@ bool DownloadJob::moveTmpFile(const SyncPath &path, bool &restartSync) { return false; } if (ioError == IoError::AccessDenied) { - LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_localpath.parent_path()).c_str()); + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_localpath.parent_path())); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; diff --git a/src/libsyncengine/jobs/network/API_v2/movejob.cpp b/src/libsyncengine/jobs/network/API_v2/movejob.cpp index cfa503048..61d42ebea 100644 --- a/src/libsyncengine/jobs/network/API_v2/movejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/movejob.cpp @@ -62,7 +62,7 @@ bool MoveJob::canRun() { return false; } if (ioError == IoError::AccessDenied) { - LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_destFilepath).c_str()); + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_destFilepath)); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp b/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp index f6f3c6e9c..05fd9fde0 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp @@ -172,7 +172,7 @@ bool AbstractUploadSession::canRun() { return false; } if (ioError == IoError::AccessDenied) { - LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_filePath).c_str()); + LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_filePath)); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 26ad7ddac..5ef12bb02 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1424,7 +1424,7 @@ void SyncPal::removeItemFromTmpBlacklist(const SyncPath &relativePath) { ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause cause) { if (relativePath.empty()) { - LOGW_SYNCPAL_WARN(_logger, L"Access error on root folder"); + LOG_SYNCPAL_WARN(_logger, "Access error on root folder"); return ExitInfo(ExitCode::SystemError, ExitCause::SyncDirAccesError); } Error error(syncDbId(), "", "", relativePath.extension() == SyncPath() ? NodeType::Directory : NodeType::File, relativePath, @@ -1433,21 +1433,21 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause NodeId localNodeId; if (localNodeId = snapshot(ReplicaSide::Local)->itemId(relativePath); localNodeId.empty()) { - // The file does not exit yet on local FS, we do not have sufficient right on the parrent. + // The file does not exit yet on local file system, or we do not have sufficient right on a parent folder. LOGW_DEBUG(_logger, - L"Item " << Utility::formatSyncPath(relativePath) << L"isn't present local FS, blacklisting the parrent."); + L"Item " << Utility::formatSyncPath(relativePath) << L"is not present local file system, blacklisting the parent item."); return handleAccessDeniedItem(relativePath.parent_path(), cause); } LOGW_SYNCPAL_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(localNodeId) - << L" is blacklisted temporarily because of access denied"); + << L" is blacklisted temporarily because of denied access."); NodeId correspondingNodeId; correspondingNodeId = snapshot(ReplicaSide::Remote)->itemId(relativePath); if (bool found; correspondingNodeId.empty() && !_syncDb->correspondingNodeId(ReplicaSide::Local, localNodeId, correspondingNodeId, found)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in SyncDb::correspondingNodeId"); + LOG_SYNCPAL_WARN(_logger, "Error in SyncDb::correspondingNodeId"); return {ExitCode::DbError, ExitCause::Unknown}; } diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index df39ccbe4..058bc6150 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -139,15 +139,15 @@ void LocalFileSystemObserverWorker::changesDetected(const std::list Date: Mon, 28 Oct 2024 16:16:09 +0100 Subject: [PATCH 38/56] Fix wrong variable name. --- src/libsyncengine/syncpal/tmpblacklistmanager.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp index 1a5a6a656..23f86fb12 100644 --- a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp +++ b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp @@ -143,7 +143,7 @@ void TmpBlacklistManager::removeItemFromTmpBlacklist(const NodeId &nodeId, Repli } void TmpBlacklistManager::removeItemFromTmpBlacklist(const SyncPath &relativePath) { - NodeId removedId; + NodeId remotedId; NodeId localId; // Find the node id of the item to be removed @@ -156,7 +156,7 @@ void TmpBlacklistManager::removeItemFromTmpBlacklist(const SyncPath &relativePat for (const auto &[nodeId, tmpInfo]: _remoteErrors) { if (Utility::isSameOrChildPath(tmpInfo.path, relativePath)) { - removedId = nodeId; + remotedId = nodeId; break; } } @@ -165,8 +165,8 @@ void TmpBlacklistManager::removeItemFromTmpBlacklist(const SyncPath &relativePat removeItemFromTmpBlacklist(localId, ReplicaSide::Local); } - if (!removedId.empty()) { - removeItemFromTmpBlacklist(removedId, ReplicaSide::Remote); + if (!remotedId.empty()) { + removeItemFromTmpBlacklist(remotedId, ReplicaSide::Remote); } } From d5e4301aa9b075122b99f7c58a7d7eb6a34c650e Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Mon, 28 Oct 2024 16:25:24 +0100 Subject: [PATCH 39/56] Print sync dir in case of read issue on Linux. --- .../file_system_observer/folderwatcher_linux.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp index 75350b513..9157f4df8 100644 --- a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp @@ -131,7 +131,7 @@ bool FolderWatcher_linux::findSubFolders(const SyncPath &dir, std::list Date: Mon, 28 Oct 2024 17:56:31 +0100 Subject: [PATCH 40/56] Add TestWorkers::testRemoveDependentOps() --- .../propagation/executor/executorworker.cpp | 4 +- .../reconciliation/syncoperation.cpp | 2 +- test/CMakeLists.txt | 2 +- test/server/workers/testworkers.cpp | 118 ++++++++++++++++-- test/server/workers/testworkers.h | 6 +- 5 files changed, 116 insertions(+), 16 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index cce1ded6e..424003d74 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -2681,7 +2681,7 @@ ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { SyncName nodeName = localNode2 ? localNode2->name() : SyncName(); if (nodeName.empty()) nodeName = remoteNode2 ? remoteNode2->name() : SyncName(); - if (localNode && localNode2 && (*localNode->id() == *localNode2->id() || localNode->isParentOf(localNode2))) { + if (localNode && localNode2 && (localNode->isParentOf(localNode2))) { LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) << L" because it depends on " << syncOp->type() << L" operation on " << SyncName2WStr(localNode->name()) << L" wich failed."); @@ -2689,7 +2689,7 @@ ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { continue; } - if (remoteNode && remoteNode2 && (*remoteNode->id() == *remoteNode2->id() || remoteNode->isParentOf(remoteNode2))) { + if (remoteNode && remoteNode2 && (remoteNode->isParentOf(remoteNode2))) { LOGW_SYNCPAL_DEBUG(_logger, L"Removing " << syncOp2->type() << L" operation on " << SyncName2WStr(nodeName) << L" because it depends on " << syncOp->type() << L" operation on " << SyncName2WStr(remoteNode->name()) << L"wich failed."); diff --git a/src/libsyncengine/reconciliation/syncoperation.cpp b/src/libsyncengine/reconciliation/syncoperation.cpp index 786f9874b..d94a6f6e3 100644 --- a/src/libsyncengine/reconciliation/syncoperation.cpp +++ b/src/libsyncengine/reconciliation/syncoperation.cpp @@ -108,7 +108,7 @@ void SyncOperationList::operator=(const SyncOperationList &other) { } void SyncOperationList::getMapIndexToOp(std::unordered_map &map, - OperationType typeFilter /*= OperationType::Unknown*/) { + OperationType typeFilter /*= OperationType::None*/) { int index = 0; for (const auto &opId: _opSortedList) { SyncOpPtr syncOp = getOp(opId); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fdb0a02be..de1a8d327 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,4 +28,4 @@ add_subdirectory(libcommonserver) add_subdirectory(libcommon) add_subdirectory(libparms) add_subdirectory(libsyncengine) -# add_subdirectory(server) +add_subdirectory(server) diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index a0254f8e5..313a34f21 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -61,6 +61,16 @@ bool TestWorkers::setPinState(int syncDbId, const SyncPath &relativeLocalPath, P return true; } +bool TestWorkers::opsExist(SyncOpPtr op) { + for (const auto &opId: _syncPal->_executorWorker->_opList) { + if (_syncPal->_syncOps->getOp(opId) == op) { + return true; + } + } + + return false; +} + void TestWorkers::setUp() { _logger = Log::instance()->getLogger(); @@ -124,7 +134,7 @@ void TestWorkers::setUp() { #if defined(__APPLE__) _vfsPtr = std::unique_ptr(new VfsMac(vfsSetupParams)); #elif defined(_WIN32) - _vfsPtr = std::unique_ptr(new VfsWin(vfsSetupParams, nullptr)); + //_vfsPtr = std::unique_ptr(new VfsWin(vfsSetupParams, nullptr)); #else _vfsPtr = std::unique_ptr(new VfsOff(vfsSetupParams)); #endif @@ -296,16 +306,106 @@ void TestWorkers::testConvertToPlaceholder() { } } -void TestWorkers::testHandleOpsExecutionError() { - SyncOpPtr opPtr = std::make_shared(); - _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::Unknown)); - _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::InvalidSnapshot)); - _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::DataError, ExitCause::DbEntryNotFound)); - _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::Unknown)); - _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::FileAccessError)); - _syncPal->_executorWorker->handleExecutorError(opPtr, ExitInfo(ExitCode::SystemError, ExitCause::DriveNotRenew)); +void TestWorkers::testRemoveDependentOps() { + { + // Nested create. + _syncPal->_syncOps->clear(); + + auto op1Create = std::make_shared(); // a/ is created. + auto op2Create = std::make_shared(); // a/b/ is created. + auto op3Create = std::make_shared(); // a/b/c/ is created. + auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, + OperationType::None, "idAn1", 1234, 1234, 1, nullptr); + auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, + OperationType::None, "idAn2", 1234, 1234, 1, nullptr); + auto affectedNode3 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an3"), NodeType::Unknown, + OperationType::None, "idAn3", 1234, 1234, 1, nullptr); + auto affectedNode4 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an4"), NodeType::Unknown, + OperationType::None, "idAn4", 1234, 1234, 1, nullptr); + + affectedNode4->setParentNode(affectedNode3); + affectedNode3->setParentNode(affectedNode2); + affectedNode2->setParentNode(affectedNode1); + + op1Create->setAffectedNode(affectedNode2); + op2Create->setAffectedNode(affectedNode3); + op3Create->setAffectedNode(affectedNode4); + + _syncPal->_syncOps->pushOp(op1Create); + _syncPal->_syncOps->pushOp(op2Create); + _syncPal->_syncOps->pushOp(op3Create); + + _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); + _syncPal->_executorWorker->removeDependentOps(op1Create); // op1Create failed, we should remove op2Create and op3Create. + + CPPUNIT_ASSERT(opsExist(op1Create)); + CPPUNIT_ASSERT(!opsExist(op2Create)); + CPPUNIT_ASSERT(!opsExist(op3Create)); + } + + { + // Nested Move. + _syncPal->_syncOps->clear(); + + auto op1Create = std::make_shared(); // a/ is created. + auto op2Move = std::make_shared(); // b/ is moved to a/b/. + + auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, + OperationType::None, "idAn1", 1234, 1234, 1, nullptr); // a/ + auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, + OperationType::None, "idAn2", 1234, 1234, 1, nullptr); // a/b + + auto correspondingNode2 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn2"), NodeType::Unknown, + OperationType::None, "idCn2", 1234, 1234, 1, nullptr); // b/ + + affectedNode2->setParentNode(affectedNode1); + op1Create->setAffectedNode(affectedNode1); + op2Move->setAffectedNode(affectedNode2); + op2Move->setCorrespondingNode(correspondingNode2); + _syncPal->_syncOps->pushOp(op1Create); + _syncPal->_syncOps->pushOp(op2Move); + + _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); + _syncPal->_executorWorker->removeDependentOps(op1Create); // op2Move failed, we should remove op2Edit. + CPPUNIT_ASSERT(opsExist(op1Create)); + CPPUNIT_ASSERT(!opsExist(op2Move)); + } + + { + // Double Move. + _syncPal->_syncOps->clear(); + + auto op1Move = std::make_shared(); // a is moved to b. + auto op2Move = std::make_shared(); // y is moved to b/y. + + auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, + OperationType::None, "idAn1", 1234, 1234, 1, nullptr); // b + auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, + OperationType::None, "idAn2", 1234, 1234, 1, nullptr); // b/y + + auto correspondingNode1 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn1"), NodeType::Unknown, + OperationType::None, "idCn1", 1234, 1234, 1, nullptr); // a + auto correspondingNode2 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn2"), NodeType::Unknown, + OperationType::None, "idCn2", 1234, 1234, 1, nullptr); // y + + affectedNode2->setParentNode(affectedNode1); + + op1Move->setAffectedNode(affectedNode1); + op1Move->setCorrespondingNode(correspondingNode1); + + op2Move->setAffectedNode(affectedNode2); + op2Move->setCorrespondingNode(correspondingNode2); + + _syncPal->_syncOps->pushOp(op1Move); + _syncPal->_syncOps->pushOp(op2Move); + + _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); + _syncPal->_executorWorker->removeDependentOps(op1Move); // op2Move failed, we should remove op2Edit. + CPPUNIT_ASSERT(opsExist(op1Move)); + CPPUNIT_ASSERT(!opsExist(op2Move)); + } } diff --git a/test/server/workers/testworkers.h b/test/server/workers/testworkers.h index 3c90fe80e..b1419c28e 100644 --- a/test/server/workers/testworkers.h +++ b/test/server/workers/testworkers.h @@ -35,7 +35,7 @@ class TestWorkers : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(TestWorkers); CPPUNIT_TEST(testCreatePlaceholder); CPPUNIT_TEST(testConvertToPlaceholder); - CPPUNIT_TEST(testHandleOpsExecutionError); + CPPUNIT_TEST(testRemoveDependentOps); CPPUNIT_TEST_SUITE_END(); public: @@ -43,13 +43,13 @@ class TestWorkers : public CppUnit::TestFixture { void tearDown() override; void testCreatePlaceholder(); void testConvertToPlaceholder(); - void testHandleOpsExecutionError(); - + void testRemoveDependentOps(); protected: static bool createPlaceholder(int syncDbId, const SyncPath &relativeLocalPath, const SyncFileItem &item); static bool convertToPlaceholder(int syncDbId, const SyncPath &relativeLocalPath, const SyncFileItem &item); static bool setPinState(int syncDbId, const SyncPath &relativeLocalPath, PinState pinState); + bool opsExist(SyncOpPtr op); log4cplus::Logger _logger; std::shared_ptr _syncPal; Sync _sync; From 9eac37b949289b657512e9d96518be6c685d4d53 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Tue, 29 Oct 2024 08:46:12 +0100 Subject: [PATCH 41/56] PR suggestion on a log. --- src/libsyncengine/jobs/network/API_v2/movejob.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyncengine/jobs/network/API_v2/movejob.cpp b/src/libsyncengine/jobs/network/API_v2/movejob.cpp index 61d42ebea..5cc9aed68 100644 --- a/src/libsyncengine/jobs/network/API_v2/movejob.cpp +++ b/src/libsyncengine/jobs/network/API_v2/movejob.cpp @@ -62,7 +62,7 @@ bool MoveJob::canRun() { return false; } if (ioError == IoError::AccessDenied) { - LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_destFilepath)); + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_destFilepath)); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; From e7b6ead5af36ce82c69044569d9ab44b376fbbba Mon Sep 17 00:00:00 2001 From: Herve Eruam <82284536+herve-er@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:50:43 +0100 Subject: [PATCH 42/56] Update src/libsyncengine/update_detection/update_detector/node.cpp Co-authored-by: Luc Guyot <162997198+luc-guyot-infomaniak@users.noreply.github.com> --- src/libsyncengine/update_detection/update_detector/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyncengine/update_detection/update_detector/node.cpp b/src/libsyncengine/update_detection/update_detector/node.cpp index 1bcdf81ba..a5c1344fb 100644 --- a/src/libsyncengine/update_detection/update_detector/node.cpp +++ b/src/libsyncengine/update_detection/update_detector/node.cpp @@ -186,7 +186,7 @@ bool Node::isParentOf(std::shared_ptr potentialChild) const { if (potentialChild->id().has_value() && potentialChild->id() == _id) return false; // potentialChild cannot be its own parent while (potentialChild) { if (!potentialChild->id().has_value()) { - LOG_ERROR(Log::instance()->getLogger(), "Node::isParentOf Node has no id"); + LOG_ERROR(Log::instance()->getLogger(), "Error in Node::isParentOf: Node has no id."); assert(false && "Node has no id"); return false; } From a8e836ae45cd8e4c624d4064af6a8ccfa510fa0c Mon Sep 17 00:00:00 2001 From: Herve Eruam <82284536+herve-er@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:51:09 +0100 Subject: [PATCH 43/56] Update src/server/appserver.cpp Co-authored-by: Luc Guyot <162997198+luc-guyot-infomaniak@users.noreply.github.com> --- src/server/appserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/appserver.cpp b/src/server/appserver.cpp index 742f0b720..2fe6c3d19 100644 --- a/src/server/appserver.cpp +++ b/src/server/appserver.cpp @@ -3972,7 +3972,7 @@ void AppServer::addError(const Error &error) { return; } } - if (toBeRemovedErrorIds.size() > 0) + if (!toBeRemovedErrorIds.empty()) if (ServerRequests::isDisplayableError(error)) { // Notify the client sendErrorsCleared(error.syncDbId()); From 761d6b886793c3d6177f546ce2a9d05e63b8d83d Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Tue, 29 Oct 2024 08:51:57 +0100 Subject: [PATCH 44/56] PR suggestion on some log. --- .../network/API_v2/upload_session/abstractuploadsession.cpp | 2 +- src/libsyncengine/syncpal/syncpal.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp b/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp index 05fd9fde0..a5e147428 100644 --- a/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp +++ b/src/libsyncengine/jobs/network/API_v2/upload_session/abstractuploadsession.cpp @@ -172,7 +172,7 @@ bool AbstractUploadSession::canRun() { return false; } if (ioError == IoError::AccessDenied) { - LOGW_WARN(_logger, L"Access denied to " << Path2WStr(_filePath)); + LOGW_WARN(_logger, L"Access denied to " << Utility::formatSyncPath(_filePath)); _exitCode = ExitCode::SystemError; _exitCause = ExitCause::FileAccessError; return false; diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 5ef12bb02..4785d7f90 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1441,7 +1441,7 @@ ExitInfo SyncPal::handleAccessDeniedItem(const SyncPath &relativePath, ExitCause LOGW_SYNCPAL_DEBUG(_logger, L"Item " << Utility::formatSyncPath(relativePath) << L" (NodeId: " << Utility::s2ws(localNodeId) - << L" is blacklisted temporarily because of denied access."); + << L" is blacklisted temporarily because of a denied access."); NodeId correspondingNodeId; correspondingNodeId = snapshot(ReplicaSide::Remote)->itemId(relativePath); From 5ffe00b5b64a493c6782111c25e79d7d7e126a00 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Tue, 29 Oct 2024 09:14:31 +0100 Subject: [PATCH 45/56] Move testRemoveDependentOps to testExecutorWorker and desacivate testWorkers. --- src/server/appserver.cpp | 14 +-- test/CMakeLists.txt | 2 +- .../executor/testexecutorworker.cpp | 113 +++++++++++++++++ .../propagation/executor/testexecutorworker.h | 3 + test/server/workers/testworkers.cpp | 114 ------------------ test/server/workers/testworkers.h | 3 - 6 files changed, 121 insertions(+), 128 deletions(-) diff --git a/src/server/appserver.cpp b/src/server/appserver.cpp index 2fe6c3d19..ee720807a 100644 --- a/src/server/appserver.cpp +++ b/src/server/appserver.cpp @@ -3953,15 +3953,15 @@ void AppServer::addError(const Error &error) { } else if (error.exitCode() == ExitCode::SystemError && error.exitCause() == ExitCause::FileAccessError) { // Remove child errors std::unordered_set toBeRemovedErrorIds; - for (const Error &parentError : errorList) { - for (const Error &childError : errorList) { + for (const Error &parentError: errorList) { + for (const Error &childError: errorList) { if (Utility::isSameOrChildPath(childError.path(), parentError.path()) && childError.dbId() != parentError.dbId()) { toBeRemovedErrorIds.insert(childError.dbId()); } } } - for (int errorId : toBeRemovedErrorIds) { + for (int errorId: toBeRemovedErrorIds) { bool found = false; if (!ParmsDb::instance()->deleteError(errorId, found)) { LOG_WARN(Log::instance()->getLogger(), "Error in ParmsDb::deleteError"); @@ -3972,13 +3972,7 @@ void AppServer::addError(const Error &error) { return; } } - if (!toBeRemovedErrorIds.empty()) - if (ServerRequests::isDisplayableError(error)) { - // Notify the client - sendErrorsCleared(error.syncDbId()); - } - - + if (!toBeRemovedErrorIds.empty()) sendErrorsCleared(error.syncDbId()); } if (!ServerRequests::isAutoResolvedError(error) && !errorAlreadyExists) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index de1a8d327..f823d153e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,4 +28,4 @@ add_subdirectory(libcommonserver) add_subdirectory(libcommon) add_subdirectory(libparms) add_subdirectory(libsyncengine) -add_subdirectory(server) +#add_subdirectory(server) diff --git a/test/libsyncengine/propagation/executor/testexecutorworker.cpp b/test/libsyncengine/propagation/executor/testexecutorworker.cpp index 48c75c995..aeb04e229 100644 --- a/test/libsyncengine/propagation/executor/testexecutorworker.cpp +++ b/test/libsyncengine/propagation/executor/testexecutorworker.cpp @@ -242,4 +242,117 @@ void TestExecutorWorker::testTargetUpdateTree() { syncOp->setTargetSide(ReplicaSide::Unknown); CPPUNIT_ASSERT_EQUAL(std::shared_ptr(nullptr), _syncPal->_executorWorker->targetUpdateTree(syncOp)); } + +void TestExecutorWorker::testRemoveDependentOps() { + { + // Nested create. + _syncPal->_syncOps->clear(); + + auto op1Create = std::make_shared(); // a/ is created. + auto op2Create = std::make_shared(); // a/b/ is created. + auto op3Create = std::make_shared(); // a/b/c/ is created. + auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, + OperationType::None, "idAn1", 1234, 1234, 1, nullptr); + auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, + OperationType::None, "idAn2", 1234, 1234, 1, nullptr); + auto affectedNode3 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an3"), NodeType::Unknown, + OperationType::None, "idAn3", 1234, 1234, 1, nullptr); + auto affectedNode4 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an4"), NodeType::Unknown, + OperationType::None, "idAn4", 1234, 1234, 1, nullptr); + + affectedNode4->setParentNode(affectedNode3); + affectedNode3->setParentNode(affectedNode2); + affectedNode2->setParentNode(affectedNode1); + + op1Create->setAffectedNode(affectedNode2); + op2Create->setAffectedNode(affectedNode3); + op3Create->setAffectedNode(affectedNode4); + + _syncPal->_syncOps->pushOp(op1Create); + _syncPal->_syncOps->pushOp(op2Create); + _syncPal->_syncOps->pushOp(op3Create); + + _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); + _syncPal->_executorWorker->removeDependentOps(op1Create); // op1Create failed, we should remove op2Create and op3Create. + + CPPUNIT_ASSERT(opsExist(op1Create)); + CPPUNIT_ASSERT(!opsExist(op2Create)); + CPPUNIT_ASSERT(!opsExist(op3Create)); + } + + { + // Nested Move. + _syncPal->_syncOps->clear(); + + auto op1Create = std::make_shared(); // a/ is created. + auto op2Move = std::make_shared(); // b/ is moved to a/b/. + + auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, + OperationType::None, "idAn1", 1234, 1234, 1, nullptr); // a/ + auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, + OperationType::None, "idAn2", 1234, 1234, 1, nullptr); // a/b + + auto correspondingNode2 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn2"), NodeType::Unknown, + OperationType::None, "idCn2", 1234, 1234, 1, nullptr); // b/ + + affectedNode2->setParentNode(affectedNode1); + + op1Create->setAffectedNode(affectedNode1); + op2Move->setAffectedNode(affectedNode2); + op2Move->setCorrespondingNode(correspondingNode2); + + _syncPal->_syncOps->pushOp(op1Create); + _syncPal->_syncOps->pushOp(op2Move); + + _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); + _syncPal->_executorWorker->removeDependentOps(op1Create); // op2Move failed, we should remove op2Edit. + CPPUNIT_ASSERT(opsExist(op1Create)); + CPPUNIT_ASSERT(!opsExist(op2Move)); + } + + { + // Double Move. + _syncPal->_syncOps->clear(); + + auto op1Move = std::make_shared(); // a is moved to b. + auto op2Move = std::make_shared(); // y is moved to b/y. + + auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, + OperationType::None, "idAn1", 1234, 1234, 1, nullptr); // b + auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, + OperationType::None, "idAn2", 1234, 1234, 1, nullptr); // b/y + + auto correspondingNode1 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn1"), NodeType::Unknown, + OperationType::None, "idCn1", 1234, 1234, 1, nullptr); // a + auto correspondingNode2 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn2"), NodeType::Unknown, + OperationType::None, "idCn2", 1234, 1234, 1, nullptr); // y + + affectedNode2->setParentNode(affectedNode1); + + op1Move->setAffectedNode(affectedNode1); + op1Move->setCorrespondingNode(correspondingNode1); + + op2Move->setAffectedNode(affectedNode2); + op2Move->setCorrespondingNode(correspondingNode2); + + _syncPal->_syncOps->pushOp(op1Move); + _syncPal->_syncOps->pushOp(op2Move); + + _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); + _syncPal->_executorWorker->removeDependentOps(op1Move); // op2Move failed, we should remove op2Edit. + CPPUNIT_ASSERT(opsExist(op1Move)); + CPPUNIT_ASSERT(!opsExist(op2Move)); + } +} + +bool TestExecutorWorker::opsExist(SyncOpPtr op) { + for (const auto &opId: _syncPal->_executorWorker->_opList) { + if (_syncPal->_syncOps->getOp(opId) == op) { + return true; + } + } + + return false; +} + } // namespace KDC diff --git a/test/libsyncengine/propagation/executor/testexecutorworker.h b/test/libsyncengine/propagation/executor/testexecutorworker.h index 032afa2d5..d4de7790f 100644 --- a/test/libsyncengine/propagation/executor/testexecutorworker.h +++ b/test/libsyncengine/propagation/executor/testexecutorworker.h @@ -32,6 +32,7 @@ class TestExecutorWorker : public CppUnit::TestFixture { CPPUNIT_TEST(testAffectedUpdateTree); CPPUNIT_TEST(testTargetUpdateTree); CPPUNIT_TEST(testLogCorrespondingNodeErrorMsg); + CPPUNIT_TEST(testRemoveDependentOps); CPPUNIT_TEST_SUITE_END(); public: @@ -44,7 +45,9 @@ class TestExecutorWorker : public CppUnit::TestFixture { void testAffectedUpdateTree(); void testTargetUpdateTree(); void testLogCorrespondingNodeErrorMsg(); + void testRemoveDependentOps(); + bool opsExist(SyncOpPtr op); SyncOpPtr generateSyncOperation(const DbNodeId dbNodeId, const SyncName &filename); std::shared_ptr _syncPal; diff --git a/test/server/workers/testworkers.cpp b/test/server/workers/testworkers.cpp index 313a34f21..58d6d3646 100644 --- a/test/server/workers/testworkers.cpp +++ b/test/server/workers/testworkers.cpp @@ -61,16 +61,6 @@ bool TestWorkers::setPinState(int syncDbId, const SyncPath &relativeLocalPath, P return true; } -bool TestWorkers::opsExist(SyncOpPtr op) { - for (const auto &opId: _syncPal->_executorWorker->_opList) { - if (_syncPal->_syncOps->getOp(opId) == op) { - return true; - } - } - - return false; -} - void TestWorkers::setUp() { _logger = Log::instance()->getLogger(); @@ -305,108 +295,4 @@ void TestWorkers::testConvertToPlaceholder() { CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, exitInfo.cause()); } } - -void TestWorkers::testRemoveDependentOps() { - { - // Nested create. - _syncPal->_syncOps->clear(); - - auto op1Create = std::make_shared(); // a/ is created. - auto op2Create = std::make_shared(); // a/b/ is created. - auto op3Create = std::make_shared(); // a/b/c/ is created. - auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, - OperationType::None, "idAn1", 1234, 1234, 1, nullptr); - auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, - OperationType::None, "idAn2", 1234, 1234, 1, nullptr); - auto affectedNode3 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an3"), NodeType::Unknown, - OperationType::None, "idAn3", 1234, 1234, 1, nullptr); - auto affectedNode4 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an4"), NodeType::Unknown, - OperationType::None, "idAn4", 1234, 1234, 1, nullptr); - - affectedNode4->setParentNode(affectedNode3); - affectedNode3->setParentNode(affectedNode2); - affectedNode2->setParentNode(affectedNode1); - - op1Create->setAffectedNode(affectedNode2); - op2Create->setAffectedNode(affectedNode3); - op3Create->setAffectedNode(affectedNode4); - - _syncPal->_syncOps->pushOp(op1Create); - _syncPal->_syncOps->pushOp(op2Create); - _syncPal->_syncOps->pushOp(op3Create); - - _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); - _syncPal->_executorWorker->removeDependentOps(op1Create); // op1Create failed, we should remove op2Create and op3Create. - - CPPUNIT_ASSERT(opsExist(op1Create)); - CPPUNIT_ASSERT(!opsExist(op2Create)); - CPPUNIT_ASSERT(!opsExist(op3Create)); - } - - { - // Nested Move. - _syncPal->_syncOps->clear(); - - auto op1Create = std::make_shared(); // a/ is created. - auto op2Move = std::make_shared(); // b/ is moved to a/b/. - - auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, - OperationType::None, "idAn1", 1234, 1234, 1, nullptr); // a/ - auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, - OperationType::None, "idAn2", 1234, 1234, 1, nullptr); // a/b - - auto correspondingNode2 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn2"), NodeType::Unknown, - OperationType::None, "idCn2", 1234, 1234, 1, nullptr); // b/ - - affectedNode2->setParentNode(affectedNode1); - - op1Create->setAffectedNode(affectedNode1); - op2Move->setAffectedNode(affectedNode2); - op2Move->setCorrespondingNode(correspondingNode2); - - _syncPal->_syncOps->pushOp(op1Create); - _syncPal->_syncOps->pushOp(op2Move); - - _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); - _syncPal->_executorWorker->removeDependentOps(op1Create); // op2Move failed, we should remove op2Edit. - CPPUNIT_ASSERT(opsExist(op1Create)); - CPPUNIT_ASSERT(!opsExist(op2Move)); - } - - { - // Double Move. - _syncPal->_syncOps->clear(); - - auto op1Move = std::make_shared(); // a is moved to b. - auto op2Move = std::make_shared(); // y is moved to b/y. - - auto affectedNode1 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an1"), NodeType::Unknown, - OperationType::None, "idAn1", 1234, 1234, 1, nullptr); // b - auto affectedNode2 = std::make_shared(ReplicaSide::Remote, Str2SyncName("an2"), NodeType::Unknown, - OperationType::None, "idAn2", 1234, 1234, 1, nullptr); // b/y - - auto correspondingNode1 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn1"), NodeType::Unknown, - OperationType::None, "idCn1", 1234, 1234, 1, nullptr); // a - auto correspondingNode2 = std::make_shared(ReplicaSide::Local, Str2SyncName("cn2"), NodeType::Unknown, - OperationType::None, "idCn2", 1234, 1234, 1, nullptr); // y - - affectedNode2->setParentNode(affectedNode1); - - op1Move->setAffectedNode(affectedNode1); - op1Move->setCorrespondingNode(correspondingNode1); - - op2Move->setAffectedNode(affectedNode2); - op2Move->setCorrespondingNode(correspondingNode2); - - _syncPal->_syncOps->pushOp(op1Move); - _syncPal->_syncOps->pushOp(op2Move); - - _syncPal->_executorWorker->_opList = _syncPal->_syncOps->opSortedList(); - _syncPal->_executorWorker->removeDependentOps(op1Move); // op2Move failed, we should remove op2Edit. - CPPUNIT_ASSERT(opsExist(op1Move)); - CPPUNIT_ASSERT(!opsExist(op2Move)); - } -} - - } // namespace KDC diff --git a/test/server/workers/testworkers.h b/test/server/workers/testworkers.h index b1419c28e..2e6ab927a 100644 --- a/test/server/workers/testworkers.h +++ b/test/server/workers/testworkers.h @@ -35,7 +35,6 @@ class TestWorkers : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(TestWorkers); CPPUNIT_TEST(testCreatePlaceholder); CPPUNIT_TEST(testConvertToPlaceholder); - CPPUNIT_TEST(testRemoveDependentOps); CPPUNIT_TEST_SUITE_END(); public: @@ -43,13 +42,11 @@ class TestWorkers : public CppUnit::TestFixture { void tearDown() override; void testCreatePlaceholder(); void testConvertToPlaceholder(); - void testRemoveDependentOps(); protected: static bool createPlaceholder(int syncDbId, const SyncPath &relativeLocalPath, const SyncFileItem &item); static bool convertToPlaceholder(int syncDbId, const SyncPath &relativeLocalPath, const SyncFileItem &item); static bool setPinState(int syncDbId, const SyncPath &relativeLocalPath, PinState pinState); - bool opsExist(SyncOpPtr op); log4cplus::Logger _logger; std::shared_ptr _syncPal; Sync _sync; From 7802b0a0e8ee0a2cafc607c03d623584df9138b8 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Tue, 29 Oct 2024 11:52:21 +0100 Subject: [PATCH 46/56] Rename isSameOrChildPath to isDescendantOrEqual. --- src/libcommonserver/utility/utility.cpp | 2 +- src/libcommonserver/utility/utility.h | 2 +- .../operationsorterworker.cpp | 8 +++---- .../conflict_finder/conflictfinderworker.cpp | 4 ++-- .../syncpal/tmpblacklistmanager.cpp | 8 +++---- .../folderwatcher_linux.cpp | 4 ++-- src/server/appserver.cpp | 2 +- test/libcommonserver/utility/testutility.cpp | 24 +++++++++---------- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/libcommonserver/utility/utility.cpp b/src/libcommonserver/utility/utility.cpp index 0fd062871..3b90ccbf2 100644 --- a/src/libcommonserver/utility/utility.cpp +++ b/src/libcommonserver/utility/utility.cpp @@ -409,7 +409,7 @@ bool Utility::isEqualNormalized(const SyncPath &a, const SyncPath &b) { return aNormalized == bNormalized; } -bool Utility::isSameOrChildPath(const SyncPath &potentialChild, const SyncPath &path) { +bool Utility::isDescendantOrEqual(const SyncPath &potentialChild, const SyncPath &path) { if (path == potentialChild) return true; for (auto it = potentialChild.begin(), it2 = path.begin(); it != potentialChild.end(); ++it, ++it2) { if (it2 == path.end()) { diff --git a/src/libcommonserver/utility/utility.h b/src/libcommonserver/utility/utility.h index b5da98cd9..2b7959767 100644 --- a/src/libcommonserver/utility/utility.h +++ b/src/libcommonserver/utility/utility.h @@ -102,7 +102,7 @@ struct COMMONSERVER_EXPORT Utility { static bool endsWithInsensitive(const SyncName &str, const SyncName &suffix); static bool isEqualInsensitive(const SyncName &a, const SyncName &b); #endif - static bool isSameOrChildPath(const SyncPath &potentialChild, const SyncPath &path); + static bool isDescendantOrEqual(const SyncPath &potentialDescendant, const SyncPath &path); /** * Normalize the SyncName parameters before comparing them. * @param a SyncName value to be compared. diff --git a/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp b/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp index ada07b66e..2ccd03951 100644 --- a/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp +++ b/src/libsyncengine/propagation/operation_sorter/operationsorterworker.cpp @@ -286,7 +286,7 @@ void OperationSorterWorker::fixMoveBeforeDelete() { } SyncPath sourcePath = moveOp->affectedNode()->moveOrigin().value(); - if (Utility::isSameOrChildPath(sourcePath.lexically_normal(), + if (Utility::isDescendantOrEqual(sourcePath.lexically_normal(), SyncPath(deleteDirPath.native() + Str("/")).lexically_normal())) { // move only if op is before moveOp moveFirstAfterSecond(deleteOp, moveOp); @@ -594,10 +594,10 @@ void OperationSorterWorker::fixMoveBeforeMoveHierarchyFlip() { } SyncPath ySourcePath = *yNode->moveOrigin(); - bool isXBelowY = Utility::isSameOrChildPath(xDestPath.lexically_normal(), + bool isXBelowY = Utility::isDescendantOrEqual(xDestPath.lexically_normal(), SyncPath(yDestPath.native() + Str("/")).lexically_normal()); if (isXBelowY) { - bool isYBelowXInDb = Utility::isSameOrChildPath(ySourcePath.lexically_normal(), + bool isYBelowXInDb = Utility::isDescendantOrEqual(ySourcePath.lexically_normal(), SyncPath(xSourcePath.native() + Str("/")).lexically_normal()); if (isYBelowXInDb) { moveFirstAfterSecond(xOp, yOp); @@ -624,7 +624,7 @@ std::optional OperationSorterWorker::fixImpossibleFirstMoveOp // impossible move if dest = source + "/" SyncPath source = (o1->affectedNode()->moveOrigin().has_value() ? o1->affectedNode()->moveOrigin().value() : ""); SyncPath dest = o1->affectedNode()->getPath(); - if (!Utility::isSameOrChildPath(dest.lexically_normal(), SyncPath(source.native() + Str("/")).lexically_normal())) { + if (!Utility::isDescendantOrEqual(dest.lexically_normal(), SyncPath(source.native() + Str("/")).lexically_normal())) { return std::nullopt; } diff --git a/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp b/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp index 140b851b2..13b4bed6f 100644 --- a/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp +++ b/src/libsyncengine/reconciliation/conflict_finder/conflictfinderworker.cpp @@ -408,9 +408,9 @@ std::optional> ConflictFinderWorker::determineMoveMoveCycl continue; } - if (Utility::isSameOrChildPath(SyncPath(localDbPath).lexically_normal(), + if (Utility::isDescendantOrEqual(SyncPath(localDbPath).lexically_normal(), SyncPath(remoteDbPath.native() + Str("/")).lexically_normal()) || - Utility::isSameOrChildPath(SyncPath(remoteDbPath).lexically_normal(), + Utility::isDescendantOrEqual(SyncPath(remoteDbPath).lexically_normal(), SyncPath(localDbPath.native() + Str("/")).lexically_normal())) { continue; } diff --git a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp index 23f86fb12..d994ddff4 100644 --- a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp +++ b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp @@ -92,7 +92,7 @@ void TmpBlacklistManager::blacklistItem(const NodeId &nodeId, const SyncPath &re std::list toBeRemoved; for (const auto &errorInfo: errors) { - if (Utility::isSameOrChildPath(errorInfo.second.path, relativePath) && errorInfo.first != nodeId) { + if (Utility::isDescendantOrEqual(errorInfo.second.path, relativePath) && errorInfo.first != nodeId) { toBeRemoved.push_back(errorInfo.first); } } @@ -148,14 +148,14 @@ void TmpBlacklistManager::removeItemFromTmpBlacklist(const SyncPath &relativePat // Find the node id of the item to be removed for (const auto &[nodeId, tmpInfo]: _localErrors) { - if (Utility::isSameOrChildPath(tmpInfo.path, relativePath)) { + if (Utility::isDescendantOrEqual(tmpInfo.path, relativePath)) { localId = nodeId; break; } } for (const auto &[nodeId, tmpInfo]: _remoteErrors) { - if (Utility::isSameOrChildPath(tmpInfo.path, relativePath)) { + if (Utility::isDescendantOrEqual(tmpInfo.path, relativePath)) { remotedId = nodeId; break; } @@ -173,7 +173,7 @@ void TmpBlacklistManager::removeItemFromTmpBlacklist(const SyncPath &relativePat bool TmpBlacklistManager::isTmpBlacklisted(const SyncPath &path, ReplicaSide side) const { auto &errors = side == ReplicaSide::Local ? _localErrors : _remoteErrors; for (const auto &errorInfo: errors) { - if (Utility::isSameOrChildPath(path, errorInfo.second.path)) return true; + if (Utility::isDescendantOrEqual(path, errorInfo.second.path)) return true; } return false; diff --git a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp index 9157f4df8..cc3766595 100644 --- a/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/folderwatcher_linux.cpp @@ -252,11 +252,11 @@ void FolderWatcher_linux::removeFoldersBelow(const SyncPath &dirPath) { // Remove the entry and all subentries while (it != _pathToWatch.end()) { auto itPath = it->first; - if (Utility::isSameOrChildPath(itPath, dirPath)) { + if (Utility::isDescendantOrEqual(itPath, dirPath)) { break; } - if (itPath != dirPath && Utility::isSameOrChildPath(itPath, pathSlash)) { + if (itPath != dirPath && Utility::isDescendantOrEqual(itPath, pathSlash)) { // order is 'foo', 'foo bar', 'foo/bar' ++it; continue; diff --git a/src/server/appserver.cpp b/src/server/appserver.cpp index fb1f28c19..69e7c2f01 100644 --- a/src/server/appserver.cpp +++ b/src/server/appserver.cpp @@ -3905,7 +3905,7 @@ void AppServer::addError(const Error &error) { std::unordered_set toBeRemovedErrorIds; for (const Error &parentError: errorList) { for (const Error &childError: errorList) { - if (Utility::isSameOrChildPath(childError.path(), parentError.path()) && + if (Utility::isDescendantOrEqual(childError.path(), parentError.path()) && childError.dbId() != parentError.dbId()) { toBeRemovedErrorIds.insert(childError.dbId()); } diff --git a/test/libcommonserver/utility/testutility.cpp b/test/libcommonserver/utility/testutility.cpp index a3fec273e..523fd4196 100644 --- a/test/libcommonserver/utility/testutility.cpp +++ b/test/libcommonserver/utility/testutility.cpp @@ -481,19 +481,19 @@ void TestUtility::testNormalizedSyncPath() { } void TestUtility::testIsSameOrParentPath() { - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("", "a")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a", "a/b")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a", "a/b/c")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b", "a/b/c")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c", "a/b/c1")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("a/b/c1", "a/b/c")); - CPPUNIT_ASSERT(!Utility::isSameOrChildPath("/a/b/c", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("", "a")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("a", "a/b")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("a", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("a/b", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("a/b/c", "a/b/c1")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("a/b/c1", "a/b/c")); + CPPUNIT_ASSERT(!Utility::isDescendantOrEqual("/a/b/c", "a/b/c")); - CPPUNIT_ASSERT(Utility::isSameOrChildPath("", "")); - CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a/b/c")); - CPPUNIT_ASSERT(Utility::isSameOrChildPath("a", "")); - CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a/b")); - CPPUNIT_ASSERT(Utility::isSameOrChildPath("a/b/c", "a")); + CPPUNIT_ASSERT(Utility::isDescendantOrEqual("", "")); + CPPUNIT_ASSERT(Utility::isDescendantOrEqual("a/b/c", "a/b/c")); + CPPUNIT_ASSERT(Utility::isDescendantOrEqual("a", "")); + CPPUNIT_ASSERT(Utility::isDescendantOrEqual("a/b/c", "a/b")); + CPPUNIT_ASSERT(Utility::isDescendantOrEqual("a/b/c", "a")); } } // namespace KDC From 431ca7d9c5ea5ce81057ae0f8d9269b2c052bfc2 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Tue, 29 Oct 2024 12:21:31 +0100 Subject: [PATCH 47/56] checkLocalTree does not need a copy of the children list, as the update tree deletion only occurs at the end of execute(). --- .../platforminconsistencycheckerworker.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp index 4a299d160..1869c9ae7 100644 --- a/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp +++ b/src/libsyncengine/reconciliation/platform_inconsistency_checker/platforminconsistencycheckerworker.cpp @@ -131,9 +131,8 @@ ExitCode PlatformInconsistencyCheckerWorker::checkLocalTree(std::shared_ptrchildren(); - auto childIt = childCopy.begin(); - for (; childIt != childCopy.end(); childIt++) { + auto it = localNode->children().begin(); + for (; it != localNode->children().end(); it++) { if (stopAsked()) { return ExitCode::Ok; } @@ -146,7 +145,7 @@ ExitCode PlatformInconsistencyCheckerWorker::checkLocalTree(std::shared_ptrsecond, parentPath / localNode->name()); + const ExitCode exitCode = checkLocalTree(it->second, parentPath / localNode->name()); if (exitCode != ExitCode::Ok) { return exitCode; } @@ -191,14 +190,6 @@ void PlatformInconsistencyCheckerWorker::blacklistNode(const std::shared_ptrupdateTree(ReplicaSide::Local)->deleteNode(nodeIDs.localId)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node " << Utility::formatSyncPath(node->getPath())); - } - - if (!nodeIDs.remoteId.empty() && !_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(nodeIDs.remoteId)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node " << Utility::formatSyncPath(node->getPath())); - } - _idsToBeRemoved.emplace_back(nodeIDs); } From b58850031a7afeeea8259e97f85c69e785039f35 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Tue, 29 Oct 2024 18:03:05 +0100 Subject: [PATCH 48/56] Fix typo --- src/libsyncengine/propagation/executor/executorworker.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 0a89ff03f..edd5ff5dc 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -649,7 +649,6 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptr Date: Wed, 30 Oct 2024 15:54:21 +0100 Subject: [PATCH 49/56] Set ExitCode::Ok to 0 --- src/libcommon/utility/types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcommon/utility/types.h b/src/libcommon/utility/types.h index bba9ce368..6f51e644f 100644 --- a/src/libcommon/utility/types.h +++ b/src/libcommon/utility/types.h @@ -172,8 +172,8 @@ enum class OperationType { None = 0x00, Create = 0x01, Move = 0x02, Edit = 0x04, std::string toString(OperationType e); enum class ExitCode { - Unknown, Ok, + Unknown, NeedRestart, // A propagation job cannot be executed because the situation that led to its creation is no longer // verified NetworkError, From e637b94214070ffd08e28dd9945ad086c069a4bf Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Wed, 30 Oct 2024 16:43:05 +0100 Subject: [PATCH 50/56] Search in tmpBlacklist by NodeID instead of SyncPath. --- src/libsyncengine/syncpal/syncpal.cpp | 4 ++++ src/libsyncengine/syncpal/syncpal.h | 1 + src/libsyncengine/syncpal/tmpblacklistmanager.cpp | 5 +++++ src/libsyncengine/syncpal/tmpblacklistmanager.h | 1 + .../localfilesystemobserverworker.cpp | 13 ++++++++++++- 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/libsyncengine/syncpal/syncpal.cpp b/src/libsyncengine/syncpal/syncpal.cpp index 4785d7f90..9e8613d5a 100644 --- a/src/libsyncengine/syncpal/syncpal.cpp +++ b/src/libsyncengine/syncpal/syncpal.cpp @@ -1406,6 +1406,10 @@ void SyncPal::blacklistTemporarily(const NodeId &nodeId, const SyncPath &relativ _tmpBlacklistManager->blacklistItem(nodeId, relativePath, side); } +bool SyncPal::isTmpBlacklisted(const NodeId &nodeId, ReplicaSide side) const { + return _tmpBlacklistManager->isTmpBlacklisted(nodeId, side); +} + bool SyncPal::isTmpBlacklisted(const SyncPath &relativePath, ReplicaSide side) const { return _tmpBlacklistManager->isTmpBlacklisted(relativePath, side); } diff --git a/src/libsyncengine/syncpal/syncpal.h b/src/libsyncengine/syncpal/syncpal.h index 87374a31f..9821651bb 100644 --- a/src/libsyncengine/syncpal/syncpal.h +++ b/src/libsyncengine/syncpal/syncpal.h @@ -252,6 +252,7 @@ class SYNCENGINE_EXPORT SyncPal : public std::enable_shared_from_this { virtual void increaseErrorCount(const NodeId &nodeId, NodeType type, const SyncPath &relativePath, ReplicaSide side); virtual int getErrorCount(const NodeId &nodeId, ReplicaSide side) const noexcept; virtual void blacklistTemporarily(const NodeId &nodeId, const SyncPath &relativePath, ReplicaSide side); + virtual bool isTmpBlacklisted(const NodeId &nodeId, ReplicaSide side) const; virtual bool isTmpBlacklisted(const SyncPath &relativePath, ReplicaSide side) const; virtual void refreshTmpBlacklist(); virtual void removeItemFromTmpBlacklist(const NodeId &nodeId, ReplicaSide side); diff --git a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp index d994ddff4..c04c852dc 100644 --- a/src/libsyncengine/syncpal/tmpblacklistmanager.cpp +++ b/src/libsyncengine/syncpal/tmpblacklistmanager.cpp @@ -179,6 +179,11 @@ bool TmpBlacklistManager::isTmpBlacklisted(const SyncPath &path, ReplicaSide sid return false; } +bool TmpBlacklistManager::isTmpBlacklisted(const NodeId &nodeId, ReplicaSide side) const { + auto &errors = side == ReplicaSide::Local ? _localErrors : _remoteErrors; + return (errors.contains(nodeId)); +} + void TmpBlacklistManager::insertInBlacklist(const NodeId &nodeId, ReplicaSide side) { const auto blacklistType_ = blackListType(side); diff --git a/src/libsyncengine/syncpal/tmpblacklistmanager.h b/src/libsyncengine/syncpal/tmpblacklistmanager.h index c57590a67..5ca834817 100644 --- a/src/libsyncengine/syncpal/tmpblacklistmanager.h +++ b/src/libsyncengine/syncpal/tmpblacklistmanager.h @@ -42,6 +42,7 @@ class TmpBlacklistManager { // Remove the item from local and/or remote blacklist void removeItemFromTmpBlacklist(const SyncPath& relativePath); bool isTmpBlacklisted(const SyncPath &path, ReplicaSide side) const; + bool isTmpBlacklisted(const NodeId &nodeId, ReplicaSide side) const; int getErrorCount(const NodeId &nodeId, ReplicaSide side) const noexcept; private: diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index df21f3297..a6b2544dc 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -36,7 +36,8 @@ static const int waitForUpdateDelay = 1000; // 1sec LocalFileSystemObserverWorker::LocalFileSystemObserverWorker(std::shared_ptr syncPal, const std::string &name, const std::string &shortName) : - FileSystemObserverWorker(syncPal, name, shortName, ReplicaSide::Local), _rootFolder(syncPal->localPath()) {} + FileSystemObserverWorker(syncPal, name, shortName, ReplicaSide::Local), + _rootFolder(syncPal->localPath()) {} LocalFileSystemObserverWorker::~LocalFileSystemObserverWorker() { LOG_SYNCPAL_DEBUG(_logger, "~LocalFileSystemObserverWorker"); @@ -100,6 +101,8 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listisTmpBlacklisted(prevNodeId, ReplicaSide::Local)) + _syncPal->removeItemFromTmpBlacklist(relativePath); if (IoHelper::checkIfPathExistsWithSameNodeId(absolutePath, prevNodeId, existsWithSameId, otherNodeId, ioError) && !existsWithSameId) { if (_snapshot->removeItem(prevNodeId)) { @@ -134,6 +137,14 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listisTmpBlacklisted(nodeId, ReplicaSide::Local)) { + _syncPal->removeItemFromTmpBlacklist(relativePath); + if (opTypeFromOS == OperationType::Edit) { + NodeId itemId = _snapshot->itemId(relativePath); + if (!itemId.empty()) _snapshot->setLastModified(itemId, 0); + } + } + bool isLink = false; if (exists) { ItemType itemType; From 814d2d90e11a142cb254bb457e9d1c1abd05adcd Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 31 Oct 2024 08:20:40 +0100 Subject: [PATCH 51/56] Fix merge in executorWorker. --- .../propagation/executor/executorworker.cpp | 321 +++++++++--------- 1 file changed, 160 insertions(+), 161 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index c64784d92..8bafca077 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -101,7 +101,7 @@ void ExecutorWorker::execute() { } } - if (JobManager::instance()->countManagedJobs() > static_cast(JobManager::instance()->maxNbThreads() * 2)) { + if (JobManager::instance()->countManagedJobs() > static_cast(JobManager::instance()->maxNbThreads()) * 2) { if (ParametersCache::isExtendedLogEnabled()) { LOG_SYNCPAL_DEBUG(_logger, "Maximum number of jobs reached"); } @@ -141,9 +141,8 @@ void ExecutorWorker::execute() { break; } default: { - LOGW_SYNCPAL_WARN(_logger, L"Unknown operation type: " - << syncOp->type() << L" on file " - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + LOGW_SYNCPAL_WARN(_logger, L"Unknown operation type: " << syncOp->type() << L" on file " + << SyncName2WStr(syncOp->affectedNode()->name())); executorExitInfo = ExitCode::DataError; } } @@ -240,7 +239,9 @@ void ExecutorWorker::initProgressManager() { } initSyncFileItem(syncOp, syncItem); - _syncPal->initProgress(syncItem); + if (!_syncPal->initProgress(syncItem)) { + LOG_SYNCPAL_WARN(_logger, "Error in SyncPal::initProgress: id=" << syncOpId); + } } } @@ -417,8 +418,8 @@ ExitInfo ExecutorWorker::handleCreateOp(SyncOpPtr syncOp, std::shared_ptrtargetSide() == ReplicaSide::Remote); !exitInfo) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to convert to placeholder for: " - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + LOGW_SYNCPAL_WARN(_logger, + L"Failed to convert to placeholder for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } @@ -432,39 +433,40 @@ ExitInfo ExecutorWorker::checkAlreadyExcluded(const SyncPath &absolutePath, cons bool alreadyExist = false; // List all items in parent dir - try{ - GetFileListJob job(_syncPal->driveDbId(), parentId); - ExitCode exitCode = job.runSynchronously(); - if (exitCode != ExitCode::Ok) { - LOG_SYNCPAL_WARN(_logger, "Error in GetFileListJob::runSynchronously for driveDbId=" - << _syncPal->driveDbId() << " nodeId=" << parentId.c_str() << " : " << exitCode << " " - << job.exitCause()); - return {exitCode, job.exitCause()}; - } - - Poco::JSON::Object::Ptr resObj = job.jsonRes(); - if (!resObj) { - LOG_SYNCPAL_WARN(Log::instance()->getLogger(), - "GetFileListJob failed for driveDbId=" << _syncPal->driveDbId() << " nodeId=" << parentId.c_str()); - return {ExitCode::BackError, ExitCause::ApiErr}; - } - - Poco::JSON::Array::Ptr dataArray = resObj->getArray(dataKey); - if (!dataArray) { - LOG_SYNCPAL_WARN(Log::instance()->getLogger(), - "GetFileListJob failed for driveDbId=" << _syncPal->driveDbId() << " nodeId=" << parentId.c_str()); - return {ExitCode::BackError, ExitCause::ApiErr}; - } - - for (Poco::JSON::Array::ConstIterator it = dataArray->begin(); it != dataArray->end(); ++it) { - Poco::JSON::Object::Ptr obj = it->extract(); - std::string name; - if (!JsonParserUtility::extractValue(obj, nameKey, name)) { + try { + GetFileListJob job(_syncPal->driveDbId(), parentId); + ExitCode exitCode = job.runSynchronously(); + if (exitCode != ExitCode::Ok) { + LOG_SYNCPAL_WARN(_logger, "Error in GetFileListJob::runSynchronously for driveDbId=" + << _syncPal->driveDbId() << " nodeId=" << parentId.c_str() << " : " << exitCode + << " " << job.exitCause()); + return {exitCode, job.exitCause()}; + } + + Poco::JSON::Object::Ptr resObj = job.jsonRes(); + if (!resObj) { + LOG_SYNCPAL_WARN(Log::instance()->getLogger(), + "GetFileListJob failed for driveDbId=" << _syncPal->driveDbId() << " nodeId=" << parentId.c_str()); + return {ExitCode::BackError, ExitCause::ApiErr}; + } + + Poco::JSON::Array::Ptr dataArray = resObj->getArray(dataKey); + if (!dataArray) { LOG_SYNCPAL_WARN(Log::instance()->getLogger(), "GetFileListJob failed for driveDbId=" << _syncPal->driveDbId() << " nodeId=" << parentId.c_str()); return {ExitCode::BackError, ExitCause::ApiErr}; } + for (Poco::JSON::Array::ConstIterator it = dataArray->begin(); it != dataArray->end(); ++it) { + Poco::JSON::Object::Ptr obj = it->extract(); + std::string name; + if (!JsonParserUtility::extractValue(obj, nameKey, name)) { + LOG_SYNCPAL_WARN( + Log::instance()->getLogger(), + "GetFileListJob failed for driveDbId=" << _syncPal->driveDbId() << " nodeId=" << parentId.c_str()); + return {ExitCode::BackError, ExitCause::ApiErr}; + } + if (name == absolutePath.filename().string()) { alreadyExist = true; break; @@ -474,7 +476,7 @@ ExitInfo ExecutorWorker::checkAlreadyExcluded(const SyncPath &absolutePath, cons LOG_WARN(Log::instance()->getLogger(), "Error in GetFileListJob::GetFileListJob for driveDbId=" << _syncPal->driveDbId() << " nodeId=" << parentId.c_str() << " error=" << e.what()); - return false; + return ExitCode::DataError; } if (!alreadyExist) { @@ -497,8 +499,8 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->parentNode()); if (!newCorrespondingParentNode) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to get corresponding parent node: " - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + LOGW_SYNCPAL_WARN(_logger, + L"Failed to get corresponding parent node: " << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; } } @@ -516,23 +518,23 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->name()).c_str()); + L"Failed to create placeholder for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } FileStat fileStat; IoError ioError = IoError::Success; if (!IoHelper::getFileStat(absoluteLocalFilePath, &fileStat, ioError)) { - LOGW_SYNCPAL_WARN(_logger, L"Error in IoHelper::getFileStat: " - << Utility::formatIoError(absoluteLocalFilePath, ioError).c_str()); + LOGW_SYNCPAL_WARN(_logger, + L"Error in IoHelper::getFileStat: " << Utility::formatIoError(absoluteLocalFilePath, ioError)); return ExitCode::SystemError; } if (ioError == IoError::NoSuchFileOrDirectory) { - LOGW_WARN(_logger, L"Item does not exist anymore: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); + LOGW_WARN(_logger, L"Item does not exist anymore: " << Utility::formatSyncPath(absoluteLocalFilePath)); return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } else if (ioError == IoError::AccessDenied) { - LOGW_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); + LOGW_WARN(_logger, L"Item misses search permission: " << Utility::formatSyncPath(absoluteLocalFilePath)); return {ExitCode::SystemError, ExitCause::FileAccessError}; } @@ -541,7 +543,7 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->lastmodified(), newNode); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } @@ -569,8 +571,7 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->createdAt().has_value() ? *syncOp->affectedNode()->createdAt() : 0, syncOp->affectedNode()->lastmodified().has_value() ? *syncOp->affectedNode()->lastmodified() : 0, true); - } catch (const std::exception &e) { - LOG_SYNCPAL_WARN(_logger, "Error in DownloadJob::DownloadJob for driveDbId=" << _syncPal->driveDbId() - << " error=" << e.what()); + } catch (std::exception const &e) { + LOGW_SYNCPAL_WARN(_logger, L"Error in DownloadJob::DownloadJob for driveDbId=" + << _syncPal->driveDbId() << L" : " << Utility::s2ws(e.what())); return ExitCode::DataError; } @@ -610,8 +611,8 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->type() == NodeType::Directory) { if (ExitInfo exitInfo = convertToPlaceholder(relativeLocalFilePath, syncOp->targetSide() == ReplicaSide::Remote); !exitInfo) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to convert to placeholder for: " - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + LOGW_SYNCPAL_WARN(_logger, + L"Failed to convert to placeholder for: " << SyncName2WStr(syncOp->affectedNode()->name())); _syncPal->setRestart(true); if (!_syncPal->updateTree(ReplicaSide::Local)->deleteNode(syncOp->affectedNode())) { @@ -627,22 +628,22 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptrdriveDbId(), absoluteLocalFilePath, newCorrespondingParentNode->id().has_value() ? *newCorrespondingParentNode->id() : std::string(), syncOp->newName()); - } catch (const std::exception &e) { - LOG_SYNCPAL_WARN(_logger, "Error in CreateDirJob::CreateDirJob for driveDbId=" << _syncPal->driveDbId() - << " error=" << e.what()); + } catch (std::exception const &e) { + LOGW_SYNCPAL_WARN(_logger, L"Error in CreateDirJob::CreateDirJob for driveDbId=" + << _syncPal->driveDbId() << L" : " << Utility::s2ws(e.what())); return ExitCode::DataError; } } else { if (ExitInfo exitInfo = convertToPlaceholder(relativeLocalFilePath, true); !exitInfo) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to convert to placeholder for: " - << SyncName2WStr(syncOp->affectedNode()->name())); + LOGW_SYNCPAL_WARN(_logger, + L"Failed to convert to placeholder for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } uint64_t filesize = 0; if (ExitInfo exitInfo = getFileSize(absoluteLocalFilePath, filesize); !exitInfo) { - LOGW_SYNCPAL_WARN(_logger, L"Error in ExecutorWorker::getFileSize for " - << Utility::formatSyncPath(absoluteLocalFilePath)); + LOGW_SYNCPAL_WARN(_logger, + L"Error in ExecutorWorker::getFileSize for " << Utility::formatSyncPath(absoluteLocalFilePath)); return exitInfo; } @@ -655,8 +656,7 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->lastmodified() ? *syncOp->affectedNode()->lastmodified() : 0, isLiteSyncActivated(), uploadSessionParallelJobs); } catch (std::exception const &e) { - LOG_SYNCPAL_WARN(_logger, - "Error in DriveUploadSession::DriveUploadSession: " << e.what()); + LOGW_SYNCPAL_WARN(_logger, L"Error in DriveUploadSession::DriveUploadSession: " << Utility::s2ws(e.what())); return ExitCode::DataError; } } else { @@ -666,8 +666,8 @@ ExitInfo ExecutorWorker::generateCreateJob(SyncOpPtr syncOp, std::shared_ptrid().has_value() ? *newCorrespondingParentNode->id() : std::string(), syncOp->affectedNode()->lastmodified() ? *syncOp->affectedNode()->lastmodified() : 0); } catch (std::exception const &e) { - LOG_SYNCPAL_WARN(_logger, "Error in UploadJob::UploadJob for driveDbId=" - << _syncPal->driveDbId() << " : " << e.what()); + LOGW_SYNCPAL_WARN(_logger, L"Error in UploadJob::UploadJob for driveDbId=" << _syncPal->driveDbId() << L" : " + << Utility::s2ws(e.what())); return ExitCode::DataError; } } @@ -733,8 +733,8 @@ ExitInfo ExecutorWorker::checkLiteSyncInfoForCreate(SyncOpPtr syncOp, const Sync ExitInfo ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath) { SyncFileItem syncItem; if (!_syncPal->getSyncFileItem(relativeLocalPath, syncItem)) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to retrieve SyncFileItem associated to item: " - << Utility::formatSyncPath(relativeLocalPath)); + LOGW_SYNCPAL_WARN(_logger, + L"Failed to retrieve SyncFileItem associated to item: " << Utility::formatSyncPath(relativeLocalPath)); return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } @@ -764,8 +764,8 @@ ExitInfo ExecutorWorker::createPlaceholder(const SyncPath &relativeLocalPath) { ExitInfo ExecutorWorker::convertToPlaceholder(const SyncPath &relativeLocalPath, bool hydrated) { SyncFileItem syncItem; if (!_syncPal->getSyncFileItem(relativeLocalPath, syncItem)) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to retrieve SyncFileItem associated to item: " - << Utility::formatSyncPath(relativeLocalPath)); + LOGW_SYNCPAL_WARN(_logger, + L"Failed to retrieve SyncFileItem associated to item: " << Utility::formatSyncPath(relativeLocalPath)); return {ExitCode::DataError, ExitCause::InvalidSnapshot}; } @@ -824,9 +824,9 @@ ExitInfo ExecutorWorker::processCreateOrConvertToPlaceholderError(const SyncPath return {ExitCode::SystemError, ExitCause::FileAccessError}; } - if (create && exists){ + if (create && exists) { return {ExitCode::SystemError, ExitCause::FileAccessError}; - }else if (!create && !exists){ + } else if (!create && !exists) { return {ExitCode::DataError, ExitCause::FileAlreadyExist}; } @@ -931,8 +931,8 @@ ExitInfo ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->createdAt().has_value() ? *syncOp->affectedNode()->createdAt() : 0, syncOp->affectedNode()->lastmodified().has_value() ? *syncOp->affectedNode()->lastmodified() : 0, false); } catch (std::exception const &e) { - LOG_SYNCPAL_WARN(_logger, "Error in DownloadJob::DownloadJob for driveDbId=" << _syncPal->driveDbId() << " : " - << e.what()); + LOGW_SYNCPAL_WARN(_logger, L"Error in DownloadJob::DownloadJob for driveDbId=" << _syncPal->driveDbId() << L" : " + << Utility::s2ws(e.what())); return ExitCode::DataError; } @@ -964,15 +964,14 @@ ExitInfo ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptrcorrespondingNode()->id()) { // Should not happen LOGW_SYNCPAL_WARN(_logger, L"Edit operation with empty corresponding node id for " - << Utility::formatSyncPath(absoluteLocalFilePath).c_str()); + << Utility::formatSyncPath(absoluteLocalFilePath)); SentryHandler::instance()->captureMessage(SentryLevel::Warning, "ExecutorWorker::generateEditJob", "Edit operation with empty corresponding node id"); return ExitCode::DataError; @@ -987,8 +986,7 @@ ExitInfo ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptraffectedNode()->lastmodified() ? *syncOp->affectedNode()->lastmodified() : 0, isLiteSyncActivated(), uploadSessionParallelJobs); } catch (std::exception const &e) { - LOG_SYNCPAL_WARN(_logger, - "Error in DriveUploadSession::DriveUploadSession: " << e.what())); + LOGW_SYNCPAL_WARN(_logger, L"Error in DriveUploadSession::DriveUploadSession: " << Utility::s2ws(e.what())); return ExitCode::DataError; }; } else { @@ -998,8 +996,8 @@ ExitInfo ExecutorWorker::generateEditJob(SyncOpPtr syncOp, std::shared_ptrcorrespondingNode()->id() ? *syncOp->correspondingNode()->id() : std::string(), syncOp->affectedNode()->lastmodified() ? *syncOp->affectedNode()->lastmodified() : 0); } catch (std::exception const &e) { - LOGW_SYNCPAL_WARN(_logger, "Error in UploadJob::UploadJob for driveDbId=" << _syncPal->driveDbId() << " : " - << e.what()); + LOGW_SYNCPAL_WARN(_logger, L"Error in UploadJob::UploadJob for driveDbId=" << _syncPal->driveDbId() << L" : " + << Utility::s2ws(e.what())); return ExitCode::DataError; } @@ -1193,8 +1191,8 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & std::shared_ptr parentNode = syncOp->newParentNode() ? syncOp->newParentNode() : syncOp->affectedNode()->parentNode(); if (!parentNode) { - LOGW_SYNCPAL_WARN(_logger, L"Parent node not found for item with " - << Utility::formatSyncPath(correspondingNode->getPath())); + LOGW_SYNCPAL_WARN(_logger, + L"Parent node not found for item with " << Utility::formatSyncPath(correspondingNode->getPath())); return ExitCode::DataError; } @@ -1211,16 +1209,15 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & << Utility::formatSyncPath(syncOp->affectedNode()->getPath())); return ExitCode::DataError; } - } - // Get the new parent node - std::shared_ptr parentNode = - syncOp->newParentNode() ? syncOp->newParentNode() : syncOp->affectedNode()->parentNode(); - if (!parentNode) { - LOGW_SYNCPAL_WARN(_logger, L"Parent node not found for item " - << Utility::formatSyncPath(correspondingNode->getPath())); - return ExitCode::DataError; - } - } + + // Get the new parent node + std::shared_ptr parentNode = + syncOp->newParentNode() ? syncOp->newParentNode() : syncOp->affectedNode()->parentNode(); + if (!parentNode) { + LOGW_SYNCPAL_WARN(_logger, + L"Parent node not found for item " << Utility::formatSyncPath(correspondingNode->getPath())); + return ExitCode::DataError; + } relativeDestLocalFilePath = parentNode->getPath() / syncOp->newName(); relativeOriginLocalFilePath = correspondingNode->getPath(); @@ -1234,15 +1231,14 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & job = std::make_shared(_syncPal->driveDbId(), correspondingNode->id().has_value() ? *correspondingNode->id() : std::string(), absoluteDestLocalFilePath); - } catch (const std::exception &e) { - LOG_SYNCPAL_WARN(_logger, "Error in RenameJob::RenameJob for driveDbId=" << _syncPal->driveDbId() - << " error=" << e.what()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + } catch (std::exception const &e) { + LOGW_SYNCPAL_WARN(_logger, L"Error in RenameJob::RenameJob for driveDbId=" << _syncPal->driveDbId() << L" : " + << Utility::s2ws(e.what()).c_str()); + return ExitCode::DataError; } } else { // This is a move + // For all conflicts involving an "undo move" operation, the correct parent is already stored in the syncOp std::shared_ptr remoteParentNode = syncOp->conflict().type() == ConflictType::MoveParentDelete || syncOp->conflict().type() == ConflictType::MoveMoveSource || @@ -1254,16 +1250,13 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & return ExitCode::DataError; } try { - } job = std::make_shared(_syncPal->driveDbId(), absoluteDestLocalFilePath, correspondingNode->id().has_value() ? *correspondingNode->id() : std::string(), remoteParentNode->id().has_value() ? *remoteParentNode->id() : std::string(), syncOp->newName()); } catch (std::exception &e) { - LOG_SYNCPAL_WARN(_logger, "Error in GetTokenFromAppPasswordJob::GetTokenFromAppPasswordJob: error=" << e.what()); - _executorExitCode = ExitCode::DataError; - _executorExitCause = ExitCause::Unknown; - return false; + LOGW_SYNCPAL_WARN(_logger, "Error in GetTokenFromAppPasswordJob::GetTokenFromAppPasswordJob: error=" << e.what()); + return ExitCode::DataError; } } @@ -1277,12 +1270,12 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & std::bind(&SyncPal::vfsForceStatus, _syncPal, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4); job->setVfsForceStatusCallback(vfsForceStatusCallback); + std::function vfsStatusCallback = std::bind(&SyncPal::vfsStatus, _syncPal, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4, std::placeholders::_5); job->setVfsStatusCallback(vfsStatusCallback); } - } } job->setAffectedFilePath(relativeDestLocalFilePath); @@ -1361,9 +1354,9 @@ ExitInfo ExecutorWorker::handleDeleteOp(SyncOpPtr syncOp, bool &ignored, bool &b } if (ExitInfo exitInfo = propagateDeleteToDbAndTree(syncOp); !exitInfo) { + LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; - return; } } else { bool exists = false; @@ -1393,9 +1386,9 @@ ExitInfo ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool bool isHydrated = false; bool isSyncing = false; int progress = 0; + if (!_syncPal->vfsStatus(absoluteLocalFilePath, isPlaceholder, isHydrated, isSyncing, progress)) { LOGW_SYNCPAL_WARN(_logger, L"Error in vfsStatus: " << Utility::formatSyncPath(absoluteLocalFilePath)); return {ExitCode::SystemError, ExitCause::FileAccessError}; - return false; } isDehydratedPlaceholder = isPlaceholder && !isHydrated; } @@ -1411,11 +1404,11 @@ ExitInfo ExecutorWorker::generateDeleteJob(SyncOpPtr syncOp, bool &ignored, bool try { job = std::make_shared( _syncPal->driveDbId(), syncOp->correspondingNode()->id() ? *syncOp->correspondingNode()->id() : std::string(), - } catch (const std::exception &e) { - LOG_SYNCPAL_WARN(_logger, - "Error in DeleteJob::DeleteJob for driveDbId=" << _syncPal->driveDbId() << " error=" << e.what()); + syncOp->affectedNode()->id() ? *syncOp->affectedNode()->id() : std::string(), absoluteLocalFilePath); + } catch (std::exception const &e) { + LOGW_SYNCPAL_WARN(_logger, L"Error in DeleteJob::DeleteJob for driveDbId=" << _syncPal->driveDbId() << L" : " + << Utility::s2ws(e.what())); return ExitCode::DataError; - return false; } } @@ -1437,6 +1430,7 @@ bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { SyncPath relativeLocalFilePath = syncOp->nodePath(ReplicaSide::Local); SyncPath absoluteLocalFilePath = _syncPal->localPath() / relativeLocalFilePath; + IoError ioError = IoError::Success; if (!IoHelper::checkIfPathExists(absoluteLocalFilePath, exists, ioError)) { LOGW_WARN(_logger, L"Error in Utility::checkIfPathExists: " << Utility::formatSyncPath(absoluteLocalFilePath)); return false; @@ -1444,7 +1438,7 @@ bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { exists = ioError != IoError::NoSuchFileOrDirectory; if (syncOp->targetSide() == ReplicaSide::Local) { - } + switch (syncOp->type()) { case OperationType::Edit: case OperationType::Move: case OperationType::Delete: { @@ -1468,7 +1462,7 @@ bool ExecutorWorker::hasRight(SyncOpPtr syncOp, bool &exists) { LOGW_SYNCPAL_WARN(_logger, L"File/directory " << Path2WStr(absoluteLocalFilePath) << L" doesn't exist anymore!"); return false; - + } std::shared_ptr newCorrespondingParentNode = nullptr; if (affectedUpdateTree(syncOp)->rootNode() == syncOp->affectedNode()->parentNode()) { newCorrespondingParentNode = targetUpdateTree(syncOp)->rootNode(); @@ -1655,17 +1649,17 @@ ExitInfo ExecutorWorker::handleManagedBackError(ExitCause jobExitCause, SyncOpPt syncOp->affectedNode()->getPath(), otherSide(syncOp->targetSide())); } - LOGW_SYNCPAL_WARN( - _logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name())); + if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { + LOGW_SYNCPAL_WARN(_logger, + L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; - return false; } if (syncOp->correspondingNode()) { if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name())); return ExitCode::DataError; - return false; } } @@ -1733,13 +1727,13 @@ ExitInfo ExecutorWorker::handleFinishedJob(std::shared_ptr job, Syn if (job->exitCode() != ExitCode::Ok) { if (networkJob && (networkJob->getStatusCode() == Poco::Net::HTTPResponse::HTTP_FORBIDDEN || + networkJob->getStatusCode() == Poco::Net::HTTPResponse::HTTP_CONFLICT)) { if (ExitInfo exitInfo = handleForbiddenAction(syncOp, relativeLocalPath, ignored); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Error in handleForbiddenAction for item: " << Utility::formatSyncPath(relativeLocalPath)); return exitInfo; - return false; + } } else if (!handleExecutorError(syncOp, {job->exitCode(), job->exitCause()})) { - } else { // Cancel all queued jobs LOGW_SYNCPAL_WARN(_logger, L"Cancelling jobs. exit code: " << job->exitCode() << L" exit cause: " << job->exitCause()); @@ -1852,6 +1846,7 @@ ExitInfo ExecutorWorker::handleForbiddenAction(SyncOpPtr syncOp, const SyncPath // original location on next sync _syncPal->setRestart(true); if (ExitInfo exitInfo = propagateDeleteToDbAndTree(syncOp); !exitInfo) { + LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } @@ -1865,11 +1860,9 @@ void ExecutorWorker::sendProgress() { _fileProgressTimer = std::chrono::steady_clock::now(); for (const auto &jobInfo: _ongoingJobs) { - if (jobInfo.second->isProgressTracked() && jobInfo.second->progressChanged()) { - if (!_syncPal->setProgress(jobInfo.second->affectedFilePath(), jobInfo.second->getProgress())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in SyncPal::setProgress: " - << Utility::formatSyncPath(jobInfo.second->affectedFilePath())); - } + if (!_syncPal->setProgress(jobInfo.second->affectedFilePath(), jobInfo.second->getProgress())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in SyncPal::setProgress: " + << Utility::formatSyncPath(jobInfo.second->affectedFilePath())); } } } @@ -1904,17 +1897,17 @@ ExitInfo ExecutorWorker::propagateConflictToDbAndTree(SyncOpPtr syncOp, bool &pr } } } - // Remove node from update tree - if (!_syncPal->updateTree(ReplicaSide::Local)->deleteNode(syncOp->conflict().localNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->conflict().localNode()->name())); - } - } - if (!_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(syncOp->conflict().remoteNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->conflict().remoteNode()->name())); - } - } + + // Remove node from update tree + if (!_syncPal->updateTree(ReplicaSide::Local)->deleteNode(syncOp->conflict().localNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->conflict().localNode()->name())); + } + + if (!_syncPal->updateTree(ReplicaSide::Remote)->deleteNode(syncOp->conflict().remoteNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" + << SyncName2WStr(syncOp->conflict().remoteNode()->name())); + } propagateChange = false; break; @@ -2012,9 +2005,9 @@ ExitInfo ExecutorWorker::propagateChangeToDbAndTree(SyncOpPtr syncOp, std::share return propagateDeleteToDbAndTree(syncOp); } default: { + LOGW_SYNCPAL_WARN(_logger, L"Unknown operation type " << syncOp->type() << L" on file " << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::SystemError; - return false; } } return ExitCode::LogicError; @@ -2046,9 +2039,9 @@ ExitInfo ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const Node SyncName remoteName = syncOp->targetSide() == ReplicaSide::Remote ? syncOp->newName() : syncOp->nodeName(ReplicaSide::Remote); if (localId.empty() || remoteId.empty()) { + LOGW_SYNCPAL_WARN(_logger, L"Empty " << (localId.empty() ? L"local" : L"remote") << L" id for item " << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; - return false; } DbNode dbNode(0, newCorrespondingParentNode->idb(), localName, remoteName, localId, remoteId, @@ -2148,10 +2141,10 @@ ExitInfo ExecutorWorker::propagateCreateToDbAndTree(SyncOpPtr syncOp, const Node updateTree->insertNode(node); if (!newCorrespondingParentNode->insertChildren(node)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in Node::insertChildren: node name=" << SyncName2WStr(node->name()) << L" parent node name=" << SyncName2WStr(newCorrespondingParentNode->name())); return ExitCode::DataError; - return false; } // Affected node does not have a valid DB ID yet, update it @@ -2185,9 +2178,9 @@ ExitInfo ExecutorWorker::propagateEditToDbAndTree(SyncOpPtr syncOp, const NodeId const SyncName remoteName = syncOp->nodeName(ReplicaSide::Remote); if (localId.empty() || remoteId.empty()) { + LOGW_SYNCPAL_WARN(_logger, L"Empty " << (localId.empty() ? L"local" : L"remote") << L" id for item " << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; - return false; } dbNode.setNodeIdLocal(localId); @@ -2265,9 +2258,9 @@ ExitInfo ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { : (syncOp->correspondingNode() ? correspondingNodeInOtherTree(syncOp->affectedNode()->parentNode()) : correspondingNode->parentNode()); if (!parentNode) { + LOGW_SYNCPAL_DEBUG(_logger, L"Failed to get corresponding parent node: " << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; - return false; } std::string localId = syncOp->targetSide() == ReplicaSide::Local @@ -2282,9 +2275,9 @@ ExitInfo ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { SyncName remoteName = syncOp->targetSide() == ReplicaSide::Remote ? syncOp->newName() : syncOp->nodeName(ReplicaSide::Remote); if (localId.empty() || remoteId.empty()) { - << SyncName2WStr(syncOp->affectedNode()->name()))); + LOGW_SYNCPAL_WARN(_logger, L"Empty " << (localId.empty() ? L"local" : L"remote") << L" id for item " + << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; - return false; } dbNode.setParentNodeId(parentNode->idb()); @@ -2329,13 +2322,15 @@ ExitInfo ExecutorWorker::propagateMoveToDbAndTree(SyncOpPtr syncOp) { correspondingNode->setName(remoteName); - LOGW_SYNCPAL_WARN(_logger, L"Error in Node::setParentNode: node name=" - << SyncName2WStr(parentNode->name()) << L" parent node name=" - << SyncName2WStr(correspondingNode->name())); + if (!correspondingNode->setParentNode(parentNode)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in Node::setParentNode: node name=" << SyncName2WStr(parentNode->name()) + << L" parent node name=" + << SyncName2WStr(correspondingNode->name())); return ExitCode::DataError; } if (!correspondingNode->parentNode()->insertChildren(correspondingNode)) { + LOGW_SYNCPAL_WARN(_logger, L"Error in Node::insertChildren: node name=" << SyncName2WStr(correspondingNode->name()) << L" parent node name=" << SyncName2WStr(correspondingNode->parentNode()->name())); return ExitCode::DataError; @@ -2353,13 +2348,15 @@ ExitInfo ExecutorWorker::propagateDeleteToDbAndTree(SyncOpPtr syncOp) { } // 3. Remove nX and nY from the update tree structures. - LOGW_SYNCPAL_WARN( - _logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name())); + if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { + LOGW_SYNCPAL_WARN(_logger, + L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; } - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); + if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { + LOGW_SYNCPAL_WARN(_logger, + L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name())); return ExitCode::DataError; } @@ -2367,6 +2364,7 @@ ExitInfo ExecutorWorker::propagateDeleteToDbAndTree(SyncOpPtr syncOp) { } ExitInfo ExecutorWorker::deleteFromDb(std::shared_ptr node) { + if (!node->idb().has_value()) { LOGW_SYNCPAL_WARN(_logger, L"Node " << SyncName2WStr(node->name()) << L" does not have a DB ID"); return {ExitCode::DataError, ExitCause::DbEntryNotFound}; } @@ -2409,13 +2407,15 @@ ExitInfo ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptraddError(error); - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->affectedNode()->name())); + if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { + LOGW_SYNCPAL_WARN( + _logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; } if (syncOp->correspondingNode()) { if (!targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { + LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->correspondingNode()->name())); return ExitCode::DataError; } @@ -2423,10 +2423,10 @@ ExitInfo ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptrexitCode() != ExitCode::Ok) { - LOGW_SYNCPAL_WARN(_logger, L"Failed to create directory: " << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + LOGW_SYNCPAL_WARN(_logger, L"Failed to create directory: " << SyncName2WStr(syncOp->affectedNode()->name())); return {job->exitCode(), job->exitCause()}; - return false; } NodeId newNodeId; @@ -2442,17 +2442,16 @@ ExitInfo ExecutorWorker::runCreateDirJob(SyncOpPtr syncOp, std::shared_ptrmodtime(); } - LOGW_SYNCPAL_WARN(_logger, - L"Failed to retreive ID for directory: " << SyncName2WStr(syncOp->affectedNode()->name())); + if (newNodeId.empty()) { + LOGW_SYNCPAL_WARN(_logger, L"Failed to retreive ID for directory: " << SyncName2WStr(syncOp->affectedNode()->name())); return {ExitCode::DataError, ExitCause::ApiErr}; - return false; } std::shared_ptr newNode = nullptr; if (ExitInfo exitInfo = propagateCreateToDbAndTree(syncOp, newNodeId, newModTime, newNode); !exitInfo) { + LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; - return false; } return ExitCode::Ok; @@ -2529,23 +2528,24 @@ void ExecutorWorker::increaseErrorCount(SyncOpPtr syncOp) { ExitInfo ExecutorWorker::getFileSize(const SyncPath &path, uint64_t &size) { IoError ioError = IoError::Unknown; + if (!IoHelper::getFileSize(path, size, ioError)) { LOGW_WARN(_logger, L"Error in IoHelper::getFileSize for " << Utility::formatIoError(path, ioError)); return ExitCode::SystemError; - return false; } + if (ioError == IoError::NoSuchFileOrDirectory) { // The synchronization will // be re-started. LOGW_WARN(_logger, L"File doesn't exist: " << Utility::formatSyncPath(path)); return ExitCode::DataError; - return false; } + if (ioError == IoError::AccessDenied) { // An action from the user is requested. LOGW_WARN(_logger, L"File search permission missing: " << Utility::formatSyncPath(path)); return {ExitCode::SystemError, ExitCause::FileAccessError}; - return false; } assert(ioError == IoError::Success); + if (ioError != IoError::Success) { LOGW_WARN(_logger, L"Unable to read file size for " << Utility::formatSyncPath(path)); return ExitCode::SystemError; } @@ -2626,14 +2626,14 @@ ExitInfo ExecutorWorker::handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo o syncOp->affectedNode()->id().has_value() ? *syncOp->affectedNode()->id() : std::string(), relativeLocalPath, ReplicaSide::Remote); if (!affectedUpdateTree(syncOp)->deleteNode(syncOp->affectedNode())) { - LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + LOGW_SYNCPAL_WARN( + _logger, L"Error in UpdateTree::deleteNode: node name=" << SyncName2WStr(syncOp->affectedNode()->name())); return ExitCode::DataError; } if (syncOp->correspondingNode() && !targetUpdateTree(syncOp)->deleteNode(syncOp->correspondingNode())) { LOGW_SYNCPAL_WARN(_logger, L"Error in UpdateTree::deleteNode: node name=" - << SyncName2WStr(syncOp->correspondingNode()->name()).c_str()); + << SyncName2WStr(syncOp->correspondingNode()->name())); return ExitCode::DataError; } _syncPal->setRestart(true); @@ -2662,7 +2662,7 @@ ExitInfo ExecutorWorker::handleOpsAlreadyExistError(SyncOpPtr syncOp, ExitInfo o _syncPal->setRestart(true); if (ExitInfo exitInfo = propagateDeleteToDbAndTree(syncOp); !exitInfo) { LOGW_SYNCPAL_WARN(_logger, L"Failed to propagate changes in DB or update tree for: " - << SyncName2WStr(syncOp->affectedNode()->name()).c_str()); + << SyncName2WStr(syncOp->affectedNode()->name())); return exitInfo; } return removeDependentOps(syncOp); @@ -2711,7 +2711,6 @@ ExitInfo ExecutorWorker::removeDependentOps(SyncOpPtr syncOp) { } if (!dependentOps.empty()) { LOGW_SYNCPAL_DEBUG(_logger, L"Removed " << dependentOps.size() << L" dependent operations."); - return false; } return ExitCode::Ok; From e09c87c48a98f3671f335d9531dc0cb4bd42592b Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 31 Oct 2024 08:32:12 +0100 Subject: [PATCH 52/56] Only setProgressComplete if bypassProgressComplete is false. --- src/libsyncengine/propagation/executor/executorworker.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 8bafca077..4084f7558 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -149,7 +149,7 @@ void ExecutorWorker::execute() { // If an operation fails but is correctly handled by handleExecutorError, execution can proceed. if (executorExitInfo.cause() == ExitCause::OperationCanceled) { - setProgressComplete(syncOp, SyncFileStatus::Error); + if (!bypassProgressComplete) setProgressComplete(syncOp, SyncFileStatus::Error); continue; } @@ -160,12 +160,10 @@ void ExecutorWorker::execute() { cancelAllOngoingJobs(); break; } else { // If the error is handled, continue the execution - setProgressComplete(syncOp, SyncFileStatus::Error); + if (!bypassProgressComplete) setProgressComplete(syncOp, SyncFileStatus::Error); continue; } - if (!bypassProgressComplete) { - setProgressComplete(syncOp, SyncFileStatus::Error); - } + if (!bypassProgressComplete) setProgressComplete(syncOp, SyncFileStatus::Error); } if (job) { From 4a4342c1dafa6958e44ae59e2d4f6f1d62632c16 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 31 Oct 2024 10:43:06 +0100 Subject: [PATCH 53/56] Fix LOGW error --- src/libsyncengine/propagation/executor/executorworker.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyncengine/propagation/executor/executorworker.cpp b/src/libsyncengine/propagation/executor/executorworker.cpp index 4084f7558..f0f519f43 100644 --- a/src/libsyncengine/propagation/executor/executorworker.cpp +++ b/src/libsyncengine/propagation/executor/executorworker.cpp @@ -1231,7 +1231,7 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & absoluteDestLocalFilePath); } catch (std::exception const &e) { LOGW_SYNCPAL_WARN(_logger, L"Error in RenameJob::RenameJob for driveDbId=" << _syncPal->driveDbId() << L" : " - << Utility::s2ws(e.what()).c_str()); + << Utility::s2ws(e.what())); return ExitCode::DataError; } } else { @@ -1253,7 +1253,7 @@ ExitInfo ExecutorWorker::generateMoveJob(SyncOpPtr syncOp, bool &ignored, bool & remoteParentNode->id().has_value() ? *remoteParentNode->id() : std::string(), syncOp->newName()); } catch (std::exception &e) { - LOGW_SYNCPAL_WARN(_logger, "Error in GetTokenFromAppPasswordJob::GetTokenFromAppPasswordJob: error=" << e.what()); + LOG_SYNCPAL_WARN(_logger, "Error in GetTokenFromAppPasswordJob::GetTokenFromAppPasswordJob: error=" << e.what()); return ExitCode::DataError; } } From 46ea81293e50d75fce2481d0110a7e8d108ded42 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 31 Oct 2024 10:55:29 +0100 Subject: [PATCH 54/56] fixTestTypes. --- test/libcommon/utility/testtypes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/libcommon/utility/testtypes.cpp b/test/libcommon/utility/testtypes.cpp index 7a678b2de..5f9188aec 100644 --- a/test/libcommon/utility/testtypes.cpp +++ b/test/libcommon/utility/testtypes.cpp @@ -87,7 +87,7 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, ec); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); - CPPUNIT_ASSERT_EQUAL(100, static_cast(ei)); + CPPUNIT_ASSERT_EQUAL(0, static_cast(ei)); ei = {ExitCode::NetworkError, ExitCause::DbAccessError}; ec = ei; From 724f998afebd5ae954b70372fd8d3f4d77a8798d Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 31 Oct 2024 10:55:29 +0100 Subject: [PATCH 55/56] fixTestTypes. --- test/libcommon/utility/testtypes.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libcommon/utility/testtypes.cpp b/test/libcommon/utility/testtypes.cpp index 7a678b2de..d7de97b4e 100644 --- a/test/libcommon/utility/testtypes.cpp +++ b/test/libcommon/utility/testtypes.cpp @@ -78,7 +78,7 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCode::Unknown, ec); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); - CPPUNIT_ASSERT_EQUAL(0, static_cast(ei)); + CPPUNIT_ASSERT_EQUAL(100, static_cast(ei)); ei = {ExitCode::Ok}; ec = ei; @@ -87,7 +87,7 @@ void TestTypes::testExitInfo() { CPPUNIT_ASSERT_EQUAL(ExitCode::Ok, ec); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, ei.cause()); CPPUNIT_ASSERT_EQUAL(ExitCause::Unknown, eca); - CPPUNIT_ASSERT_EQUAL(100, static_cast(ei)); + CPPUNIT_ASSERT_EQUAL(0, static_cast(ei)); ei = {ExitCode::NetworkError, ExitCause::DbAccessError}; ec = ei; From 64c01a5461790b485612afcf17d8b2e15831f6e5 Mon Sep 17 00:00:00 2001 From: Herve Eruam Date: Thu, 31 Oct 2024 11:48:21 +0100 Subject: [PATCH 56/56] Remove redundant check of tmpblacklist. --- .../file_system_observer/localfilesystemobserverworker.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp index 755f43273..ee8d367b6 100644 --- a/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp +++ b/src/libsyncengine/update_detection/file_system_observer/localfilesystemobserverworker.cpp @@ -87,13 +87,6 @@ void LocalFileSystemObserverWorker::changesDetected(const std::listlocalPath(), absolutePath); - if (_syncPal->isTmpBlacklisted(relativePath, ReplicaSide::Local)) { - _syncPal->removeItemFromTmpBlacklist(relativePath); - if (opTypeFromOS == OperationType::Edit) { - NodeId itemId = _snapshot->itemId(relativePath); - if (!itemId.empty()) _snapshot->setLastModified(itemId, 0); - } - } // Check if exists with same nodeId if (opTypeFromOS == OperationType::Delete) { NodeId prevNodeId = _snapshot->itemId(relativePath);