diff --git a/src/gui/socketapi.cpp b/src/gui/socketapi.cpp index 3be2d1455c09d..b1b4e7eaa7efa 100644 --- a/src/gui/socketapi.cpp +++ b/src/gui/socketapi.cpp @@ -1178,8 +1178,11 @@ DirectEditor* SocketApi::getDirectEditorForLocalFile(const QString &localFile) auto capabilities = fileData.folder->accountState()->account()->capabilities(); if (fileData.folder && fileData.folder->accountState()->isConnected()) { + const auto record = fileData.journalRecord(); + const auto mimeMatchMode = record.isVirtualFile() ? QMimeDatabase::MatchExtension : QMimeDatabase::MatchDefault; + QMimeDatabase db; - QMimeType type = db.mimeTypeForFile(localFile); + QMimeType type = db.mimeTypeForFile(localFile, mimeMatchMode); DirectEditor* editor = capabilities.getDirectEditorForMimetype(type); if (!editor) { diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 35a4a9300be21..6679d917ee0d3 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -61,6 +61,16 @@ set(libsync_SRCS vfs/suffix/vfs_suffix.cpp ) +if (WIN32) + set(libsync_SRCS ${libsync_SRCS} + vfs/cfapi/cfapiwrapper.cpp + vfs/cfapi/hydrationjob.cpp + vfs/cfapi/vfs_cfapi.cpp + ) + add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_WIN10) + list(APPEND OS_SPECIFIC_LINK_LIBRARIES cldapi) +endif() + if(TOKEN_AUTH_ONLY) set (libsync_SRCS ${libsync_SRCS} creds/tokencredentials.cpp) else() diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 6bc3ba59262ca..09f0dae2d085f 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -860,8 +860,8 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( auto postProcessLocalNew = [item, localEntry, this]() { if (localEntry.isVirtualFile) { // Remove the spurious file if it looks like a placeholder file - // (we know placeholder files contain " ") - if (localEntry.size <= 1) { + // (we know placeholder files contain " ", but only in the suffix case) + if (localEntry.size <= 1 || !isVfsWithSuffix()) { qCWarning(lcDisco) << "Wiping virtual file without db entry for" << _currentFolder._local + "/" + localEntry.name; item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Down; diff --git a/src/libsync/theme.cpp b/src/libsync/theme.cpp index fa0ccb0c4da55..97f0b3697a0c5 100644 --- a/src/libsync/theme.cpp +++ b/src/libsync/theme.cpp @@ -738,7 +738,8 @@ QPixmap Theme::createColorAwarePixmap(const QString &name) bool Theme::showVirtualFilesOption() const { - return ConfigFile().showExperimentalOptions(); + const auto vfsMode = bestAvailableVfsMode(); + return ConfigFile().showExperimentalOptions() || vfsMode == Vfs::WindowsCfApi; } } // end namespace client diff --git a/src/libsync/vfs/cfapi/cfapiwrapper.cpp b/src/libsync/vfs/cfapi/cfapiwrapper.cpp new file mode 100644 index 0000000000000..62ffcffa1c426 --- /dev/null +++ b/src/libsync/vfs/cfapi/cfapiwrapper.cpp @@ -0,0 +1,501 @@ +/* + * Copyright (C) by Kevin Ottens + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "cfapiwrapper.h" + +#include "common/utility.h" +#include "hydrationjob.h" +#include "vfs_cfapi.h" + +#include +#include +#include +#include + +#include +#include +#include + +Q_LOGGING_CATEGORY(lcCfApiWrapper, "nextcloud.sync.vfs.cfapi.wrapper", QtInfoMsg) + +#define FIELD_SIZE( type, field ) ( sizeof( ( (type*)0 )->field ) ) +#define CF_SIZE_OF_OP_PARAM( field ) \ + ( FIELD_OFFSET( CF_OPERATION_PARAMETERS, field ) + \ + FIELD_SIZE( CF_OPERATION_PARAMETERS, field ) ) + +namespace { +void cfApiSendTransferInfo(const CF_CONNECTION_KEY &connectionKey, const CF_TRANSFER_KEY &transferKey, NTSTATUS status, void *buffer, qint64 offset, qint64 length) +{ + + CF_OPERATION_INFO opInfo = { 0 }; + CF_OPERATION_PARAMETERS opParams = { 0 }; + + opInfo.StructSize = sizeof(opInfo); + opInfo.Type = CF_OPERATION_TYPE_TRANSFER_DATA; + opInfo.ConnectionKey = connectionKey; + opInfo.TransferKey = transferKey; + opParams.ParamSize = CF_SIZE_OF_OP_PARAM(TransferData); + opParams.TransferData.CompletionStatus = status; + opParams.TransferData.Buffer = buffer; + opParams.TransferData.Offset.QuadPart = offset; + opParams.TransferData.Length.QuadPart = length; + + const qint64 result = CfExecute(&opInfo, &opParams); + Q_ASSERT(result == S_OK); + if (result != S_OK) { + qCCritical(lcCfApiWrapper) << "Couldn't send transfer info" << QString::number(transferKey.QuadPart, 16) << ":" << _com_error(result).ErrorMessage(); + } +} + + +void CALLBACK cfApiFetchDataCallback(const CF_CALLBACK_INFO *callbackInfo, const CF_CALLBACK_PARAMETERS *callbackParameters) +{ + const auto sendTransferError = [=] { + cfApiSendTransferInfo(callbackInfo->ConnectionKey, + callbackInfo->TransferKey, + STATUS_UNSUCCESSFUL, + nullptr, + callbackParameters->FetchData.RequiredFileOffset.QuadPart, + callbackParameters->FetchData.RequiredLength.QuadPart); + }; + + const auto sendTransferInfo = [=](QByteArray &data, qint64 offset) { + cfApiSendTransferInfo(callbackInfo->ConnectionKey, + callbackInfo->TransferKey, + STATUS_SUCCESS, + data.data(), + offset, + data.length()); + }; + + auto vfs = reinterpret_cast(callbackInfo->CallbackContext); + Q_ASSERT(vfs->metaObject()->className() == QByteArrayLiteral("OCC::VfsCfApi")); + const auto path = QString(QString::fromWCharArray(callbackInfo->VolumeDosName) + QString::fromWCharArray(callbackInfo->NormalizedPath)); + const auto requestId = QString::number(callbackInfo->TransferKey.QuadPart, 16); + + const auto invokeResult = QMetaObject::invokeMethod(vfs, [=] { vfs->requestHydration(requestId, path); }, Qt::QueuedConnection); + if (!invokeResult) { + qCCritical(lcCfApiWrapper) << "Failed to trigger hydration for" << path << requestId; + sendTransferError(); + return; + } + + // Block and wait for vfs to signal back the hydration is ready + bool hydrationRequestResult = false; + QEventLoop loop; + QObject::connect(vfs, &OCC::VfsCfApi::hydrationRequestReady, &loop, [&](const QString &id) { + if (requestId == id) { + hydrationRequestResult = true; + loop.quit(); + } + }); + QObject::connect(vfs, &OCC::VfsCfApi::hydrationRequestFailed, &loop, [&](const QString &id) { + if (requestId == id) { + hydrationRequestResult = false; + loop.quit(); + } + }); + loop.exec(); + QObject::disconnect(vfs, nullptr, &loop, nullptr); + qCInfo(lcCfApiWrapper) << "VFS replied for hydration of" << path << requestId << "status was:" << hydrationRequestResult; + + if (!hydrationRequestResult) { + sendTransferError(); + return; + } + + QLocalSocket socket; + socket.connectToServer(requestId); + const auto connectResult = socket.waitForConnected(); + if (!connectResult) { + qCWarning(lcCfApiWrapper) << "Couldn't connect the socket" << requestId << socket.error() << socket.errorString(); + sendTransferError(); + return; + } + + qint64 offset = 0; + + QObject::connect(&socket, &QLocalSocket::readyRead, &loop, [&] { + auto data = socket.readAll(); + if (data.isEmpty()) { + qCWarning(lcCfApiWrapper) << "Unexpected empty data received" << requestId; + sendTransferError(); + loop.quit(); + return; + } + sendTransferInfo(data, offset); + offset += data.length(); + }); + + QObject::connect(vfs, &OCC::VfsCfApi::hydrationRequestFinished, &loop, [&](const QString &id, int s) { + if (requestId == id) { + const auto status = static_cast(s); + qCInfo(lcCfApiWrapper) << "Hydration done for" << path << requestId << status; + if (status != OCC::HydrationJob::Success) { + sendTransferError(); + } + loop.quit(); + } + }); + + loop.exec(); +} + +CF_CALLBACK_REGISTRATION cfApiCallbacks[] = { + { CF_CALLBACK_TYPE_FETCH_DATA, cfApiFetchDataCallback }, + CF_CALLBACK_REGISTRATION_END +}; + +DWORD sizeToDWORD(size_t size) +{ + return OCC::Utility::convertSizeToDWORD(size); +} + +void deletePlaceholderInfo(CF_PLACEHOLDER_BASIC_INFO *info) +{ + auto byte = reinterpret_cast(info); + delete[] byte; +} + +std::wstring pathForHandle(const OCC::CfApiWrapper::FileHandle &handle) +{ + wchar_t buffer[MAX_PATH]; + const qint64 result = GetFinalPathNameByHandle(handle.get(), buffer, MAX_PATH, VOLUME_NAME_DOS); + Q_ASSERT(result < MAX_PATH); + return std::wstring(buffer); +} + +OCC::PinState cfPinStateToPinState(CF_PIN_STATE state) +{ + switch (state) { + case CF_PIN_STATE_UNSPECIFIED: + return OCC::PinState::Unspecified; + case CF_PIN_STATE_PINNED: + return OCC::PinState::AlwaysLocal; + case CF_PIN_STATE_UNPINNED: + return OCC::PinState::OnlineOnly; + case CF_PIN_STATE_INHERIT: + return OCC::PinState::Inherited; + default: + Q_UNREACHABLE(); + return OCC::PinState::Inherited; + } +} + +CF_PIN_STATE pinStateToCfPinState(OCC::PinState state) +{ + switch (state) { + case OCC::PinState::Inherited: + return CF_PIN_STATE_INHERIT; + case OCC::PinState::AlwaysLocal: + return CF_PIN_STATE_PINNED; + case OCC::PinState::OnlineOnly: + return CF_PIN_STATE_UNPINNED; + case OCC::PinState::Unspecified: + return CF_PIN_STATE_UNSPECIFIED; + default: + Q_UNREACHABLE(); + return CF_PIN_STATE_UNSPECIFIED; + } +} + +CF_SET_PIN_FLAGS pinRecurseModeToCfSetPinFlags(OCC::CfApiWrapper::SetPinRecurseMode mode) +{ + switch (mode) { + case OCC::CfApiWrapper::NoRecurse: + return CF_SET_PIN_FLAG_NONE; + case OCC::CfApiWrapper::Recurse: + return CF_SET_PIN_FLAG_RECURSE; + case OCC::CfApiWrapper::ChildrenOnly: + return CF_SET_PIN_FLAG_RECURSE_ONLY; + default: + Q_UNREACHABLE(); + return CF_SET_PIN_FLAG_NONE; + } +} + +} + +OCC::CfApiWrapper::ConnectionKey::ConnectionKey() + : _data(new CF_CONNECTION_KEY, [](void *p) { delete reinterpret_cast(p); }) +{ +} + +OCC::CfApiWrapper::FileHandle::FileHandle() + : _data(nullptr, [](void *) {}) +{ +} + +OCC::CfApiWrapper::FileHandle::FileHandle(void *data, Deleter deleter) + : _data(data, deleter) +{ +} + +OCC::CfApiWrapper::PlaceHolderInfo::PlaceHolderInfo() + : _data(nullptr, [](CF_PLACEHOLDER_BASIC_INFO *) {}) +{ +} + +OCC::CfApiWrapper::PlaceHolderInfo::PlaceHolderInfo(CF_PLACEHOLDER_BASIC_INFO *data, Deleter deleter) + : _data(data, deleter) +{ +} + +OCC::Optional OCC::CfApiWrapper::PlaceHolderInfo::pinState() const +{ + Q_ASSERT(_data); + if (!_data) { + return {}; + } + + return cfPinStateToPinState(_data->PinState); +} + +OCC::Result OCC::CfApiWrapper::registerSyncRoot(const QString &path, const QString &providerName, const QString &providerVersion) +{ + const auto p = path.toStdWString(); + const auto name = providerName.toStdWString(); + const auto version = providerVersion.toStdWString(); + + CF_SYNC_REGISTRATION info; + info.ProviderName = name.data(); + info.ProviderVersion = version.data(); + info.SyncRootIdentity = nullptr; + info.SyncRootIdentityLength = 0; + info.FileIdentity = nullptr; + info.FileIdentityLength = 0; + + CF_SYNC_POLICIES policies; + policies.Hydration.Primary = CF_HYDRATION_POLICY_FULL; + policies.Hydration.Modifier = CF_HYDRATION_POLICY_MODIFIER_NONE; + policies.Population.Primary = CF_POPULATION_POLICY_ALWAYS_FULL; + policies.Population.Modifier = CF_POPULATION_POLICY_MODIFIER_NONE; + policies.InSync = CF_INSYNC_POLICY_PRESERVE_INSYNC_FOR_SYNC_ENGINE; + policies.HardLink = CF_HARDLINK_POLICY_NONE; + + const qint64 result = CfRegisterSyncRoot(p.data(), &info, &policies, CF_REGISTER_FLAG_UPDATE); + Q_ASSERT(result == S_OK); + if (result != S_OK) { + return QString::fromWCharArray(_com_error(result).ErrorMessage()); + } else { + return {}; + } +} + +OCC::Result OCC::CfApiWrapper::unegisterSyncRoot(const QString &path) +{ + const auto p = path.toStdWString(); + const qint64 result = CfUnregisterSyncRoot(p.data()); + Q_ASSERT(result == S_OK); + if (result != S_OK) { + return QString::fromWCharArray(_com_error(result).ErrorMessage()); + } else { + return {}; + } +} + +OCC::Result OCC::CfApiWrapper::connectSyncRoot(const QString &path, OCC::VfsCfApi *context) +{ + auto key = ConnectionKey(); + const auto p = path.toStdWString(); + const qint64 result = CfConnectSyncRoot(p.data(), + cfApiCallbacks, + context, + CF_CONNECT_FLAG_REQUIRE_PROCESS_INFO | CF_CONNECT_FLAG_REQUIRE_FULL_FILE_PATH, + static_cast(key.get())); + Q_ASSERT(result == S_OK); + if (result != S_OK) { + return QString::fromWCharArray(_com_error(result).ErrorMessage()); + } else { + return key; + } +} + +OCC::Result OCC::CfApiWrapper::disconnectSyncRoot(ConnectionKey &&key) +{ + const qint64 result = CfDisconnectSyncRoot(*static_cast(key.get())); + Q_ASSERT(result == S_OK); + if (result != S_OK) { + return QString::fromWCharArray(_com_error(result).ErrorMessage()); + } else { + return {}; + } +} + +bool OCC::CfApiWrapper::isSparseFile(const QString &path) +{ + const auto p = path.toStdWString(); + const auto attributes = GetFileAttributes(p.data()); + return (attributes & FILE_ATTRIBUTE_SPARSE_FILE) != 0; +} + +OCC::CfApiWrapper::FileHandle OCC::CfApiWrapper::handleForPath(const QString &path) +{ + if (QFileInfo(path).isDir()) { + HANDLE handle = nullptr; + const qint64 openResult = CfOpenFileWithOplock(path.toStdWString().data(), CF_OPEN_FILE_FLAG_NONE, &handle); + if (openResult == S_OK) { + return {handle, CfCloseHandle}; + } + } else { + const auto handle = CreateFile(path.toStdWString().data(), 0, 0, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (handle != INVALID_HANDLE_VALUE) { + return {handle, [](HANDLE h) { CloseHandle(h); }}; + } + } + + return {}; +} + +OCC::CfApiWrapper::PlaceHolderInfo OCC::CfApiWrapper::findPlaceholderInfo(const FileHandle &handle) +{ + Q_ASSERT(handle); + + constexpr auto fileIdMaxLength = 128; + const auto infoSize = sizeof(CF_PLACEHOLDER_BASIC_INFO) + fileIdMaxLength; + auto info = PlaceHolderInfo(reinterpret_cast(new char[infoSize]), deletePlaceholderInfo); + const qint64 result = CfGetPlaceholderInfo(handle.get(), CF_PLACEHOLDER_INFO_BASIC, info.get(), sizeToDWORD(infoSize), nullptr); + + if (result == S_OK) { + return info; + } else { + return {}; + } +} + +OCC::Result OCC::CfApiWrapper::setPinState(const FileHandle &handle, PinState state, SetPinRecurseMode mode) +{ + const auto cfState = pinStateToCfPinState(state); + const auto flags = pinRecurseModeToCfSetPinFlags(mode); + + const qint64 result = CfSetPinState(handle.get(), cfState, flags, nullptr); + if (result == S_OK) { + return {}; + } else { + qCWarning(lcCfApiWrapper) << "Couldn't set pin state" << state << "for" << pathForHandle(handle) << "with recurse mode" << mode << ":" << _com_error(result).ErrorMessage(); + return "Couldn't set pin state"; + } +} + +OCC::Result OCC::CfApiWrapper::createPlaceholderInfo(const QString &path, time_t modtime, qint64 size, const QByteArray &fileId) +{ + const auto fileInfo = QFileInfo(path); + const auto localBasePath = QDir::toNativeSeparators(fileInfo.path()).toStdWString(); + const auto relativePath = fileInfo.fileName().toStdWString(); + + const auto fileIdentity = QString::fromUtf8(fileId).toStdWString(); + + CF_PLACEHOLDER_CREATE_INFO cloudEntry; + cloudEntry.FileIdentity = fileIdentity.data(); + const auto fileIdentitySize = (fileIdentity.length() + 1) * sizeof(wchar_t); + cloudEntry.FileIdentityLength = sizeToDWORD(fileIdentitySize); + + cloudEntry.RelativeFileName = relativePath.data(); + cloudEntry.Flags = CF_PLACEHOLDER_CREATE_FLAG_MARK_IN_SYNC; + cloudEntry.FsMetadata.FileSize.QuadPart = size; + cloudEntry.FsMetadata.BasicInfo.FileAttributes = FILE_ATTRIBUTE_NORMAL; + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.CreationTime); + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.LastWriteTime); + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.LastAccessTime); + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &cloudEntry.FsMetadata.BasicInfo.ChangeTime); + + if (fileInfo.isDir()) { + cloudEntry.Flags |= CF_PLACEHOLDER_CREATE_FLAG_DISABLE_ON_DEMAND_POPULATION; + cloudEntry.FsMetadata.BasicInfo.FileAttributes = FILE_ATTRIBUTE_DIRECTORY; + cloudEntry.FsMetadata.FileSize.QuadPart = 0; + } + + const qint64 result = CfCreatePlaceholders(localBasePath.data(), &cloudEntry, 1, CF_CREATE_FLAG_NONE, nullptr); + if (result != S_OK) { + qCWarning(lcCfApiWrapper) << "Couldn't create placeholder info for" << path << ":" << _com_error(result).ErrorMessage(); + return "Couldn't create placeholder info"; + } + + const auto parentHandle = handleForPath(QDir::toNativeSeparators(QFileInfo(path).absolutePath())); + const auto parentInfo = findPlaceholderInfo(parentHandle); + const auto state = parentInfo && parentInfo->PinState == CF_PIN_STATE_UNPINNED ? CF_PIN_STATE_UNPINNED : CF_PIN_STATE_INHERIT; + + const auto handle = handleForPath(path); + if (!setPinState(handle, cfPinStateToPinState(state), NoRecurse)) { + return "Couldn't set the default inherit pin state"; + } + + return {}; +} + +OCC::Result OCC::CfApiWrapper::updatePlaceholderInfo(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath) +{ + Q_ASSERT(handle); + + const auto info = replacesPath.isEmpty() ? findPlaceholderInfo(handle) + : findPlaceholderInfo(handleForPath(replacesPath)); + if (!info) { + return "Can't update non existing placeholder info"; + } + + const auto previousPinState = cfPinStateToPinState(info->PinState); + const auto fileIdentity = QString::fromUtf8(fileId).toStdWString(); + const auto fileIdentitySize = (fileIdentity.length() + 1) * sizeof(wchar_t); + + CF_FS_METADATA metadata; + metadata.FileSize.QuadPart = size; + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.CreationTime); + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.LastWriteTime); + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.LastAccessTime); + OCC::Utility::UnixTimeToLargeIntegerFiletime(modtime, &metadata.BasicInfo.ChangeTime); + + const qint64 result = CfUpdatePlaceholder(handle.get(), &metadata, + fileIdentity.data(), sizeToDWORD(fileIdentitySize), + nullptr, 0, CF_UPDATE_FLAG_NONE, nullptr, nullptr); + + if (result != S_OK) { + qCWarning(lcCfApiWrapper) << "Couldn't update placeholder info for" << pathForHandle(handle) << ":" << _com_error(result).ErrorMessage(); + return "Couldn't update placeholder info"; + } + + // Pin state tends to be lost on updates, so restore it every time + if (!setPinState(handle, previousPinState, NoRecurse)) { + return "Couldn't restore pin state"; + } + + return {}; +} + +OCC::Result OCC::CfApiWrapper::convertToPlaceholder(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath) +{ + Q_ASSERT(handle); + + const auto fileIdentity = QString::fromUtf8(fileId).toStdWString(); + const auto fileIdentitySize = (fileIdentity.length() + 1) * sizeof(wchar_t); + const qint64 result = CfConvertToPlaceholder(handle.get(), fileIdentity.data(), sizeToDWORD(fileIdentitySize), CF_CONVERT_FLAG_NONE, nullptr, nullptr); + Q_ASSERT(result == S_OK); + if (result != S_OK) { + qCCritical(lcCfApiWrapper) << "Couldn't convert to placeholder" << pathForHandle(handle) << ":" << _com_error(result).ErrorMessage(); + return "Couldn't convert to placeholder"; + } + + const auto originalHandle = handleForPath(replacesPath); + const auto originalInfo = originalHandle ? findPlaceholderInfo(originalHandle) : PlaceHolderInfo(nullptr, deletePlaceholderInfo); + if (!originalInfo) { + const auto stateResult = setPinState(handle, PinState::Inherited, NoRecurse); + Q_ASSERT(stateResult); + return stateResult; + } else { + const auto state = cfPinStateToPinState(originalInfo->PinState); + const auto stateResult = setPinState(handle, state, NoRecurse); + Q_ASSERT(stateResult); + return stateResult; + } +} diff --git a/src/libsync/vfs/cfapi/cfapiwrapper.h b/src/libsync/vfs/cfapi/cfapiwrapper.h new file mode 100644 index 0000000000000..888242bf1959b --- /dev/null +++ b/src/libsync/vfs/cfapi/cfapiwrapper.h @@ -0,0 +1,99 @@ +/* + * Copyright (C) by Kevin Ottens + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include + +#include "owncloudlib.h" +#include "common/pinstate.h" +#include "common/result.h" + +struct CF_PLACEHOLDER_BASIC_INFO; + +namespace OCC { + +class VfsCfApi; + +namespace CfApiWrapper +{ + +class OWNCLOUDSYNC_EXPORT ConnectionKey +{ +public: + ConnectionKey(); + inline void *get() const { return _data.get(); } + +private: + std::unique_ptr _data; +}; + +class OWNCLOUDSYNC_EXPORT FileHandle +{ +public: + using Deleter = void (*)(void *); + + FileHandle(); + FileHandle(void *data, Deleter deleter); + + inline void *get() const { return _data.get(); } + inline explicit operator bool() const noexcept { return static_cast(_data); } + +private: + std::unique_ptr _data; +}; + +class OWNCLOUDSYNC_EXPORT PlaceHolderInfo +{ +public: + using Deleter = void (*)(CF_PLACEHOLDER_BASIC_INFO *); + + PlaceHolderInfo(); + PlaceHolderInfo(CF_PLACEHOLDER_BASIC_INFO *data, Deleter deleter); + + inline CF_PLACEHOLDER_BASIC_INFO *get() const noexcept { return _data.get(); } + inline CF_PLACEHOLDER_BASIC_INFO *operator->() const noexcept { return _data.get(); } + inline explicit operator bool() const noexcept { return static_cast(_data); } + + Optional pinState() const; + +private: + std::unique_ptr _data; +}; + +OWNCLOUDSYNC_EXPORT Result registerSyncRoot(const QString &path, const QString &providerName, const QString &providerVersion); +OWNCLOUDSYNC_EXPORT Result unegisterSyncRoot(const QString &path); + +OWNCLOUDSYNC_EXPORT Result connectSyncRoot(const QString &path, VfsCfApi *context); +OWNCLOUDSYNC_EXPORT Result disconnectSyncRoot(ConnectionKey &&key); + +OWNCLOUDSYNC_EXPORT bool isSparseFile(const QString &path); + +OWNCLOUDSYNC_EXPORT FileHandle handleForPath(const QString &path); + +OWNCLOUDSYNC_EXPORT PlaceHolderInfo findPlaceholderInfo(const FileHandle &handle); + +enum SetPinRecurseMode { + NoRecurse = 0, + Recurse, + ChildrenOnly +}; + +OWNCLOUDSYNC_EXPORT Result setPinState(const FileHandle &handle, PinState state, SetPinRecurseMode mode); +OWNCLOUDSYNC_EXPORT Result createPlaceholderInfo(const QString &path, time_t modtime, qint64 size, const QByteArray &fileId); +OWNCLOUDSYNC_EXPORT Result updatePlaceholderInfo(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath = QString()); +OWNCLOUDSYNC_EXPORT Result convertToPlaceholder(const FileHandle &handle, time_t modtime, qint64 size, const QByteArray &fileId, const QString &replacesPath); + +} + +} // namespace OCC diff --git a/src/libsync/vfs/cfapi/hydrationjob.cpp b/src/libsync/vfs/cfapi/hydrationjob.cpp new file mode 100644 index 0000000000000..05af54e47f0a1 --- /dev/null +++ b/src/libsync/vfs/cfapi/hydrationjob.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (C) by Kevin Ottens + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "hydrationjob.h" + +#include "common/syncjournaldb.h" +#include "propagatedownload.h" + +#include +#include + +Q_LOGGING_CATEGORY(lcHydration, "nextcloud.sync.vfs.hydrationjob", QtInfoMsg) + +OCC::HydrationJob::HydrationJob(QObject *parent) + : QObject(parent) +{ + connect(this, &HydrationJob::finished, this, &HydrationJob::deleteLater); +} + +OCC::AccountPtr OCC::HydrationJob::account() const +{ + return _account; +} + +void OCC::HydrationJob::setAccount(const AccountPtr &account) +{ + _account = account; +} + +QString OCC::HydrationJob::remotePath() const +{ + return _remotePath; +} + +void OCC::HydrationJob::setRemotePath(const QString &remotePath) +{ + _remotePath = remotePath; +} + +QString OCC::HydrationJob::localPath() const +{ + return _localPath; +} + +void OCC::HydrationJob::setLocalPath(const QString &localPath) +{ + _localPath = localPath; +} + +OCC::SyncJournalDb *OCC::HydrationJob::journal() const +{ + return _journal; +} + +void OCC::HydrationJob::setJournal(SyncJournalDb *journal) +{ + _journal = journal; +} + +QString OCC::HydrationJob::requestId() const +{ + return _requestId; +} + +void OCC::HydrationJob::setRequestId(const QString &requestId) +{ + _requestId = requestId; +} + +QString OCC::HydrationJob::folderPath() const +{ + return _folderPath; +} + +void OCC::HydrationJob::setFolderPath(const QString &folderPath) +{ + _folderPath = folderPath; +} + +OCC::HydrationJob::Status OCC::HydrationJob::status() const +{ + return _status; +} + +void OCC::HydrationJob::start() +{ + Q_ASSERT(_account); + Q_ASSERT(_journal); + Q_ASSERT(!_remotePath.isEmpty() && !_localPath.isEmpty()); + Q_ASSERT(!_requestId.isEmpty() && !_folderPath.isEmpty()); + + Q_ASSERT(_remotePath.endsWith('/')); + Q_ASSERT(_localPath.endsWith('/')); + Q_ASSERT(!_folderPath.startsWith('/')); + + _server = new QLocalServer(this); + const auto listenResult = _server->listen(_requestId); + if (!listenResult) { + qCCritical(lcHydration) << "Couldn't get server to listen" << _requestId << _localPath << _folderPath; + emitFinished(Error); + return; + } + + qCInfo(lcHydration) << "Server ready, waiting for connections" << _requestId << _localPath << _folderPath; + connect(_server, &QLocalServer::newConnection, this, &HydrationJob::onNewConnection); +} + +void OCC::HydrationJob::emitFinished(Status status) +{ + _status = status; + if (status == Success) { + _socket->disconnectFromServer(); + connect(_socket, &QLocalSocket::disconnected, this, [=]{ + _socket->close(); + emit finished(this); + }); + } else { + _socket->close(); + emit finished(this); + } +} + +void OCC::HydrationJob::onNewConnection() +{ + Q_ASSERT(!_socket); + Q_ASSERT(!_job); + + qCInfo(lcHydration) << "Got new connection starting GETFileJob" << _requestId << _folderPath; + _socket = _server->nextPendingConnection(); + _job = new GETFileJob(_account, _remotePath + _folderPath, _socket, {}, {}, 0, this); + connect(_job, &GETFileJob::finishedSignal, this, &HydrationJob::onGetFinished); + _job->start(); +} + +void OCC::HydrationJob::onGetFinished() +{ + qCInfo(lcHydration) << "GETFileJob finished" << _requestId << _folderPath << _job->reply()->error(); + + if (_job->reply()->error()) { + emitFinished(Error); + return; + } + + SyncJournalFileRecord record; + _journal->getFileRecord(_folderPath, &record); + Q_ASSERT(record.isValid()); + if (!record.isValid()) { + qCWarning(lcHydration) << "Couldn't find record to update after hydration" << _requestId << _folderPath; + emitFinished(Error); + return; + } + + record._type = ItemTypeFile; + _journal->setFileRecord(record); + emitFinished(Success); +} diff --git a/src/libsync/vfs/cfapi/hydrationjob.h b/src/libsync/vfs/cfapi/hydrationjob.h new file mode 100644 index 0000000000000..52709abe35cb6 --- /dev/null +++ b/src/libsync/vfs/cfapi/hydrationjob.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) by Kevin Ottens + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include + +#include "account.h" + +class QLocalServer; +class QLocalSocket; + +namespace OCC { +class GETFileJob; +class SyncJournalDb; + +class OWNCLOUDSYNC_EXPORT HydrationJob : public QObject +{ + Q_OBJECT +public: + enum Status { + Success = 0, + Error, + }; + Q_ENUM(Status) + + explicit HydrationJob(QObject *parent = nullptr); + + AccountPtr account() const; + void setAccount(const AccountPtr &account); + + QString remotePath() const; + void setRemotePath(const QString &remotePath); + + QString localPath() const; + void setLocalPath(const QString &localPath); + + SyncJournalDb *journal() const; + void setJournal(SyncJournalDb *journal); + + QString requestId() const; + void setRequestId(const QString &requestId); + + QString folderPath() const; + void setFolderPath(const QString &folderPath); + + Status status() const; + + void start(); + +signals: + void finished(HydrationJob *job); + +private: + void emitFinished(Status status); + + void onNewConnection(); + void onGetFinished(); + + AccountPtr _account; + QString _remotePath; + QString _localPath; + SyncJournalDb *_journal = nullptr; + + QString _requestId; + QString _folderPath; + + QLocalServer *_server = nullptr; + QLocalSocket *_socket = nullptr; + GETFileJob *_job = nullptr; + Status _status = Success; +}; + +} // namespace OCC diff --git a/src/libsync/vfs/cfapi/vfs_cfapi.cpp b/src/libsync/vfs/cfapi/vfs_cfapi.cpp new file mode 100644 index 0000000000000..4a73ad968ca31 --- /dev/null +++ b/src/libsync/vfs/cfapi/vfs_cfapi.cpp @@ -0,0 +1,394 @@ +/* + * Copyright (C) by Kevin Ottens + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "vfs_cfapi.h" + +#include +#include + +#include "cfapiwrapper.h" +#include "hydrationjob.h" +#include "syncfileitem.h" +#include "filesystem.h" +#include "common/syncjournaldb.h" + +#include +#include + +Q_LOGGING_CATEGORY(lcCfApi, "nextcloud.sync.vfs.cfapi", QtInfoMsg) + +namespace cfapi { +using namespace OCC::CfApiWrapper; +} + +namespace OCC { + +class VfsCfApiPrivate +{ +public: + QList hydrationJobs; + cfapi::ConnectionKey connectionKey; +}; + +VfsCfApi::VfsCfApi(QObject *parent) + : Vfs(parent) + , d(new VfsCfApiPrivate) +{ +} + +VfsCfApi::~VfsCfApi() = default; + +Vfs::Mode VfsCfApi::mode() const +{ + return WindowsCfApi; +} + +QString VfsCfApi::fileSuffix() const +{ + return {}; +} + +void VfsCfApi::startImpl(const VfsSetupParams ¶ms) +{ + const auto localPath = QDir::toNativeSeparators(params.filesystemPath); + + const auto registerResult = cfapi::registerSyncRoot(localPath, params.providerName, params.providerVersion); + if (!registerResult) { + qCCritical(lcCfApi) << "Initialization failed, couldn't register sync root:" << registerResult.error(); + return; + } + + auto connectResult = cfapi::connectSyncRoot(localPath, this); + if (!connectResult) { + qCCritical(lcCfApi) << "Initialization failed, couldn't connect sync root:" << connectResult.error(); + return; + } + + d->connectionKey = *std::move(connectResult); +} + +void VfsCfApi::stop() +{ + const auto result = cfapi::disconnectSyncRoot(std::move(d->connectionKey)); + if (!result) { + qCCritical(lcCfApi) << "Disconnect failed for" << QDir::toNativeSeparators(params().filesystemPath) << ":" << result.error(); + } +} + +void VfsCfApi::unregisterFolder() +{ + const auto localPath = QDir::toNativeSeparators(params().filesystemPath); + const auto result = cfapi::unegisterSyncRoot(localPath); + if (!result) { + qCCritical(lcCfApi) << "Unregistration failed for" << localPath << ":" << result.error(); + } +} + +bool VfsCfApi::socketApiPinStateActionsShown() const +{ + return true; +} + +bool VfsCfApi::isHydrating() const +{ + return !d->hydrationJobs.isEmpty(); +} + +Result VfsCfApi::updateMetadata(const QString &filePath, time_t modtime, qint64 size, const QByteArray &fileId) +{ + const auto localPath = QDir::toNativeSeparators(filePath); + const auto handle = cfapi::handleForPath(localPath); + if (handle) { + return cfapi::updatePlaceholderInfo(handle, modtime, size, fileId); + } else { + qCWarning(lcCfApi) << "Couldn't update metadata for non existing file" << localPath; + return "Couldn't update metadata"; + } +} + +Result VfsCfApi::createPlaceholder(const SyncFileItem &item) +{ + Q_ASSERT(params().filesystemPath.endsWith('/')); + const auto localPath = QDir::toNativeSeparators(params().filesystemPath + item._file); + const auto result = cfapi::createPlaceholderInfo(localPath, item._modtime, item._size, item._fileId); + return result; +} + +Result VfsCfApi::dehydratePlaceholder(const SyncFileItem &item) +{ + const auto previousPin = pinState(item._file); + + if (!QFile::remove(_setupParams.filesystemPath + item._file)) { + return QStringLiteral("Couldn't remove %1 to fulfill dehydration").arg(item._file); + } + + const auto r = createPlaceholder(item); + if (!r) { + return r; + } + + if (previousPin) { + if (*previousPin == PinState::AlwaysLocal) { + setPinState(item._file, PinState::Unspecified); + } else { + setPinState(item._file, *previousPin); + } + } + + return {}; +} + +void VfsCfApi::convertToPlaceholder(const QString &filename, const SyncFileItem &item, const QString &replacesFile) +{ + const auto localPath = QDir::toNativeSeparators(filename); + const auto replacesPath = QDir::toNativeSeparators(replacesFile); + + const auto handle = cfapi::handleForPath(localPath); + if (cfapi::findPlaceholderInfo(handle)) { + cfapi::updatePlaceholderInfo(handle, item._modtime, item._size, item._fileId, replacesPath); + } else { + cfapi::convertToPlaceholder(handle, item._modtime, item._size, item._fileId, replacesPath); + } +} + +bool VfsCfApi::needsMetadataUpdate(const SyncFileItem &item) +{ + return false; +} + +bool VfsCfApi::isDehydratedPlaceholder(const QString &filePath) +{ + const auto path = QDir::toNativeSeparators(filePath); + return cfapi::isSparseFile(path); +} + +bool VfsCfApi::statTypeVirtualFile(csync_file_stat_t *stat, void *statData) +{ + const auto ffd = static_cast(statData); + + const auto isDirectory = (ffd->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + const auto isSparseFile = (ffd->dwFileAttributes & FILE_ATTRIBUTE_SPARSE_FILE) != 0; + const auto isPinned = (ffd->dwFileAttributes & FILE_ATTRIBUTE_PINNED) != 0; + const auto isUnpinned = (ffd->dwFileAttributes & FILE_ATTRIBUTE_UNPINNED) != 0; + const auto hasReparsePoint = (ffd->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; + const auto hasCloudTag = (ffd->dwReserved0 & IO_REPARSE_TAG_CLOUD) != 0; + + // It's a dir with a reparse point due to the placeholder info (hence the cloud tag) + // if we don't remove the reparse point flag the discovery will end up thinking + // it is a file... let's prevent it + if (isDirectory && hasReparsePoint && hasCloudTag) { + ffd->dwFileAttributes &= ~FILE_ATTRIBUTE_REPARSE_POINT; + return false; + } else if (isSparseFile && isPinned) { + stat->type = ItemTypeVirtualFileDownload; + return true; + } else if (!isSparseFile && isUnpinned){ + stat->type = ItemTypeVirtualFileDehydration; + return true; + } else if (isSparseFile) { + stat->type = ItemTypeVirtualFile; + return true; + } + + return false; +} + +bool VfsCfApi::setPinState(const QString &folderPath, PinState state) +{ + const auto localPath = QDir::toNativeSeparators(params().filesystemPath + folderPath); + const auto handle = cfapi::handleForPath(localPath); + if (handle) { + if (cfapi::setPinState(handle, state, cfapi::Recurse)) { + return true; + } else { + return false; + } + } else { + qCWarning(lcCfApi) << "Couldn't update pin state for non existing file" << localPath; + return false; + } +} + +Optional VfsCfApi::pinState(const QString &folderPath) +{ + const auto localPath = QDir::toNativeSeparators(params().filesystemPath + folderPath); + const auto handle = cfapi::handleForPath(localPath); + if (!handle) { + qCWarning(lcCfApi) << "Couldn't find pin state for non existing file" << localPath; + return {}; + } + + const auto info = cfapi::findPlaceholderInfo(handle); + if (!info) { + qCWarning(lcCfApi) << "Couldn't find pin state for regular non-placeholder file" << localPath; + return {}; + } + + return info.pinState(); +} + +Vfs::AvailabilityResult VfsCfApi::availability(const QString &folderPath) +{ + const auto basePinState = pinState(folderPath); + const auto hydrationAndPinStates = computeRecursiveHydrationAndPinStates(folderPath, basePinState); + + const auto pin = hydrationAndPinStates.pinState; + const auto hydrationStatus = hydrationAndPinStates.hydrationStatus; + + if (hydrationStatus.hasDehydrated) { + if (hydrationStatus.hasHydrated) + return VfsItemAvailability::Mixed; + if (pin && *pin == PinState::OnlineOnly) + return VfsItemAvailability::OnlineOnly; + else + return VfsItemAvailability::AllDehydrated; + } else if (hydrationStatus.hasHydrated) { + if (pin && *pin == PinState::AlwaysLocal) + return VfsItemAvailability::AlwaysLocal; + else + return VfsItemAvailability::AllHydrated; + } + return AvailabilityError::NoSuchItem; +} + +void VfsCfApi::requestHydration(const QString &requestId, const QString &path) +{ + qCInfo(lcCfApi) << "Received request to hydrate" << path << requestId; + const auto root = QDir::toNativeSeparators(params().filesystemPath); + Q_ASSERT(path.startsWith(root)); + + const auto relativePath = QDir::fromNativeSeparators(path.mid(root.length())); + const auto journal = params().journal; + + // Set in the database that we should download the file + SyncJournalFileRecord record; + journal->getFileRecord(relativePath, &record); + if (!record.isValid()) { + qCInfo(lcCfApi) << "Couldn't hydrate, did not find file in db"; + emit hydrationRequestFailed(requestId); + return; + } + + if (!record.isVirtualFile()) { + qCInfo(lcCfApi) << "Couldn't hydrate, the file is not virtual"; + emit hydrationRequestFailed(requestId); + return; + } + + // This is impossible to handle with CfAPI since the file size is generally different + // between the encrypted and the decrypted file which would make CfAPI reject the hydration + // of the placeholder with decrypted data + if (record._isE2eEncrypted || !record._e2eMangledName.isEmpty()) { + qCInfo(lcCfApi) << "Couldn't hydrate, the file is E2EE this is not supported"; + emit hydrationRequestFailed(requestId); + return; + } + + // All good, let's hydrate now + scheduleHydrationJob(requestId, relativePath); +} + +void VfsCfApi::fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus) +{ + Q_UNUSED(systemFileName); + Q_UNUSED(fileStatus); +} + +void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &folderPath) +{ + Q_ASSERT(std::none_of(std::cbegin(d->hydrationJobs), std::cend(d->hydrationJobs), [=](HydrationJob *job) { + return job->requestId() == requestId || job->folderPath() == folderPath; + })); + + if (d->hydrationJobs.isEmpty()) { + emit beginHydrating(); + } + + auto job = new HydrationJob(this); + job->setAccount(params().account); + job->setRemotePath(params().remotePath); + job->setLocalPath(params().filesystemPath); + job->setJournal(params().journal); + job->setRequestId(requestId); + job->setFolderPath(folderPath); + connect(job, &HydrationJob::finished, this, &VfsCfApi::onHydrationJobFinished); + d->hydrationJobs << job; + job->start(); + emit hydrationRequestReady(requestId); +} + +void VfsCfApi::onHydrationJobFinished(HydrationJob *job) +{ + Q_ASSERT(d->hydrationJobs.contains(job)); + qCInfo(lcCfApi) << "Hydration job finished" << job->requestId() << job->folderPath() << job->status(); + emit hydrationRequestFinished(job->requestId(), job->status()); + d->hydrationJobs.removeAll(job); + if (d->hydrationJobs.isEmpty()) { + emit doneHydrating(); + } +} + +VfsCfApi::HydratationAndPinStates VfsCfApi::computeRecursiveHydrationAndPinStates(const QString &folderPath, const Optional &basePinState) +{ + Q_ASSERT(!folderPath.endsWith('/')); + QFileInfo info(params().filesystemPath + folderPath); + + if (!info.exists()) { + return {}; + } + + const auto effectivePin = pinState(folderPath); + const auto pinResult = (!effectivePin && !basePinState) ? Optional() + : (!effectivePin || !basePinState) ? PinState::Inherited + : (*effectivePin == *basePinState) ? *effectivePin + : PinState::Inherited; + + if (info.isDir()) { + const auto dirState = HydratationAndPinStates { + pinResult, + {} + }; + const auto dir = QDir(info.absoluteFilePath()); + Q_ASSERT(dir.exists()); + const auto children = dir.entryList(); + return std::accumulate(std::cbegin(children), std::cend(children), dirState, [=](const HydratationAndPinStates ¤tState, const QString &name) { + if (name == QStringLiteral("..") || name == QStringLiteral(".")) { + return currentState; + } + + const auto path = folderPath + '/' + name; + const auto states = computeRecursiveHydrationAndPinStates(path, currentState.pinState); + return HydratationAndPinStates { + states.pinState, + { + states.hydrationStatus.hasHydrated || currentState.hydrationStatus.hasHydrated, + states.hydrationStatus.hasDehydrated || currentState.hydrationStatus.hasDehydrated, + } + }; + }); + } else { // file case + const auto isDehydrated = isDehydratedPlaceholder(info.absoluteFilePath()); + return { + pinResult, + { + !isDehydrated, + isDehydrated + } + }; + } +} + +} // namespace OCC + +OCC_DEFINE_VFS_FACTORY("win", OCC::VfsCfApi) diff --git a/src/libsync/vfs/cfapi/vfs_cfapi.h b/src/libsync/vfs/cfapi/vfs_cfapi.h new file mode 100644 index 0000000000000..e4a4d00742662 --- /dev/null +++ b/src/libsync/vfs/cfapi/vfs_cfapi.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) by Kevin Ottens + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include +#include + +#include "common/vfs.h" + +namespace OCC { +class HydrationJob; +class VfsCfApiPrivate; + +class VfsCfApi : public Vfs +{ + Q_OBJECT + +public: + explicit VfsCfApi(QObject *parent = nullptr); + ~VfsCfApi(); + + Mode mode() const override; + QString fileSuffix() const override; + + void stop() override; + void unregisterFolder() override; + + bool socketApiPinStateActionsShown() const override; + bool isHydrating() const override; + + Result updateMetadata(const QString &filePath, time_t modtime, qint64 size, const QByteArray &fileId) override; + + Result createPlaceholder(const SyncFileItem &item) override; + Result dehydratePlaceholder(const SyncFileItem &item) override; + void convertToPlaceholder(const QString &filename, const SyncFileItem &item, const QString &replacesFile) override; + + bool needsMetadataUpdate(const SyncFileItem &) override; + bool isDehydratedPlaceholder(const QString &filePath) override; + bool statTypeVirtualFile(csync_file_stat_t *stat, void *statData) override; + + bool setPinState(const QString &folderPath, PinState state) override; + Optional pinState(const QString &folderPath) override; + AvailabilityResult availability(const QString &folderPath) override; + +public slots: + void requestHydration(const QString &requestId, const QString &path); + void fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus) override; + +signals: + void hydrationRequestReady(const QString &requestId); + void hydrationRequestFailed(const QString &requestId); + void hydrationRequestFinished(const QString &requestId, int status); + +protected: + void startImpl(const VfsSetupParams ¶ms) override; + +private: + void scheduleHydrationJob(const QString &requestId, const QString &folderPath); + void onHydrationJobFinished(HydrationJob *job); + + struct HasHydratedDehydrated { + bool hasHydrated = false; + bool hasDehydrated = false; + }; + struct HydratationAndPinStates { + Optional pinState; + HasHydratedDehydrated hydrationStatus; + }; + HydratationAndPinStates computeRecursiveHydrationAndPinStates(const QString &path, const Optional &basePinState); + + QScopedPointer d; +}; + +} // namespace OCC diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8932d2b80ecd3..b13f6bff16fc1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -73,6 +73,10 @@ if( UNIX AND NOT APPLE ) nextcloud_add_test(InotifyWatcher "${FolderWatcher_SRC}") endif(UNIX AND NOT APPLE) +if (WIN32) + nextcloud_add_test(SyncCfApi "") +endif() + nextcloud_add_benchmark(LargeSync "") SET(FolderMan_SRC ../src/gui/folderman.cpp) diff --git a/test/testsynccfapi.cpp b/test/testsynccfapi.cpp new file mode 100644 index 0000000000000..30634b1a2f956 --- /dev/null +++ b/test/testsynccfapi.cpp @@ -0,0 +1,1189 @@ +/* + * This software is in the public domain, furnished "as is", without technical + * support, and with no warranty, express or implied, as to its usefulness for + * any purpose. + * + */ + +#include +#include "syncenginetestutils.h" +#include "common/vfs.h" +#include "config.h" +#include + +#include "vfs/cfapi/cfapiwrapper.h" + +namespace cfapi { +using namespace OCC::CfApiWrapper; +} + +#define CFVERIFY_VIRTUAL(folder, path) \ + QVERIFY(QFileInfo((folder).localPath() + (path)).exists()); \ + QVERIFY(cfapi::isSparseFile((folder).localPath() + (path))); \ + QVERIFY(dbRecord((folder), (path)).isValid()); \ + QCOMPARE(dbRecord((folder), (path))._type, ItemTypeVirtualFile); + +#define CFVERIFY_NONVIRTUAL(folder, path) \ + QVERIFY(QFileInfo((folder).localPath() + (path)).exists()); \ + QVERIFY(!cfapi::isSparseFile((folder).localPath() + (path))); \ + QVERIFY(dbRecord((folder), (path)).isValid()); \ + QCOMPARE(dbRecord((folder), (path))._type, ItemTypeFile); + +#define CFVERIFY_GONE(folder, path) \ + QVERIFY(!QFileInfo((folder).localPath() + (path)).exists()); \ + QVERIFY(!dbRecord((folder), (path)).isValid()); + +using namespace OCC; + +enum ErrorKind : int { + NoError = 0, + // Lower code are corresponding to HTTP error code + Timeout = 1000, +}; + +void setPinState(const QString &path, PinState state, cfapi::SetPinRecurseMode mode) +{ + Q_ASSERT(mode == cfapi::Recurse || mode == cfapi::NoRecurse); + + const auto p = QDir::toNativeSeparators(path); + const auto handle = cfapi::handleForPath(p); + Q_ASSERT(handle); + + const auto result = cfapi::setPinState(handle, state, mode); + Q_ASSERT(result); + + if (mode == cfapi::NoRecurse) { + const auto result = cfapi::setPinState(handle, PinState::Inherited, cfapi::ChildrenOnly); + Q_ASSERT(result); + } +} + +bool itemInstruction(const ItemCompletedSpy &spy, const QString &path, const SyncInstructions instr) +{ + auto item = spy.findItem(path); + return item->_instruction == instr; +} + +SyncJournalFileRecord dbRecord(FakeFolder &folder, const QString &path) +{ + SyncJournalFileRecord record; + folder.syncJournal().getFileRecord(path, &record); + return record; +} + +void triggerDownload(FakeFolder &folder, const QByteArray &path) +{ + auto &journal = folder.syncJournal(); + SyncJournalFileRecord record; + journal.getFileRecord(path, &record); + if (!record.isValid()) + return; + record._type = ItemTypeVirtualFileDownload; + journal.setFileRecord(record); + journal.schedulePathForRemoteDiscovery(record._path); +} + +void markForDehydration(FakeFolder &folder, const QByteArray &path) +{ + auto &journal = folder.syncJournal(); + SyncJournalFileRecord record; + journal.getFileRecord(path, &record); + if (!record.isValid()) + return; + record._type = ItemTypeVirtualFileDehydration; + journal.setFileRecord(record); + journal.schedulePathForRemoteDiscovery(record._path); +} + +QSharedPointer setupVfs(FakeFolder &folder) +{ + auto cfapiVfs = QSharedPointer(createVfsFromPlugin(Vfs::WindowsCfApi).release()); + QObject::connect(&folder.syncEngine().syncFileStatusTracker(), &SyncFileStatusTracker::fileStatusChanged, + cfapiVfs.data(), &Vfs::fileStatusChanged); + folder.switchToVfs(cfapiVfs); + + setPinState(folder.localPath(), PinState::Unspecified, cfapi::NoRecurse); + + return cfapiVfs; +} + +class TestSyncCfApi : public QObject +{ + Q_OBJECT + +private slots: + void testVirtualFileLifecycle_data() + { + QTest::addColumn("doLocalDiscovery"); + + QTest::newRow("full local discovery") << true; + QTest::newRow("skip local discovery") << false; + } + + void testVirtualFileLifecycle() + { + QFETCH(bool, doLocalDiscovery); + + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + ItemCompletedSpy completeSpy(fakeFolder); + + auto cleanup = [&]() { + completeSpy.clear(); + if (!doLocalDiscovery) + fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem); + }; + cleanup(); + + // Create a virtual file for a new remote file + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().insert("A/a1", 64); + auto someDate = QDateTime(QDate(1984, 07, 30), QTime(1,3,2)); + fakeFolder.remoteModifier().setModTime("A/a1", someDate); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 64); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate); + QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW)); + cleanup(); + + // Another sync doesn't actually lead to changes + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 64); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate); + QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(completeSpy.isEmpty()); + cleanup(); + + // Not even when the remote is rediscovered + fakeFolder.syncJournal().forceRemoteDiscoveryNextSync(); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 64); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate); + QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(completeSpy.isEmpty()); + cleanup(); + + // Neither does a remote change + fakeFolder.remoteModifier().appendByte("A/a1"); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 65); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate); + QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_UPDATE_METADATA)); + QCOMPARE(dbRecord(fakeFolder, "A/a1")._fileSize, 65); + cleanup(); + + // If the local virtual file is removed, this will be propagated remotely + if (!doLocalDiscovery) + fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, { "A" }); + fakeFolder.localModifier().remove("A/a1"); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); + QVERIFY(!fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid()); + cleanup(); + + // Recreate a1 before carrying on with the other tests + fakeFolder.remoteModifier().insert("A/a1", 65); + fakeFolder.remoteModifier().setModTime("A/a1", someDate); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 65); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").lastModified(), someDate); + QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW)); + cleanup(); + + // Remote rename is propagated + fakeFolder.remoteModifier().rename("A/a1", "A/a1m"); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(!QFileInfo(fakeFolder.localPath() + "A/a1").exists()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1m"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1m").size(), 65); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1m").lastModified(), someDate); + QVERIFY(!fakeFolder.currentRemoteState().find("A/a1")); + QVERIFY(fakeFolder.currentRemoteState().find("A/a1m")); + QVERIFY( + itemInstruction(completeSpy, "A/a1m", CSYNC_INSTRUCTION_RENAME) + || (itemInstruction(completeSpy, "A/a1m", CSYNC_INSTRUCTION_NEW) + && itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_REMOVE))); + QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid()); + cleanup(); + + // Remote remove is propagated + fakeFolder.remoteModifier().remove("A/a1m"); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(!QFileInfo(fakeFolder.localPath() + "A/a1m").exists()); + QVERIFY(!fakeFolder.currentRemoteState().find("A/a1m")); + QVERIFY(itemInstruction(completeSpy, "A/a1m", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a1m").isValid()); + cleanup(); + + // Edge case: Local virtual file but no db entry for some reason + fakeFolder.remoteModifier().insert("A/a2", 32); + fakeFolder.remoteModifier().insert("A/a3", 33); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a2"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a2").size(), 32); + CFVERIFY_VIRTUAL(fakeFolder, "A/a3"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a3").size(), 33); + cleanup(); + + fakeFolder.syncEngine().journal()->deleteFileRecord("A/a2"); + fakeFolder.syncEngine().journal()->deleteFileRecord("A/a3"); + fakeFolder.remoteModifier().remove("A/a3"); + fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::FilesystemOnly); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a2"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a2").size(), 32); + QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_UPDATE_METADATA)); + QVERIFY(!QFileInfo(fakeFolder.localPath() + "A/a3").exists()); + QVERIFY(itemInstruction(completeSpy, "A/a3", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(!dbRecord(fakeFolder, "A/a3").isValid()); + cleanup(); + } + + void testVirtualFileConflict() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + ItemCompletedSpy completeSpy(fakeFolder); + + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + // Create a virtual file for a new remote file + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().insert("A/a1", 11); + fakeFolder.remoteModifier().insert("A/a2", 12); + fakeFolder.remoteModifier().mkdir("B"); + fakeFolder.remoteModifier().insert("B/b1", 21); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_VIRTUAL(fakeFolder, "B/b1"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1").size(), 11); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a2").size(), 12); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "B/b1").size(), 21); + cleanup(); + + // All the files are touched on the server + fakeFolder.remoteModifier().appendByte("A/a1"); + fakeFolder.remoteModifier().appendByte("A/a2"); + fakeFolder.remoteModifier().appendByte("B/b1"); + + // A: the correct file and a conflicting file are added + // B: user adds a *directory* locally + fakeFolder.localModifier().remove("A/a1"); + fakeFolder.localModifier().insert("A/a1", 12); + fakeFolder.localModifier().remove("A/a2"); + fakeFolder.localModifier().insert("A/a2", 10); + fakeFolder.localModifier().remove("B/b1"); + fakeFolder.localModifier().mkdir("B/b1"); + fakeFolder.localModifier().insert("B/b1/foo"); + QVERIFY(fakeFolder.syncOnce()); + + // Everything is CONFLICT + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_CONFLICT)); + QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_CONFLICT)); + QVERIFY(itemInstruction(completeSpy, "B/b1", CSYNC_INSTRUCTION_CONFLICT)); + + // conflict files should exist + QCOMPARE(fakeFolder.syncJournal().conflictRecordPaths().size(), 2); + + // nothing should have the virtual file tag + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_NONVIRTUAL(fakeFolder, "B/b1"); + + cleanup(); + } + + void testWithNormalSync() + { + FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + ItemCompletedSpy completeSpy(fakeFolder); + + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + // No effect sync + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + cleanup(); + + // Existing files are propagated just fine in both directions + fakeFolder.localModifier().appendByte("A/a1"); + fakeFolder.localModifier().insert("A/a3"); + fakeFolder.remoteModifier().appendByte("A/a2"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + cleanup(); + + // New files on the remote create virtual files + fakeFolder.remoteModifier().insert("A/new", 42); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/new"); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/new").size(), 42); + QVERIFY(fakeFolder.currentRemoteState().find("A/new")); + QVERIFY(itemInstruction(completeSpy, "A/new", CSYNC_INSTRUCTION_NEW)); + cleanup(); + } + + void testVirtualFileDownload() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + ItemCompletedSpy completeSpy(fakeFolder); + + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + // Create a virtual file for remote files + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().insert("A/a1"); + fakeFolder.remoteModifier().insert("A/a2"); + fakeFolder.remoteModifier().insert("A/a3"); + fakeFolder.remoteModifier().insert("A/a4"); + fakeFolder.remoteModifier().insert("A/a5"); + fakeFolder.remoteModifier().insert("A/a6"); + fakeFolder.remoteModifier().insert("A/a7"); + fakeFolder.remoteModifier().insert("A/b1"); + fakeFolder.remoteModifier().insert("A/b2"); + fakeFolder.remoteModifier().insert("A/b3"); + fakeFolder.remoteModifier().insert("A/b4"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a3"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a4"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a5"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a6"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a7"); + CFVERIFY_VIRTUAL(fakeFolder, "A/b1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/b2"); + CFVERIFY_VIRTUAL(fakeFolder, "A/b3"); + CFVERIFY_VIRTUAL(fakeFolder, "A/b4"); + + cleanup(); + + // Download by changing the db entry + triggerDownload(fakeFolder, "A/a1"); + triggerDownload(fakeFolder, "A/a2"); + triggerDownload(fakeFolder, "A/a3"); + triggerDownload(fakeFolder, "A/a4"); + triggerDownload(fakeFolder, "A/a5"); + triggerDownload(fakeFolder, "A/a6"); + triggerDownload(fakeFolder, "A/a7"); + triggerDownload(fakeFolder, "A/b1"); + triggerDownload(fakeFolder, "A/b2"); + triggerDownload(fakeFolder, "A/b3"); + triggerDownload(fakeFolder, "A/b4"); + + // Remote complications + fakeFolder.remoteModifier().appendByte("A/a2"); + fakeFolder.remoteModifier().remove("A/a3"); + fakeFolder.remoteModifier().rename("A/a4", "A/a4m"); + fakeFolder.remoteModifier().appendByte("A/b2"); + fakeFolder.remoteModifier().remove("A/b3"); + fakeFolder.remoteModifier().rename("A/b4", "A/b4m"); + + // Local complications + fakeFolder.localModifier().remove("A/a5"); + fakeFolder.localModifier().insert("A/a5"); + fakeFolder.localModifier().remove("A/a6"); + fakeFolder.localModifier().insert("A/a6"); + + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC)); + QCOMPARE(completeSpy.findItem("A/a1")->_type, ItemTypeVirtualFileDownload); + QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_SYNC)); + QCOMPARE(completeSpy.findItem("A/a2")->_type, ItemTypeVirtualFileDownload); + QVERIFY(itemInstruction(completeSpy, "A/a3", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "A/a4m", CSYNC_INSTRUCTION_NEW)); + QVERIFY(itemInstruction(completeSpy, "A/a4", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "A/a5", CSYNC_INSTRUCTION_CONFLICT)); + QVERIFY(itemInstruction(completeSpy, "A/a6", CSYNC_INSTRUCTION_CONFLICT)); + QVERIFY(itemInstruction(completeSpy, "A/a7", CSYNC_INSTRUCTION_SYNC)); + QVERIFY(itemInstruction(completeSpy, "A/b1", CSYNC_INSTRUCTION_SYNC)); + QVERIFY(itemInstruction(completeSpy, "A/b2", CSYNC_INSTRUCTION_SYNC)); + QVERIFY(itemInstruction(completeSpy, "A/b3", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "A/b4m", CSYNC_INSTRUCTION_NEW)); + QVERIFY(itemInstruction(completeSpy, "A/b4", CSYNC_INSTRUCTION_REMOVE)); + + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_GONE(fakeFolder, "A/a3"); + CFVERIFY_GONE(fakeFolder, "A/a4"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a4m"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a5"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a6"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a7"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/b1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/b2"); + CFVERIFY_GONE(fakeFolder, "A/b3"); + CFVERIFY_GONE(fakeFolder, "A/b4"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/b4m"); + + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + } + + void testVirtualFileDownloadResume() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + ItemCompletedSpy completeSpy(fakeFolder); + + auto cleanup = [&]() { + completeSpy.clear(); + fakeFolder.syncJournal().wipeErrorBlacklist(); + }; + cleanup(); + + // Create a virtual file for remote files + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().insert("A/a1"); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + cleanup(); + + // Download by changing the db entry + triggerDownload(fakeFolder, "A/a1"); + fakeFolder.serverErrorPaths().append("A/a1", 500); + QVERIFY(!fakeFolder.syncOnce()); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC)); + QVERIFY(cfapi::isSparseFile(fakeFolder.localPath() + "A/a1")); + QVERIFY(QFileInfo(fakeFolder.localPath() + "A/a1").exists()); + QCOMPARE(dbRecord(fakeFolder, "A/a1")._type, ItemTypeVirtualFileDownload); + cleanup(); + + fakeFolder.serverErrorPaths().clear(); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC)); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1"); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + } + + void testNewFilesNotVirtual() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().insert("A/a1"); + QVERIFY(fakeFolder.syncOnce()); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + + setPinState(fakeFolder.localPath(), PinState::AlwaysLocal, cfapi::NoRecurse); + + // Create a new remote file, it'll not be virtual + fakeFolder.remoteModifier().insert("A/a2"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2"); + } + + void testDownloadRecursive() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + // Create a virtual file for remote files + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().mkdir("A/Sub"); + fakeFolder.remoteModifier().mkdir("A/Sub/SubSub"); + fakeFolder.remoteModifier().mkdir("A/Sub2"); + fakeFolder.remoteModifier().mkdir("B"); + fakeFolder.remoteModifier().mkdir("B/Sub"); + fakeFolder.remoteModifier().insert("A/a1"); + fakeFolder.remoteModifier().insert("A/a2"); + fakeFolder.remoteModifier().insert("A/Sub/a3"); + fakeFolder.remoteModifier().insert("A/Sub/a4"); + fakeFolder.remoteModifier().insert("A/Sub/SubSub/a5"); + fakeFolder.remoteModifier().insert("A/Sub2/a6"); + fakeFolder.remoteModifier().insert("B/b1"); + fakeFolder.remoteModifier().insert("B/Sub/b2"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/a3"); + CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/a4"); + CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/SubSub/a5"); + CFVERIFY_VIRTUAL(fakeFolder, "A/Sub2/a6"); + CFVERIFY_VIRTUAL(fakeFolder, "B/b1"); + CFVERIFY_VIRTUAL(fakeFolder, "B/Sub/b2"); + + // Download All file in the directory A/Sub + // (as in Folder::downloadVirtualFile) + fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("A/Sub"); + + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a3"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a4"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/SubSub/a5"); + CFVERIFY_VIRTUAL(fakeFolder, "A/Sub2/a6"); + CFVERIFY_VIRTUAL(fakeFolder, "B/b1"); + CFVERIFY_VIRTUAL(fakeFolder, "B/Sub/b2"); + + // Add a file in a subfolder that was downloaded + // Currently, this continue to add it as a virtual file. + fakeFolder.remoteModifier().insert("A/Sub/SubSub/a7"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "A/Sub/SubSub/a7"); + + // Now download all files in "A" + fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("A"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/a2"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a3"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/a4"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/SubSub/a5"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub/SubSub/a7"); + CFVERIFY_NONVIRTUAL(fakeFolder, "A/Sub2/a6"); + CFVERIFY_VIRTUAL(fakeFolder, "B/b1"); + CFVERIFY_VIRTUAL(fakeFolder, "B/Sub/b2"); + + // Now download remaining files in "B" + fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("B"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + } + + void testRenameVirtual() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + ItemCompletedSpy completeSpy(fakeFolder); + + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + fakeFolder.remoteModifier().insert("file1", 128, 'C'); + fakeFolder.remoteModifier().insert("file2", 256, 'C'); + fakeFolder.remoteModifier().insert("file3", 256, 'C'); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "file1"); + CFVERIFY_VIRTUAL(fakeFolder, "file2"); + CFVERIFY_VIRTUAL(fakeFolder, "file3"); + + cleanup(); + + fakeFolder.localModifier().rename("file1", "renamed1"); + fakeFolder.localModifier().rename("file2", "renamed2"); + triggerDownload(fakeFolder, "file2"); + triggerDownload(fakeFolder, "file3"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_GONE(fakeFolder, "file1"); + CFVERIFY_VIRTUAL(fakeFolder, "renamed1"); + + QVERIFY(fakeFolder.currentRemoteState().find("renamed1")); + QVERIFY(itemInstruction(completeSpy, "renamed1", CSYNC_INSTRUCTION_RENAME)); + + // file2 has a conflict between the download request and the rename: + // the rename wins, the download is ignored + + CFVERIFY_GONE(fakeFolder, "file2"); + CFVERIFY_VIRTUAL(fakeFolder, "renamed2"); + + QVERIFY(fakeFolder.currentRemoteState().find("renamed2")); + QVERIFY(itemInstruction(completeSpy, "renamed2", CSYNC_INSTRUCTION_RENAME)); + + QVERIFY(itemInstruction(completeSpy, "file3", CSYNC_INSTRUCTION_SYNC)); + CFVERIFY_NONVIRTUAL(fakeFolder, "file3"); + cleanup(); + } + + void testRenameVirtual2() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + ItemCompletedSpy completeSpy(fakeFolder); + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + fakeFolder.remoteModifier().insert("case3", 128, 'C'); + fakeFolder.remoteModifier().insert("case4", 256, 'C'); + QVERIFY(fakeFolder.syncOnce()); + + triggerDownload(fakeFolder, "case4"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "case3"); + CFVERIFY_NONVIRTUAL(fakeFolder, "case4"); + + cleanup(); + + // Case 1: non-virtual, foo -> bar (tested elsewhere) + // Case 2: virtual, foo -> bar (tested elsewhere) + + // Case 3: virtual, foo.oc -> bar.oc (db hydrate) + fakeFolder.localModifier().rename("case3", "case3-rename"); + triggerDownload(fakeFolder, "case3"); + + // Case 4: non-virtual foo -> bar (db dehydrate) + fakeFolder.localModifier().rename("case4", "case4-rename"); + markForDehydration(fakeFolder, "case4"); + + QVERIFY(fakeFolder.syncOnce()); + + // Case 3: the rename went though, hydration is forgotten + CFVERIFY_GONE(fakeFolder, "case3"); + CFVERIFY_VIRTUAL(fakeFolder, "case3-rename"); + QVERIFY(!fakeFolder.currentRemoteState().find("case3")); + QVERIFY(fakeFolder.currentRemoteState().find("case3-rename")); + QVERIFY(itemInstruction(completeSpy, "case3-rename", CSYNC_INSTRUCTION_RENAME)); + + // Case 4: the rename went though, dehydration is forgotten + CFVERIFY_GONE(fakeFolder, "case4"); + CFVERIFY_NONVIRTUAL(fakeFolder, "case4-rename"); + QVERIFY(!fakeFolder.currentRemoteState().find("case4")); + QVERIFY(fakeFolder.currentRemoteState().find("case4-rename")); + QVERIFY(itemInstruction(completeSpy, "case4-rename", CSYNC_INSTRUCTION_RENAME)); + } + + // Dehydration via sync works + void testSyncDehydration() + { + FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; + setupVfs(fakeFolder); + + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + ItemCompletedSpy completeSpy(fakeFolder); + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + // + // Mark for dehydration and check + // + + markForDehydration(fakeFolder, "A/a1"); + + markForDehydration(fakeFolder, "A/a2"); + fakeFolder.remoteModifier().appendByte("A/a2"); + // expect: normal dehydration + + markForDehydration(fakeFolder, "B/b1"); + fakeFolder.remoteModifier().remove("B/b1"); + // expect: local removal + + markForDehydration(fakeFolder, "B/b2"); + fakeFolder.remoteModifier().rename("B/b2", "B/b3"); + // expect: B/b2 is gone, B/b3 is NEW placeholder + + markForDehydration(fakeFolder, "C/c1"); + fakeFolder.localModifier().appendByte("C/c1"); + // expect: no dehydration, upload of c1 + + markForDehydration(fakeFolder, "C/c2"); + fakeFolder.localModifier().appendByte("C/c2"); + fakeFolder.remoteModifier().appendByte("C/c2"); + fakeFolder.remoteModifier().appendByte("C/c2"); + // expect: no dehydration, conflict + + QVERIFY(fakeFolder.syncOnce()); + + auto isDehydrated = [&](const QString &path) { + return cfapi::isSparseFile(fakeFolder.localPath() + path) + && QFileInfo(fakeFolder.localPath() + path).exists(); + }; + auto hasDehydratedDbEntries = [&](const QString &path) { + SyncJournalFileRecord rec; + fakeFolder.syncJournal().getFileRecord(path, &rec); + return rec.isValid() && rec._type == ItemTypeVirtualFile; + }; + + QVERIFY(isDehydrated("A/a1")); + QVERIFY(hasDehydratedDbEntries("A/a1")); + QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_SYNC)); + QCOMPARE(completeSpy.findItem("A/a1")->_type, ItemTypeVirtualFileDehydration); + QCOMPARE(completeSpy.findItem("A/a1")->_file, QStringLiteral("A/a1")); + QVERIFY(isDehydrated("A/a2")); + QVERIFY(hasDehydratedDbEntries("A/a2")); + QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_SYNC)); + QCOMPARE(completeSpy.findItem("A/a2")->_type, ItemTypeVirtualFileDehydration); + + QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b1").exists()); + QVERIFY(!fakeFolder.currentRemoteState().find("B/b1")); + QVERIFY(itemInstruction(completeSpy, "B/b1", CSYNC_INSTRUCTION_REMOVE)); + + QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b2").exists()); + QVERIFY(!fakeFolder.currentRemoteState().find("B/b2")); + QVERIFY(isDehydrated("B/b3")); + QVERIFY(hasDehydratedDbEntries("B/b3")); + QVERIFY(itemInstruction(completeSpy, "B/b2", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "B/b3", CSYNC_INSTRUCTION_NEW)); + + QCOMPARE(fakeFolder.currentRemoteState().find("C/c1")->size, 25); + QVERIFY(itemInstruction(completeSpy, "C/c1", CSYNC_INSTRUCTION_SYNC)); + + QCOMPARE(fakeFolder.currentRemoteState().find("C/c2")->size, 26); + QVERIFY(itemInstruction(completeSpy, "C/c2", CSYNC_INSTRUCTION_CONFLICT)); + cleanup(); + + auto expectedRemoteState = fakeFolder.currentRemoteState(); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentRemoteState(), expectedRemoteState); + + QVERIFY(isDehydrated("A/a1")); + QVERIFY(hasDehydratedDbEntries("A/a1")); + QVERIFY(isDehydrated("A/a2")); + QVERIFY(hasDehydratedDbEntries("A/a2")); + + QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b1").exists()); + QVERIFY(!QFileInfo(fakeFolder.localPath() + "B/b2").exists()); + QVERIFY(isDehydrated("B/b3")); + QVERIFY(hasDehydratedDbEntries("B/b3")); + + QVERIFY(QFileInfo(fakeFolder.localPath() + "C/c1").exists()); + QVERIFY(dbRecord(fakeFolder, "C/c1").isValid()); + QVERIFY(!isDehydrated("C/c1")); + QVERIFY(!hasDehydratedDbEntries("C/c1")); + + QVERIFY(QFileInfo(fakeFolder.localPath() + "C/c2").exists()); + QVERIFY(dbRecord(fakeFolder, "C/c2").isValid()); + QVERIFY(!isDehydrated("C/c2")); + QVERIFY(!hasDehydratedDbEntries("C/c2")); + } + + void testWipeVirtualSuffixFiles() + { + FakeFolder fakeFolder{ FileInfo{} }; + setupVfs(fakeFolder); + + // Create a suffix-vfs baseline + + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().mkdir("A/B"); + fakeFolder.remoteModifier().insert("f1"); + fakeFolder.remoteModifier().insert("A/a1"); + fakeFolder.remoteModifier().insert("A/a3"); + fakeFolder.remoteModifier().insert("A/B/b1"); + fakeFolder.localModifier().mkdir("A"); + fakeFolder.localModifier().mkdir("A/B"); + fakeFolder.localModifier().insert("f2"); + fakeFolder.localModifier().insert("A/a2"); + fakeFolder.localModifier().insert("A/B/b2"); + + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "f1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a1"); + CFVERIFY_VIRTUAL(fakeFolder, "A/a3"); + CFVERIFY_VIRTUAL(fakeFolder, "A/B/b1"); + + // Make local changes to a3 + fakeFolder.localModifier().remove("A/a3"); + fakeFolder.localModifier().insert("A/a3", 100); + + // Now wipe the virtuals + SyncEngine::wipeVirtualFiles(fakeFolder.localPath(), fakeFolder.syncJournal(), *fakeFolder.syncEngine().syncOptions()._vfs); + + CFVERIFY_GONE(fakeFolder, "f1"); + CFVERIFY_GONE(fakeFolder, "A/a1"); + QVERIFY(QFileInfo(fakeFolder.localPath() + "A/a3").exists()); + QVERIFY(!dbRecord(fakeFolder, "A/a3").isValid()); + CFVERIFY_GONE(fakeFolder, "A/B/b1"); + + fakeFolder.switchToVfs(QSharedPointer(new VfsOff)); + ItemCompletedSpy completeSpy(fakeFolder); + QVERIFY(fakeFolder.syncOnce()); + + QVERIFY(fakeFolder.currentLocalState().find("A")); + QVERIFY(fakeFolder.currentLocalState().find("A/B")); + QVERIFY(fakeFolder.currentLocalState().find("A/B/b1")); + QVERIFY(fakeFolder.currentLocalState().find("A/B/b2")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1")); + QVERIFY(fakeFolder.currentLocalState().find("A/a2")); + QVERIFY(fakeFolder.currentLocalState().find("A/a3")); + QVERIFY(fakeFolder.currentLocalState().find("f1")); + QVERIFY(fakeFolder.currentLocalState().find("f2")); + + // a3 has a conflict + QVERIFY(itemInstruction(completeSpy, "A/a3", CSYNC_INSTRUCTION_CONFLICT)); + + // conflict files should exist + QCOMPARE(fakeFolder.syncJournal().conflictRecordPaths().size(), 1); + } + + void testNewVirtuals() + { + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + fakeFolder.remoteModifier().mkdir("local"); + fakeFolder.remoteModifier().mkdir("online"); + fakeFolder.remoteModifier().mkdir("unspec"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse); + + // Test 1: root is Unspecified + fakeFolder.remoteModifier().insert("file1"); + fakeFolder.remoteModifier().insert("online/file1"); + fakeFolder.remoteModifier().insert("local/file1"); + fakeFolder.remoteModifier().insert("unspec/file1"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "file1"); + CFVERIFY_VIRTUAL(fakeFolder, "online/file1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "local/file1"); + CFVERIFY_VIRTUAL(fakeFolder, "unspec/file1"); + + // Test 2: change root to AlwaysLocal + setPinState(fakeFolder.localPath(), PinState::AlwaysLocal, cfapi::Recurse); + // Need to force pin state for the subfolders again + setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse); + + fakeFolder.remoteModifier().insert("file2"); + fakeFolder.remoteModifier().insert("online/file2"); + fakeFolder.remoteModifier().insert("local/file2"); + fakeFolder.remoteModifier().insert("unspec/file2"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_NONVIRTUAL(fakeFolder, "file2"); + CFVERIFY_VIRTUAL(fakeFolder, "online/file2"); + CFVERIFY_NONVIRTUAL(fakeFolder, "local/file2"); + CFVERIFY_VIRTUAL(fakeFolder, "unspec/file2"); + + // root file1 was hydrated due to its new pin state + CFVERIFY_NONVIRTUAL(fakeFolder, "file1"); + + // file1 is unchanged in the explicitly pinned subfolders + CFVERIFY_VIRTUAL(fakeFolder, "online/file1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "local/file1"); + CFVERIFY_VIRTUAL(fakeFolder, "unspec/file1"); + + // Test 3: change root to OnlineOnly + setPinState(fakeFolder.localPath(), PinState::OnlineOnly, cfapi::Recurse); + // Need to force pin state for the subfolders again + setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse); + + fakeFolder.remoteModifier().insert("file3"); + fakeFolder.remoteModifier().insert("online/file3"); + fakeFolder.remoteModifier().insert("local/file3"); + fakeFolder.remoteModifier().insert("unspec/file3"); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "file3"); + CFVERIFY_VIRTUAL(fakeFolder, "online/file3"); + CFVERIFY_NONVIRTUAL(fakeFolder, "local/file3"); + CFVERIFY_VIRTUAL(fakeFolder, "unspec/file3"); + + // root file1 was dehydrated due to its new pin state + CFVERIFY_VIRTUAL(fakeFolder, "file1"); + + // file1 is unchanged in the explicitly pinned subfolders + CFVERIFY_VIRTUAL(fakeFolder, "online/file1"); + CFVERIFY_NONVIRTUAL(fakeFolder, "local/file1"); + CFVERIFY_VIRTUAL(fakeFolder, "unspec/file1"); + } + + void testAvailability() + { + FakeFolder fakeFolder{ FileInfo() }; + auto vfs = setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + fakeFolder.remoteModifier().mkdir("local"); + fakeFolder.remoteModifier().mkdir("local/sub"); + fakeFolder.remoteModifier().mkdir("online"); + fakeFolder.remoteModifier().mkdir("online/sub"); + fakeFolder.remoteModifier().mkdir("unspec"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse); + setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::Recurse); + + fakeFolder.remoteModifier().insert("file1"); + fakeFolder.remoteModifier().insert("online/file1"); + fakeFolder.remoteModifier().insert("online/file2"); + fakeFolder.remoteModifier().insert("local/file1"); + fakeFolder.remoteModifier().insert("local/file2"); + fakeFolder.remoteModifier().insert("unspec/file1"); + QVERIFY(fakeFolder.syncOnce()); + + // root is unspecified + QCOMPARE(*vfs->availability("file1"), VfsItemAvailability::AllDehydrated); + QCOMPARE(*vfs->availability("local"), VfsItemAvailability::AlwaysLocal); + QCOMPARE(*vfs->availability("local/file1"), VfsItemAvailability::AlwaysLocal); + QCOMPARE(*vfs->availability("online"), VfsItemAvailability::OnlineOnly); + QCOMPARE(*vfs->availability("online/file1"), VfsItemAvailability::OnlineOnly); + QCOMPARE(*vfs->availability("unspec"), VfsItemAvailability::AllDehydrated); + QCOMPARE(*vfs->availability("unspec/file1"), VfsItemAvailability::AllDehydrated); + + // Subitem pin states can ruin "pure" availabilities + setPinState(fakeFolder.localPath() + "local/sub", PinState::OnlineOnly, cfapi::NoRecurse); + QCOMPARE(*vfs->availability("local"), VfsItemAvailability::AllHydrated); + setPinState(fakeFolder.localPath() + "online/sub", PinState::Unspecified, cfapi::NoRecurse); + QCOMPARE(*vfs->availability("online"), VfsItemAvailability::AllDehydrated); + + triggerDownload(fakeFolder, "unspec/file1"); + setPinState(fakeFolder.localPath() + "local/file2", PinState::OnlineOnly, cfapi::NoRecurse); + setPinState(fakeFolder.localPath() + "online/file2", PinState::AlwaysLocal, cfapi::NoRecurse); + QVERIFY(fakeFolder.syncOnce()); + + QCOMPARE(*vfs->availability("unspec"), VfsItemAvailability::AllHydrated); + QCOMPARE(*vfs->availability("local"), VfsItemAvailability::Mixed); + QCOMPARE(*vfs->availability("online"), VfsItemAvailability::Mixed); + + vfs->setPinState("local", PinState::AlwaysLocal); + vfs->setPinState("online", PinState::OnlineOnly); + QVERIFY(fakeFolder.syncOnce()); + + QCOMPARE(*vfs->availability("online"), VfsItemAvailability::OnlineOnly); + QCOMPARE(*vfs->availability("local"), VfsItemAvailability::AlwaysLocal); + + auto r = vfs->availability("nonexistant"); + QVERIFY(!r); + QCOMPARE(r.error(), Vfs::AvailabilityError::NoSuchItem); + } + + void testPinStateLocals() + { + FakeFolder fakeFolder{ FileInfo() }; + auto vfs = setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + fakeFolder.remoteModifier().mkdir("local"); + fakeFolder.remoteModifier().mkdir("online"); + fakeFolder.remoteModifier().mkdir("unspec"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::NoRecurse); + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::NoRecurse); + setPinState(fakeFolder.localPath() + "unspec", PinState::Unspecified, cfapi::NoRecurse); + + fakeFolder.localModifier().insert("file1"); + fakeFolder.localModifier().insert("online/file1"); + fakeFolder.localModifier().insert("online/file2"); + fakeFolder.localModifier().insert("local/file1"); + fakeFolder.localModifier().insert("unspec/file1"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + // root is unspecified + QCOMPARE(*vfs->pinState("file1"), PinState::Unspecified); + QCOMPARE(*vfs->pinState("local/file1"), PinState::AlwaysLocal); + QCOMPARE(*vfs->pinState("online/file1"), PinState::Unspecified); + QCOMPARE(*vfs->pinState("unspec/file1"), PinState::Unspecified); + + // Sync again: bad pin states of new local files usually take effect on second sync + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + // When a file in an online-only folder is renamed, it retains its pin + fakeFolder.localModifier().rename("online/file1", "online/file1rename"); + fakeFolder.remoteModifier().rename("online/file2", "online/file2rename"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(*vfs->pinState("online/file1rename"), PinState::Unspecified); + QCOMPARE(*vfs->pinState("online/file2rename"), PinState::Unspecified); + + // When a folder is renamed, the pin states inside should be retained + fakeFolder.localModifier().rename("online", "onlinerenamed1"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(*vfs->pinState("onlinerenamed1"), PinState::OnlineOnly); + QCOMPARE(*vfs->pinState("onlinerenamed1/file1rename"), PinState::Unspecified); + + fakeFolder.remoteModifier().rename("onlinerenamed1", "onlinerenamed2"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(*vfs->pinState("onlinerenamed2"), PinState::OnlineOnly); + QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::Unspecified); + + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + // When a file is deleted and later a new file has the same name, the old pin + // state isn't preserved. + QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::Unspecified); + fakeFolder.remoteModifier().remove("onlinerenamed2/file1rename"); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(!vfs->pinState("onlinerenamed2/file1rename")); + fakeFolder.remoteModifier().insert("onlinerenamed2/file1rename"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::OnlineOnly); + + // When a file is hydrated or dehydrated due to pin state it retains its pin state + vfs->setPinState("onlinerenamed2/file1rename", PinState::AlwaysLocal); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(fakeFolder.currentLocalState().find("onlinerenamed2/file1rename")); + QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::AlwaysLocal); + + vfs->setPinState("onlinerenamed2", PinState::Unspecified); + vfs->setPinState("onlinerenamed2/file1rename", PinState::OnlineOnly); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "onlinerenamed2/file1rename"); + + QCOMPARE(*vfs->pinState("onlinerenamed2/file1rename"), PinState::OnlineOnly); + } + + void testIncompatiblePins() + { + FakeFolder fakeFolder{ FileInfo() }; + auto vfs = setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + fakeFolder.remoteModifier().mkdir("local"); + fakeFolder.remoteModifier().mkdir("online"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + setPinState(fakeFolder.localPath() + "local", PinState::AlwaysLocal, cfapi::NoRecurse); + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::NoRecurse); + + fakeFolder.localModifier().insert("local/file1"); + fakeFolder.localModifier().insert("online/file1"); + QVERIFY(fakeFolder.syncOnce()); + + markForDehydration(fakeFolder, "local/file1"); + triggerDownload(fakeFolder, "online/file1"); + + // the sync sets the changed files pin states to unspecified + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_NONVIRTUAL(fakeFolder, "online/file1"); + CFVERIFY_VIRTUAL(fakeFolder, "local/file1"); + + QCOMPARE(*vfs->pinState("online/file1"), PinState::Unspecified); + QCOMPARE(*vfs->pinState("local/file1"), PinState::Unspecified); + + // no change on another sync + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_NONVIRTUAL(fakeFolder, "online/file1"); + CFVERIFY_VIRTUAL(fakeFolder, "local/file1"); + } + + void testOpeningOnlineFileTriggersDownload_data() + { + QTest::addColumn("errorKind"); + QTest::newRow("no error") << static_cast(NoError); + QTest::newRow("400") << 400; + QTest::newRow("401") << 401; + QTest::newRow("403") << 403; + QTest::newRow("404") << 404; + QTest::newRow("500") << 500; + QTest::newRow("503") << 503; + QTest::newRow("Timeout") << static_cast(Timeout); + } + + void testOpeningOnlineFileTriggersDownload() + { + QFETCH(int, errorKind); + + FakeFolder fakeFolder{ FileInfo() }; + setupVfs(fakeFolder); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + fakeFolder.remoteModifier().mkdir("online"); + fakeFolder.remoteModifier().mkdir("online/sub"); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + setPinState(fakeFolder.localPath() + "online", PinState::OnlineOnly, cfapi::Recurse); + + fakeFolder.remoteModifier().insert("online/sub/file1", 10 * 1024 * 1024); + QVERIFY(fakeFolder.syncOnce()); + + CFVERIFY_VIRTUAL(fakeFolder, "online/sub/file1"); + + // Setup error case if needed + if (errorKind == Timeout) { + fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &req, QIODevice *) -> QNetworkReply * { + if (req.url().path().endsWith("online/sub/file1")) { + return new FakeHangingReply(op, req, this); + } + return nullptr; + }); + } else if (errorKind != NoError) { + fakeFolder.serverErrorPaths().append("online/sub/file1", errorKind); + } + + // So the test that test timeout finishes fast + QScopedValueRollback setHttpTimeout(AbstractNetworkJob::httpTimeout, errorKind == Timeout ? 1 : 10000); + + // Simulate another process requesting the open + QEventLoop loop; + bool openResult = false; + bool readResult = false; + std::thread t([&] { + QFile file(fakeFolder.localPath() + "online/sub/file1"); + openResult = file.open(QFile::ReadOnly); + readResult = !file.readAll().isEmpty(); + file.close(); + QMetaObject::invokeMethod(&loop, &QEventLoop::quit, Qt::QueuedConnection); + }); + loop.exec(); + t.join(); + + if (errorKind == NoError) { + CFVERIFY_NONVIRTUAL(fakeFolder, "online/sub/file1"); + } else { + CFVERIFY_VIRTUAL(fakeFolder, "online/sub/file1"); + } + + // Nothing should change + ItemCompletedSpy completeSpy(fakeFolder); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(completeSpy.isEmpty()); + + if (errorKind == NoError) { + CFVERIFY_NONVIRTUAL(fakeFolder, "online/sub/file1"); + } else { + CFVERIFY_VIRTUAL(fakeFolder, "online/sub/file1"); + } + } +}; + +QTEST_GUILESS_MAIN(TestSyncCfApi) +#include "testsynccfapi.moc"