diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index fab99be586e3c..cd124bd3b653b 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -24,6 +24,15 @@ set(libsync_SRCS capabilities.cpp clientproxy.h clientproxy.cpp + clientstatusreporting.h + clientstatusreporting.cpp + clientstatusreportingcommon.h + clientstatusreportingcommon.cpp + clientstatusreportingdatabase.h + clientstatusreportingdatabase.cpp + clientstatusreportingnetwork.h + clientstatusreportingnetwork.cpp + clientstatusreportingrecord.h cookiejar.h cookiejar.cpp discovery.h @@ -171,7 +180,7 @@ IF (NOT APPLE) ) ENDIF(NOT APPLE) -find_package(Qt5 REQUIRED COMPONENTS WebSockets Xml) +find_package(Qt5 REQUIRED COMPONENTS WebSockets Xml Sql) add_library(nextcloudsync SHARED ${libsync_SRCS}) add_library(Nextcloud::sync ALIAS nextcloudsync) @@ -186,6 +195,7 @@ target_link_libraries(nextcloudsync Qt5::Network Qt5::WebSockets Qt5::Xml + Qt5::Sql ) if (NOT TOKEN_AUTH_ONLY) diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index 9e9101b01e6b3..bea76c53b9069 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -284,6 +284,25 @@ void Account::setPushNotificationsReconnectInterval(int interval) _pushNotificationsReconnectTimer.setInterval(interval); } +void Account::trySetupClientStatusReporting() +{ + if (!_capabilities.isClientStatusReportingEnabled()) { + _clientStatusReporting.reset(); + return; + } + + if (!_clientStatusReporting) { + _clientStatusReporting = std::make_unique(this); + } +} + +void Account::reportClientStatus(const ClientStatusReportingStatus status) const +{ + if (_clientStatusReporting) { + _clientStatusReporting->reportClientStatus(status); + } +} + void Account::trySetupPushNotifications() { // Stop the timer to prevent parallel setup attempts @@ -669,6 +688,8 @@ void Account::setCapabilities(const QVariantMap &caps) setupUserStatusConnector(); trySetupPushNotifications(); + + trySetupClientStatusReporting(); } void Account::setupUserStatusConnector() diff --git a/src/libsync/account.h b/src/libsync/account.h index 523fc7137cd9d..e118b53fb2f25 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -33,6 +33,7 @@ #include "capabilities.h" #include "clientsideencryption.h" +#include "clientstatusreporting.h" #include "common/utility.h" #include "syncfileitem.h" @@ -305,6 +306,10 @@ class OWNCLOUDSYNC_EXPORT Account : public QObject [[nodiscard]] PushNotifications *pushNotifications() const; void setPushNotificationsReconnectInterval(int interval); + void trySetupClientStatusReporting(); + + void reportClientStatus(const ClientStatusReportingStatus status) const; + [[nodiscard]] std::shared_ptr userStatusConnector() const; void setLockFileState(const QString &serverRelativePath, @@ -439,6 +444,8 @@ private slots: PushNotifications *_pushNotifications = nullptr; + std::unique_ptr _clientStatusReporting; + std::shared_ptr _userStatusConnector; QHash> _lockStatusChangeInprogress; diff --git a/src/libsync/bulkpropagatorjob.cpp b/src/libsync/bulkpropagatorjob.cpp index 0f0a7db3fcfa4..58e1ab82f0c95 100644 --- a/src/libsync/bulkpropagatorjob.cpp +++ b/src/libsync/bulkpropagatorjob.cpp @@ -366,6 +366,9 @@ void BulkPropagatorJob::slotPutFinishedOneFile(const BulkUploadItem &singleFile, singleFile._item->_requestId = job->requestId(); if (singleFile._item->_httpErrorCode != 200) { commonErrorHandling(singleFile._item, fileReply[QStringLiteral("message")].toString()); + const auto exceptionParsed = getExceptionFromReply(job->reply()); + singleFile._item->_errorExceptionName = exceptionParsed.first; + singleFile._item->_errorExceptionMessage = exceptionParsed.second; return; } diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index 386880b12ab03..14c51744a228b 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -198,6 +198,15 @@ bool Capabilities::hasActivities() const return _capabilities.contains("activity"); } +bool Capabilities::isClientStatusReportingEnabled() const +{ + if (!_capabilities.contains(QStringLiteral("security_guard"))) { + return false; + } + const auto securityGuardCaps = _capabilities[QStringLiteral("security_guard")].toMap(); + return securityGuardCaps.contains(QStringLiteral("diagnostics")) && securityGuardCaps[QStringLiteral("diagnostics")].toBool(); +} + QList Capabilities::supportedChecksumTypes() const { QList list; diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h index 0ebc25d80901d..9a7f587f00b99 100644 --- a/src/libsync/capabilities.h +++ b/src/libsync/capabilities.h @@ -97,6 +97,8 @@ class OWNCLOUDSYNC_EXPORT Capabilities /// return true if the activity app is enabled [[nodiscard]] bool hasActivities() const; + [[nodiscard]] bool isClientStatusReportingEnabled() const; + /** * Returns the checksum types the server understands. * diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3278c450dfeba..dd9263b024d62 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1262,6 +1262,7 @@ bool ClientSideEncryption::sensitiveDataRemaining() const void ClientSideEncryption::failedToInitialize(const AccountPtr &account) { forgetSensitiveData(account); + account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); Q_EMIT initializationFinished(); } @@ -1775,6 +1776,7 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) if (metadataKeys.isEmpty()) { qCDebug(lcCse()) << "Could not migrate. No metadata keys found!"; + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); return; } @@ -1787,6 +1789,7 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) if (_metadataKey.isEmpty()) { qCDebug(lcCse()) << "Could not setup existing metadata with missing metadataKeys!"; + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); return; } @@ -1861,6 +1864,7 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) } else { _metadataKey.clear(); _files.clear(); + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); return; } } @@ -1899,6 +1903,7 @@ QByteArray FolderMetadata::decryptData(const QByteArray &data) const if (decryptResult.isEmpty()) { qCDebug(lcCse()) << "ERROR. Could not decrypt the metadata key"; + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); return {}; } return QByteArray::fromBase64(decryptResult); @@ -1916,6 +1921,7 @@ QByteArray FolderMetadata::decryptDataUsingKey(const QByteArray &data, if (decryptResult.isEmpty()) { qCDebug(lcCse()) << "ERROR. Could not decrypt"; + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); return {}; } @@ -1979,6 +1985,7 @@ QByteArray FolderMetadata::encryptedMetadata() const { if (_metadataKey.isEmpty()) { qCDebug(lcCse) << "Metadata generation failed! Empty metadata key!"; + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); return {}; } const auto version = _account->capabilities().clientSideEncryptionVersion(); @@ -2000,6 +2007,7 @@ QByteArray FolderMetadata::encryptedMetadata() const { QString encryptedEncrypted = encryptJsonObject(encryptedDoc.toJson(QJsonDocument::Compact), _metadataKey); if (encryptedEncrypted.isEmpty()) { + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); qCDebug(lcCse) << "Metadata generation failed!"; } QJsonObject file; @@ -2087,6 +2095,7 @@ bool FolderMetadata::moveFromFileDropToFiles() if (decryptedKey.isEmpty() || decryptedAuthenticationTag.isEmpty() || decryptedInitializationVector.isEmpty()) { qCDebug(lcCseMetadata) << "failed to decrypt filedrop entry" << it.key(); + _account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); continue; } diff --git a/src/libsync/clientstatusreporting.cpp b/src/libsync/clientstatusreporting.cpp new file mode 100644 index 0000000000000..682ff206e1c31 --- /dev/null +++ b/src/libsync/clientstatusreporting.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#include "clientstatusreporting.h" + +#include "account.h" +#include "clientstatusreportingdatabase.h" +#include "clientstatusreportingnetwork.h" +#include "clientstatusreportingrecord.h" + +namespace OCC +{ +Q_LOGGING_CATEGORY(lcClientStatusReporting, "nextcloud.sync.clientstatusreporting", QtInfoMsg) + +ClientStatusReporting::ClientStatusReporting(Account *account) +{ + for (int i = 0; i < static_cast(ClientStatusReportingStatus::Count); ++i) { + const auto statusString = clientStatusstatusStringFromNumber(static_cast(i)); + _statusStrings[i] = statusString; + } + + if (_statusStrings.size() < static_cast(ClientStatusReportingStatus::Count)) { + return; + } + + _database = QSharedPointer::create(account); + if (!_database->isInitialized()) { + return; + } + + _reporter = std::make_unique(account, _database); + if (!_reporter->isInitialized()) { + return; + } + + _isInitialized = true; +} + +ClientStatusReporting::~ClientStatusReporting() = default; + +void ClientStatusReporting::reportClientStatus(const ClientStatusReportingStatus status) const +{ + if (!_isInitialized) { + return; + } + + Q_ASSERT(static_cast(status) >= 0 && static_cast(status) < static_cast(ClientStatusReportingStatus::Count)); + if (static_cast(status) < 0 || static_cast(status) >= static_cast(ClientStatusReportingStatus::Count)) { + qCDebug(lcClientStatusReporting) << "Trying to report invalid status:" << static_cast(status); + return; + } + + ClientStatusReportingRecord record; + record._name = _statusStrings[static_cast(status)]; + record._status = static_cast(status); + record._lastOccurence = QDateTime::currentDateTimeUtc().toMSecsSinceEpoch(); + const auto result = _database->setClientStatusReportingRecord(record); + if (!result.isValid()) { + qCDebug(lcClientStatusReporting) << "Could not report client status:" << result.error(); + } +} +} diff --git a/src/libsync/clientstatusreporting.h b/src/libsync/clientstatusreporting.h new file mode 100644 index 0000000000000..54aaf57853363 --- /dev/null +++ b/src/libsync/clientstatusreporting.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudlib.h" +#include +#include "clientstatusreportingcommon.h" + +#include + +#include +#include +#include + +namespace OCC { + +class Account; +class ClientStatusReportingDatabase; +class ClientStatusReportingNetwork; +struct ClientStatusReportingRecord; + +class OWNCLOUDSYNC_EXPORT ClientStatusReporting +{ +public: + explicit ClientStatusReporting(Account *account); + ~ClientStatusReporting(); + +private: + // reporting must happen via Account + void reportClientStatus(const ClientStatusReportingStatus status) const; + + bool _isInitialized = false; + + QHash _statusStrings; + + QSharedPointer _database; + + std::unique_ptr _reporter; + + friend class Account; +}; +} diff --git a/src/libsync/clientstatusreportingcommon.cpp b/src/libsync/clientstatusreportingcommon.cpp new file mode 100644 index 0000000000000..fabbf2b0fbbaf --- /dev/null +++ b/src/libsync/clientstatusreportingcommon.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "clientstatusreportingcommon.h" +#include + +namespace OCC { +Q_LOGGING_CATEGORY(lcClientStatusReportingCommon, "nextcloud.sync.clientstatusreportingcommon", QtInfoMsg) + +QByteArray clientStatusstatusStringFromNumber(const ClientStatusReportingStatus status) +{ + Q_ASSERT(static_cast(status) >= 0 && static_cast(status) < static_cast(ClientStatusReportingStatus::Count)); + if (static_cast(status) < 0 || static_cast(status) >= static_cast(ClientStatusReportingStatus::Count)) { + qCDebug(lcClientStatusReportingCommon) << "Invalid status:" << static_cast(status); + return {}; + } + + switch (status) { + case ClientStatusReportingStatus::DownloadError_Cannot_Create_File: + return QByteArrayLiteral("DownloadResult.CANNOT_CREATE_FILE"); + case ClientStatusReportingStatus::DownloadError_Conflict: + return QByteArrayLiteral("DownloadResult.CONFLICT"); + case ClientStatusReportingStatus::DownloadError_ConflictCaseClash: + return QByteArrayLiteral("DownloadResult.CONFLICT_CASECLASH"); + case ClientStatusReportingStatus::DownloadError_ConflictInvalidCharacters: + return QByteArrayLiteral("DownloadResult.CONFLICT_INVALID_CHARACTERS"); + case ClientStatusReportingStatus::DownloadError_No_Free_Space: + return QByteArrayLiteral("DownloadResult.NO_FREE_SPACE"); + case ClientStatusReportingStatus::DownloadError_ServerError: + return QByteArrayLiteral("DownloadResult.SERVER_ERROR"); + case ClientStatusReportingStatus::DownloadError_Virtual_File_Hydration_Failure: + return QByteArrayLiteral("DownloadResult.VIRTUAL_FILE_HYDRATION_FAILURE"); + case ClientStatusReportingStatus::E2EeError_GeneralError: + return QByteArrayLiteral("E2EeError.General"); + case ClientStatusReportingStatus::UploadError_Conflict: + return QByteArrayLiteral("UploadResult.CONFLICT_CASECLASH"); + case ClientStatusReportingStatus::UploadError_ConflictInvalidCharacters: + return QByteArrayLiteral("UploadResult.CONFLICT_INVALID_CHARACTERS"); + case ClientStatusReportingStatus::UploadError_No_Free_Space: + return QByteArrayLiteral("UploadResult.NO_FREE_SPACE"); + case ClientStatusReportingStatus::UploadError_No_Write_Permissions: + return QByteArrayLiteral("UploadResult.NO_WRITE_PERMISSIONS"); + case ClientStatusReportingStatus::UploadError_ServerError: + return QByteArrayLiteral("UploadResult.SERVER_ERROR"); + case ClientStatusReportingStatus::UploadError_Virus_Detected: + return QByteArrayLiteral("UploadResult.VIRUS_DETECTED"); + case ClientStatusReportingStatus::Count: + return {}; + }; + return {}; +} +} diff --git a/src/libsync/clientstatusreportingcommon.h b/src/libsync/clientstatusreportingcommon.h new file mode 100644 index 0000000000000..4e08ca1efa6fd --- /dev/null +++ b/src/libsync/clientstatusreportingcommon.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudlib.h" +#include + +namespace OCC { +enum class ClientStatusReportingStatus { + DownloadError_Cannot_Create_File = 0, + DownloadError_Conflict, + DownloadError_ConflictCaseClash, + DownloadError_ConflictInvalidCharacters, + DownloadError_No_Free_Space, + DownloadError_ServerError, + DownloadError_Virtual_File_Hydration_Failure, + E2EeError_GeneralError, + UploadError_Conflict, + UploadError_ConflictInvalidCharacters, + UploadError_No_Free_Space, + UploadError_No_Write_Permissions, + UploadError_ServerError, + UploadError_Virus_Detected, + Count, +}; +QByteArray OWNCLOUDSYNC_EXPORT clientStatusstatusStringFromNumber(const ClientStatusReportingStatus status); +} diff --git a/src/libsync/clientstatusreportingdatabase.cpp b/src/libsync/clientstatusreportingdatabase.cpp new file mode 100644 index 0000000000000..169df5e13fffe --- /dev/null +++ b/src/libsync/clientstatusreportingdatabase.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#include "clientstatusreportingdatabase.h" + +#include "account.h" +#include + +#include +#include +#include + +namespace +{ +constexpr auto lastSentReportTimestamp = "lastClientStatusReportSentTime"; +constexpr auto statusNamesHash = "statusNamesHash"; +} + +namespace OCC +{ +Q_LOGGING_CATEGORY(lcClientStatusReportingDatabase, "nextcloud.sync.clientstatusreportingdatabase", QtInfoMsg) + +ClientStatusReportingDatabase::ClientStatusReportingDatabase(const Account *account) +{ + const auto dbPath = makeDbPath(account); + _database = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE")); + _database.setDatabaseName(dbPath); + + if (!_database.open()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not setup client reporting, database connection error."; + return; + } + + QSqlQuery query; + const auto prepareResult = + query.prepare(QStringLiteral("CREATE TABLE IF NOT EXISTS clientstatusreporting(" + "name VARCHAR(4096) PRIMARY KEY," + "status INTEGER(8)," + "count INTEGER," + "lastOccurrence INTEGER(8))")); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not setup client clientstatusreporting table:" << query.lastError().text(); + return; + } + + if (!query.prepare(QStringLiteral("CREATE TABLE IF NOT EXISTS keyvalue(key VARCHAR(4096), value VARCHAR(4096), PRIMARY KEY(key))")) || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not setup client keyvalue table:" << query.lastError().text(); + return; + } + + if (!updateStatusNamesHash()) { + return; + } + + _isInitialized = true; +} + +ClientStatusReportingDatabase::~ClientStatusReportingDatabase() +{ + if (_database.isOpen()) { + _database.close(); + } +} + +QVector ClientStatusReportingDatabase::getClientStatusReportingRecords() const +{ + QVector records; + + QMutexLocker locker(&_mutex); + + QSqlQuery query; + if (!query.prepare(QStringLiteral("SELECT * FROM clientstatusreporting")) || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not get records from clientstatusreporting:" << query.lastError().text(); + return records; + } + + while (query.next()) { + ClientStatusReportingRecord record; + record._status = query.value(query.record().indexOf(QStringLiteral("status"))).toLongLong(); + record._name = query.value(query.record().indexOf(QStringLiteral("name"))).toByteArray(); + record._numOccurences = query.value(query.record().indexOf(QStringLiteral("count"))).toLongLong(); + record._lastOccurence = query.value(query.record().indexOf(QStringLiteral("lastOccurrence"))).toLongLong(); + records.push_back(record); + } + return records; +} + +Result ClientStatusReportingDatabase::deleteClientStatusReportingRecords() const +{ + QSqlQuery query; + if (!query.prepare(QStringLiteral("DELETE FROM clientstatusreporting")) || !query.exec()) { + const auto errorMessage = query.lastError().text(); + qCDebug(lcClientStatusReportingDatabase) << "Could not delete records from clientstatusreporting:" << errorMessage; + return errorMessage; + } + return {}; +} + +Result ClientStatusReportingDatabase::setClientStatusReportingRecord(const ClientStatusReportingRecord &record) const +{ + Q_ASSERT(record.isValid()); + if (!record.isValid()) { + qCDebug(lcClientStatusReportingDatabase) << "Failed to set ClientStatusReportingRecord"; + return {QStringLiteral("Invalid parameter")}; + } + + const auto recordCopy = record; + + QMutexLocker locker(&_mutex); + + QSqlQuery query; + + const auto prepareResult = query.prepare( + QStringLiteral("INSERT OR REPLACE INTO clientstatusreporting (name, status, count, lastOccurrence) VALUES(:name, :status, :count, :lastOccurrence) ON CONFLICT(name) " + "DO UPDATE SET count = count + 1, lastOccurrence = :lastOccurrence;")); + query.bindValue(QStringLiteral(":name"), recordCopy._name); + query.bindValue(QStringLiteral(":status"), recordCopy._status); + query.bindValue(QStringLiteral(":count"), 1); + query.bindValue(QStringLiteral(":lastOccurrence"), recordCopy._lastOccurence); + + if (!prepareResult || !query.exec()) { + const auto errorMessage = query.lastError().text(); + qCDebug(lcClientStatusReportingDatabase) << "Could not report client status:" << errorMessage; + return errorMessage; + } + + return {}; +} + +QString ClientStatusReportingDatabase::makeDbPath(const Account *account) const +{ + if (!dbPathForTesting.isEmpty()) { + return dbPathForTesting; + } + const auto databaseId = QStringLiteral("%1@%2").arg(account->davUser(), account->url().toString()); + const auto databaseIdHash = QCryptographicHash::hash(databaseId.toUtf8(), QCryptographicHash::Md5); + + return ConfigFile().configPath() + QStringLiteral(".userdata_%1.db").arg(QString::fromLatin1(databaseIdHash.left(6).toHex())); +} + +bool ClientStatusReportingDatabase::updateStatusNamesHash() const +{ + QByteArray statusNamesContatenated; + for (int i = 0; i < static_cast(ClientStatusReportingStatus::Count); ++i) { + statusNamesContatenated += clientStatusstatusStringFromNumber(static_cast(i)); + } + statusNamesContatenated += QByteArray::number(static_cast(ClientStatusReportingStatus::Count)); + const auto statusNamesHashCurrent = QCryptographicHash::hash(statusNamesContatenated, QCryptographicHash::Md5).toHex(); + const auto statusNamesHashFromDb = getStatusNamesHash(); + + if (statusNamesHashCurrent != statusNamesHashFromDb) { + auto result = deleteClientStatusReportingRecords(); + if (!result.isValid()) { + return false; + } + + result = setStatusNamesHash(statusNamesHashCurrent); + if (!result.isValid()) { + return false; + } + } + return true; +} + +QVector ClientStatusReportingDatabase::getTableColumns(const QString &table) const +{ + QVector columns; + QSqlQuery query; + const auto prepareResult = query.prepare(QStringLiteral("PRAGMA table_info('%1');").arg(table)); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could get table columns" << query.lastError().text(); + return columns; + } + while (query.next()) { + columns.append(query.value(1).toByteArray()); + } + return columns; +} + +bool ClientStatusReportingDatabase::addColumn(const QString &tableName, const QString &columnName, const QString &dataType, const bool withIndex) const +{ + const auto columns = getTableColumns(tableName); + const auto latin1ColumnName = columnName.toLatin1(); + if (columns.indexOf(latin1ColumnName) == -1) { + QSqlQuery query; + const auto prepareResult = query.prepare(QStringLiteral("ALTER TABLE %1 ADD COLUMN %2 %3;").arg(tableName, columnName, dataType)); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << QStringLiteral("Failed to update table %1 structure: add %2 column").arg(tableName, columnName) << query.lastError().text(); + return false; + } + + if (withIndex) { + const auto prepareResult = query.prepare(QStringLiteral("CREATE INDEX %1_%2 ON %1(%2);").arg(tableName, columnName)); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << QStringLiteral("Failed to update table %1 structure: create index %2 column").arg(tableName, columnName) << query.lastError().text(); + return false; + } + } + } + return true; +} + +quint64 ClientStatusReportingDatabase::getLastSentReportTimestamp() const +{ + QMutexLocker locker(&_mutex); + QSqlQuery query; + const auto prepareResult = query.prepare(QStringLiteral("SELECT value FROM keyvalue WHERE key = (:key)")); + query.bindValue(QStringLiteral(":key"), lastSentReportTimestamp); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not get last sent report timestamp from keyvalue table. No such record:" << lastSentReportTimestamp; + return 0; + } + if (!query.next()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not get last sent report timestamp from keyvalue table:" << query.lastError().text(); + return 0; + } + return query.value(query.record().indexOf(QStringLiteral("value"))).toULongLong(); +} + +Result ClientStatusReportingDatabase::setStatusNamesHash(const QByteArray &hash) const +{ + QMutexLocker locker(&_mutex); + QSqlQuery query; + const auto prepareResult = query.prepare(QStringLiteral("INSERT OR REPLACE INTO keyvalue (key, value) VALUES(:key, :value);")); + query.bindValue(QStringLiteral(":key"), statusNamesHash); + query.bindValue(QStringLiteral(":value"), hash); + if (!prepareResult || !query.exec()) { + const auto errorMessage = query.lastError().text(); + qCDebug(lcClientStatusReportingDatabase) << "Could not set status names hash." << errorMessage; + return errorMessage; + } + return {}; +} + +QByteArray ClientStatusReportingDatabase::getStatusNamesHash() const +{ + QMutexLocker locker(&_mutex); + QSqlQuery query; + const auto prepareResult = query.prepare(QStringLiteral("SELECT value FROM keyvalue WHERE key = (:key)")); + query.bindValue(QStringLiteral(":key"), statusNamesHash); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not get status names hash. No such record:" << statusNamesHash; + return {}; + } + if (!query.next()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not get status names hash:" << query.lastError().text(); + return {}; + } + return query.value(query.record().indexOf(QStringLiteral("value"))).toByteArray(); +} + +bool ClientStatusReportingDatabase::isInitialized() const +{ + return _isInitialized; +} + +void ClientStatusReportingDatabase::setLastSentReportTimestamp(const quint64 timestamp) const +{ + QMutexLocker locker(&_mutex); + QSqlQuery query; + const auto prepareResult = query.prepare(QStringLiteral("INSERT OR REPLACE INTO keyvalue (key, value) VALUES(:key, :value);")); + query.bindValue(QStringLiteral(":key"), lastSentReportTimestamp); + query.bindValue(QStringLiteral(":value"), timestamp); + if (!prepareResult || !query.exec()) { + qCDebug(lcClientStatusReportingDatabase) << "Could not set last sent report timestamp from keyvalue table. No such record:" << lastSentReportTimestamp; + return; + } +} +QString ClientStatusReportingDatabase::dbPathForTesting; +} diff --git a/src/libsync/clientstatusreportingdatabase.h b/src/libsync/clientstatusreportingdatabase.h new file mode 100644 index 0000000000000..d3ef3c7161de2 --- /dev/null +++ b/src/libsync/clientstatusreportingdatabase.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudlib.h" +#include +#include "clientstatusreportingcommon.h" +#include "clientstatusreportingrecord.h" + +#include +#include +#include +#include +#include +#include + +namespace OCC { + +class Account; + +class OWNCLOUDSYNC_EXPORT ClientStatusReportingDatabase +{ +public: + explicit ClientStatusReportingDatabase(const Account *account); + ~ClientStatusReportingDatabase(); + + [[nodiscard]] Result setClientStatusReportingRecord(const ClientStatusReportingRecord &record) const; + [[nodiscard]] QVector getClientStatusReportingRecords() const; + [[nodiscard]] Result deleteClientStatusReportingRecords() const; + + void setLastSentReportTimestamp(const quint64 timestamp) const; + [[nodiscard]] quint64 getLastSentReportTimestamp() const; + + [[nodiscard]] Result setStatusNamesHash(const QByteArray &hash) const; + [[nodiscard]] QByteArray getStatusNamesHash() const; + + [[nodiscard]] bool isInitialized() const; + +private: + [[nodiscard]] QString makeDbPath(const Account *account) const; + [[nodiscard]] bool updateStatusNamesHash() const; + [[nodiscard]] QVector getTableColumns(const QString &table) const; + [[nodiscard]]bool addColumn(const QString &tableName, const QString &columnName, const QString &dataType, const bool withIndex = false) const; + +public: + // this must be set in unit tests on init + static QString dbPathForTesting; + +private: + QSqlDatabase _database; + + bool _isInitialized = false; + + // inspired by SyncJournalDb + mutable QRecursiveMutex _mutex; +}; +} diff --git a/src/libsync/clientstatusreportingnetwork.cpp b/src/libsync/clientstatusreportingnetwork.cpp new file mode 100644 index 0000000000000..bee15f37bb564 --- /dev/null +++ b/src/libsync/clientstatusreportingnetwork.cpp @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#include "clientstatusreportingnetwork.h" + +#include "account.h" +#include "clientstatusreportingdatabase.h" +#include "clientstatusreportingrecord.h" +#include + +namespace +{ +constexpr auto statusReportCategoryE2eErrors = "e2e_errors"; +constexpr auto statusReportCategoryProblems = "problems"; +constexpr auto statusReportCategorySyncConflicts = "sync_conflicts"; +constexpr auto statusReportCategoryVirus = "virus_detected"; +} + +namespace OCC +{ +Q_LOGGING_CATEGORY(lcClientStatusReportingNetwork, "nextcloud.sync.clientstatusreportingnetwork", QtInfoMsg) + +ClientStatusReportingNetwork::ClientStatusReportingNetwork(Account *account, const QSharedPointer database, QObject *parent) + : QObject(parent) + , _account(account) + , _database(database) +{ + init(); +} + +ClientStatusReportingNetwork::~ClientStatusReportingNetwork() +{ +} + +void ClientStatusReportingNetwork::init() +{ + Q_ASSERT(!_isInitialized); + if (_isInitialized) { + return; + } + + _clientStatusReportingSendTimer.setInterval(clientStatusReportingTrySendTimerInterval); + connect(&_clientStatusReportingSendTimer, &QTimer::timeout, this, &ClientStatusReportingNetwork::sendReportToServer); + _clientStatusReportingSendTimer.start(); + + _isInitialized = true; +} + +bool ClientStatusReportingNetwork::isInitialized() const +{ + return _isInitialized; +} + +void ClientStatusReportingNetwork::sendReportToServer() +{ + if (!_isInitialized) { + qCWarning(lcClientStatusReportingNetwork) << "Could not send report to server. Status reporting is not initialized"; + return; + } + + const auto lastSentReportTime = _database->getLastSentReportTimestamp(); + if (QDateTime::currentDateTimeUtc().toMSecsSinceEpoch() - lastSentReportTime < repordSendIntervalMs) { + return; + } + + const auto report = prepareReport(); + if (report.isEmpty()) { + return; + } + + if (!_account) { + return; + } + + const auto clientStatusReportingJob = new JsonApiJob(_account->sharedFromThis(), QStringLiteral("ocs/v2.php/apps/security_guard/diagnostics")); + clientStatusReportingJob->setBody(QJsonDocument::fromVariant(report)); + clientStatusReportingJob->setVerb(SimpleApiJob::Verb::Put); + connect(clientStatusReportingJob, &JsonApiJob::jsonReceived, [this](const QJsonDocument &json, int statusCode) { + const auto isSuccess = statusCode == HttpErrorCodeNone || statusCode == HttpErrorCodeSuccess || statusCode == HttpErrorCodeSuccessCreated + || statusCode == HttpErrorCodeSuccessNoContent; + if (isSuccess) { + const auto metaFromJson = json.object().value(QStringLiteral("ocs")).toObject().value(QStringLiteral("meta")).toObject(); + const auto codeFromJson = metaFromJson.value(QStringLiteral("statuscode")).toInt(); + if (codeFromJson == HttpErrorCodeNone || codeFromJson == HttpErrorCodeSuccess || codeFromJson == HttpErrorCodeSuccessCreated + || codeFromJson == HttpErrorCodeSuccessNoContent) { + reportToServerSentSuccessfully(); + return; + } + qCDebug(lcClientStatusReportingNetwork) << "Received error when sending client report statusCode:" << statusCode << "codeFromJson:" << codeFromJson; + } + }); + clientStatusReportingJob->start(); +} + +void ClientStatusReportingNetwork::reportToServerSentSuccessfully() +{ + qCInfo(lcClientStatusReportingNetwork) << "Report sent successfully"; + if (!_database->deleteClientStatusReportingRecords()) { + qCDebug(lcClientStatusReportingNetwork) << "Could not delete records after sending the report"; + } + _database->setLastSentReportTimestamp(QDateTime::currentDateTimeUtc().toMSecsSinceEpoch()); +} + +QVariantMap ClientStatusReportingNetwork::prepareReport() const +{ + const auto records = _database->getClientStatusReportingRecords(); + if (records.isEmpty()) { + return {}; + } + + QVariantMap report; + report[statusReportCategorySyncConflicts] = QVariantMap{}; + report[statusReportCategoryProblems] = QVariantMap{}; + report[statusReportCategoryVirus] = QVariantMap{}; + report[statusReportCategoryE2eErrors] = QVariantMap{}; + + QVariantMap e2eeErrors; + QVariantMap problems; + QVariantMap syncConflicts; + QVariantMap virusDetectedErrors; + + for (const auto &record : records) { + const auto categoryKey = classifyStatus(static_cast(record._status)); + + if (categoryKey.isEmpty()) { + qCDebug(lcClientStatusReportingNetwork) << "Could not classify status:"; + continue; + } + + if (categoryKey == statusReportCategoryE2eErrors) { + const auto initialCount = e2eeErrors[QStringLiteral("count")].toInt(); + e2eeErrors[QStringLiteral("count")] = initialCount + record._numOccurences; + e2eeErrors[QStringLiteral("oldest")] = record._lastOccurence; + report[categoryKey] = e2eeErrors; + } else if (categoryKey == statusReportCategoryProblems) { + problems[record._name] = QVariantMap{{QStringLiteral("count"), record._numOccurences}, {QStringLiteral("oldest"), record._lastOccurence}}; + report[categoryKey] = problems; + } else if (categoryKey == statusReportCategorySyncConflicts) { + const auto initialCount = syncConflicts[QStringLiteral("count")].toInt(); + syncConflicts[QStringLiteral("count")] = initialCount + record._numOccurences; + syncConflicts[QStringLiteral("oldest")] = record._lastOccurence; + report[categoryKey] = syncConflicts; + } else if (categoryKey == statusReportCategoryVirus) { + const auto initialCount = virusDetectedErrors[QStringLiteral("count")].toInt(); + virusDetectedErrors[QStringLiteral("count")] = initialCount + record._numOccurences; + virusDetectedErrors[QStringLiteral("oldest")] = record._lastOccurence; + report[categoryKey] = virusDetectedErrors; + } + } + return report; +} + +QByteArray ClientStatusReportingNetwork::classifyStatus(const ClientStatusReportingStatus status) +{ + Q_ASSERT(static_cast(status) >= 0 && static_cast(status) < static_cast(ClientStatusReportingStatus::Count)); + if (static_cast(status) < 0 || static_cast(status) >= static_cast(ClientStatusReportingStatus::Count)) { + qCDebug(lcClientStatusReportingNetwork) << "Invalid status:" << static_cast(status); + return {}; + } + + switch (status) { + case ClientStatusReportingStatus::DownloadError_Conflict: + case ClientStatusReportingStatus::DownloadError_ConflictCaseClash: + case ClientStatusReportingStatus::DownloadError_ConflictInvalidCharacters: + case ClientStatusReportingStatus::UploadError_Conflict: + case ClientStatusReportingStatus::UploadError_ConflictInvalidCharacters: + return statusReportCategorySyncConflicts; + case ClientStatusReportingStatus::DownloadError_Cannot_Create_File: + case ClientStatusReportingStatus::DownloadError_No_Free_Space: + case ClientStatusReportingStatus::DownloadError_ServerError: + case ClientStatusReportingStatus::DownloadError_Virtual_File_Hydration_Failure: + case ClientStatusReportingStatus::UploadError_No_Free_Space: + case ClientStatusReportingStatus::UploadError_No_Write_Permissions: + case ClientStatusReportingStatus::UploadError_ServerError: + return statusReportCategoryProblems; + case ClientStatusReportingStatus::UploadError_Virus_Detected: + return statusReportCategoryVirus; + case ClientStatusReportingStatus::E2EeError_GeneralError: + return statusReportCategoryE2eErrors; + case ClientStatusReportingStatus::Count: + return {}; + }; + return {}; +} + +int ClientStatusReportingNetwork::clientStatusReportingTrySendTimerInterval = 1000 * 60 * 2; // check if the time has come, every 2 minutes +quint64 ClientStatusReportingNetwork::repordSendIntervalMs = 24 * 60 * 60 * 1000; // once every 24 hours +} diff --git a/src/libsync/clientstatusreportingnetwork.h b/src/libsync/clientstatusreportingnetwork.h new file mode 100644 index 0000000000000..0b4d10c65c520 --- /dev/null +++ b/src/libsync/clientstatusreportingnetwork.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudlib.h" +#include +#include "clientstatusreportingcommon.h" + +#include +#include +#include +#include +#include +#include + +namespace OCC { + +class Account; +class ClientStatusReportingDatabase; +struct ClientStatusReportingRecord; + +class OWNCLOUDSYNC_EXPORT ClientStatusReportingNetwork : public QObject +{ + Q_OBJECT +public: + explicit ClientStatusReportingNetwork(Account *account, const QSharedPointer database, QObject *parent = nullptr); + ~ClientStatusReportingNetwork() override; + +private: + void init(); + + [[nodiscard]] QVariantMap prepareReport() const; + void reportToServerSentSuccessfully(); + +private slots: + void sendReportToServer(); + +public: + [[nodiscard]] bool isInitialized() const; + + static QByteArray classifyStatus(const ClientStatusReportingStatus status); + + static int clientStatusReportingTrySendTimerInterval; + static quint64 repordSendIntervalMs; + // this must be set in unit tests on init + static QString dbPathForTesting; + +private: + Account *_account = nullptr; + + QSharedPointer _database; + + bool _isInitialized = false; + + QTimer _clientStatusReportingSendTimer; +}; +} diff --git a/src/libsync/clientstatusreportingrecord.h b/src/libsync/clientstatusreportingrecord.h new file mode 100644 index 0000000000000..94f190a0ee210 --- /dev/null +++ b/src/libsync/clientstatusreportingrecord.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once +#include "owncloudlib.h" + +#include +#include + +namespace OCC +{ +/** + * @brief The ClientStatusReportingRecord class + * @ingroup libsync + */ + +struct OWNCLOUDSYNC_EXPORT ClientStatusReportingRecord { + QByteArray _name; + int _status = -1; + quint64 _numOccurences = 1; + quint64 _lastOccurence = 0; + + [[nodiscard]] inline bool isValid() const + { + return _status >= 0 && !_name.isEmpty() && _lastOccurence > 0; + } +}; +} diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index f11b9ab907fef..dc56ec602b85e 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1709,11 +1709,13 @@ bool ProcessDirectoryJob::checkPermissions(const OCC::SyncFileItemPtr &item) // No permissions set return true; } else if (item->isDirectory() && !perms.hasPermission(RemotePermissions::CanAddSubDirectories)) { + _discoveryData->_account->reportClientStatus(ClientStatusReportingStatus::UploadError_No_Write_Permissions); qCWarning(lcDisco) << "checkForPermission: ERROR" << item->_file; item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_errorString = tr("Not allowed because you don't have permission to add subfolders to that folder"); return false; } else if (!item->isDirectory() && !perms.hasPermission(RemotePermissions::CanAddFile)) { + _discoveryData->_account->reportClientStatus(ClientStatusReportingStatus::UploadError_No_Write_Permissions); qCWarning(lcDisco) << "checkForPermission: ERROR" << item->_file; item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_errorString = tr("Not allowed because you don't have permission to add files in that folder"); diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 4fc397a7a4129..b299bdb0aa387 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -30,6 +30,13 @@ class QDomDocument; namespace OCC { +constexpr auto HttpErrorCodeNone = 0; +constexpr auto HttpErrorCodeSuccess = 200; +constexpr auto HttpErrorCodeSuccessCreated = 201; +constexpr auto HttpErrorCodeSuccessNoContent = 204; +constexpr auto HttpErrorCodeBadRequest = 400; +constexpr auto HttpErrorCodeUnsupportedMediaType = 415; + struct HttpError { int code; // HTTP error code diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index 4745b4f846fbe..118bf86d55630 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -229,6 +229,8 @@ void PropagateItemJob::done(const SyncFileItem::Status statusArg, const QString _item->_status = statusArg; + reportClientStatuses(); + if (_item->_isRestoration) { if (_item->_status == SyncFileItem::Success || _item->_status == SyncFileItem::Conflict) { @@ -336,6 +338,42 @@ bool PropagateItemJob::hasEncryptedAncestor() const return false; } +void PropagateItemJob::reportClientStatuses() +{ + if (_item->_status == SyncFileItem::Status::Conflict) { + if (_item->_direction == SyncFileItem::Direction::Up) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::UploadError_Conflict); + } else { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_Conflict); + } + } else if (_item->_status == SyncFileItem::Status::FileNameClash) { + if (_item->_direction == SyncFileItem::Direction::Up) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::UploadError_ConflictInvalidCharacters); + } else { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_ConflictInvalidCharacters); + } + } else if (_item->_status == SyncFileItem::Status::FileNameInvalidOnServer) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::UploadError_ConflictInvalidCharacters); + } else if (_item->_status == SyncFileItem::Status::FileNameInvalid) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_ConflictInvalidCharacters); + } else if (_item->_httpErrorCode != HttpErrorCodeNone && _item->_httpErrorCode != HttpErrorCodeSuccess && _item->_httpErrorCode != HttpErrorCodeSuccessCreated + && _item->_httpErrorCode != HttpErrorCodeSuccessNoContent) { + if (_item->_direction == SyncFileItem::Up) { + const auto isCodeBadReqOrUnsupportedMediaType = (_item->_httpErrorCode == HttpErrorCodeBadRequest || _item->_httpErrorCode == HttpErrorCodeUnsupportedMediaType); + const auto isExceptionInfoPresent = !_item->_errorExceptionName.isEmpty() && !_item->_errorExceptionMessage.isEmpty(); + if (isCodeBadReqOrUnsupportedMediaType && isExceptionInfoPresent + && _item->_errorExceptionName.contains(QStringLiteral("UnsupportedMediaType")) + && _item->_errorExceptionMessage.contains(QStringLiteral("virus"), Qt::CaseInsensitive)) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::UploadError_Virus_Detected); + } else { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::UploadError_ServerError); + } + } else { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_ServerError); + } + } +} + // ================================================================================ PropagateItemJob *OwncloudPropagator::createJob(const SyncFileItemPtr &item) @@ -917,6 +955,7 @@ bool OwncloudPropagator::createConflict(const SyncFileItemPtr &item, } _journal->setConflictRecord(conflictRecord); + account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_Conflict); // Create a new upload job if the new conflict file should be uploaded if (account()->capabilities().uploadConflictFiles()) { @@ -989,6 +1028,7 @@ OCC::Optional OwncloudPropagator::createCaseClashConflict(const SyncFil } _journal->setCaseConflictRecord(conflictRecord); + account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_ConflictCaseClash); // Need a new sync to detect the created copy of the conflicting file _anotherSyncNeeded = true; diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index 440b98551e51f..06e840fb106fb 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -194,6 +194,8 @@ protected slots: void slotRestoreJobFinished(SyncFileItem::Status status); private: + void reportClientStatuses(); + QScopedPointer _restoreJob; JobParallelism _parallelism = FullParallelism; diff --git a/src/libsync/owncloudpropagator_p.h b/src/libsync/owncloudpropagator_p.h index 6337b2f3cfb9c..beb150648db87 100644 --- a/src/libsync/owncloudpropagator_p.h +++ b/src/libsync/owncloudpropagator_p.h @@ -47,7 +47,6 @@ inline bool fileIsStillChanging(const OCC::SyncFileItem &item) } namespace OCC { - inline QByteArray getEtagFromReply(QNetworkReply *reply) { QByteArray ocEtag = parseEtag(reply->rawHeader("OC-ETag")); @@ -62,6 +61,52 @@ inline QByteArray getEtagFromReply(QNetworkReply *reply) return ret; } +inline QPair getExceptionFromReply(QNetworkReply * const reply) +{ + Q_ASSERT(reply); + if (!reply) { + qDebug() << "Could not parse null reply"; + return {}; + } + const auto httpStatusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + // only for BadRequest and UnsupportedMediaType + if (httpStatusCode != 400 && httpStatusCode != 415) { + return {}; + } + + // we must not modify the buffer + const auto replyBody = reply->peek(reply->bytesAvailable()); + + // parse exception name + auto exceptionNameStartIndex = replyBody.indexOf(QByteArrayLiteral("")); + if (exceptionNameStartIndex == -1) { + qDebug() << "Could not parse exception. No tag."; + return {}; + } + exceptionNameStartIndex += QByteArrayLiteral("").size(); + const auto exceptionNameEndIndex = replyBody.indexOf('<', exceptionNameStartIndex); + if (exceptionNameEndIndex == -1) { + return {}; + } + const auto exceptionName = replyBody.mid(exceptionNameStartIndex, exceptionNameEndIndex - exceptionNameStartIndex); + + // parse exception message + auto exceptionMessageStartIndex = replyBody.indexOf(QByteArrayLiteral(""), exceptionNameEndIndex); + if (exceptionMessageStartIndex == -1) { + qDebug() << "Could not parse exception. No tag."; + return {exceptionName, {}}; + } + exceptionMessageStartIndex += QByteArrayLiteral("").size(); + const auto exceptionMessageEndIndex = replyBody.indexOf('<', exceptionMessageStartIndex); + if (exceptionMessageEndIndex == -1) { + return {exceptionName, {}}; + } + + const auto exceptionMessage = replyBody.mid(exceptionMessageStartIndex, exceptionMessageEndIndex - exceptionMessageStartIndex); + + return {exceptionName, exceptionMessage}; +} + /** * Given an error from the network, map to a SyncFileItem::Status error */ diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 6d7d5e2ebee0b..afdf33d491b42 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -675,6 +675,7 @@ void PropagateDownloadFile::startDownload() if (_tmpFile.exists()) FileSystem::setFileReadOnly(_tmpFile.fileName(), false); if (!_tmpFile.open(QIODevice::Append | QIODevice::Unbuffered)) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_Cannot_Create_File); qCWarning(lcPropagateDownload) << "could not open temporary file" << _tmpFile.fileName(); done(SyncFileItem::NormalError, _tmpFile.errorString(), ErrorCategory::GenericError); return; @@ -1259,6 +1260,7 @@ void PropagateDownloadFile::downloadFinished() emit propagator()->touchedFile(filename); // The fileChanged() check is done above to generate better error messages. if (!FileSystem::uncheckedRenameReplace(_tmpFile.fileName(), filename, &error)) { + propagator()->account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_Cannot_Create_File); qCWarning(lcPropagateDownload) << QString("Rename failed: %1 => %2").arg(_tmpFile.fileName()).arg(filename); // If the file is locked, we want to retry this sync when it // becomes available again, otherwise try again directly diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index f74727ec7bb80..78b04205cc361 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -112,6 +112,9 @@ bool PollJob::finished() _item->_requestId = requestId(); _item->_status = classifyError(err, _item->_httpErrorCode); _item->_errorString = errorString(); + const auto exceptionParsed = getExceptionFromReply(reply()); + _item->_errorExceptionName = exceptionParsed.first; + _item->_errorExceptionMessage = exceptionParsed.second; if (_item->_status == SyncFileItem::FatalError || _item->_httpErrorCode >= 400) { if (_item->_status != SyncFileItem::FatalError diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index 8be463ecd21c9..f751c58253fbe 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -411,6 +411,9 @@ void PropagateUploadFileNG::slotPutFinished() _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); _item->_requestId = job->requestId(); commonErrorHandling(job); + const auto exceptionParsed = getExceptionFromReply(job->reply()); + _item->_errorExceptionName = exceptionParsed.first; + _item->_errorExceptionMessage = exceptionParsed.second; return; } @@ -496,6 +499,9 @@ void PropagateUploadFileNG::slotMoveJobFinished() if (err != QNetworkReply::NoError) { commonErrorHandling(job); + const auto exceptionParsed = getExceptionFromReply(job->reply()); + _item->_errorExceptionName = exceptionParsed.first; + _item->_errorExceptionMessage = exceptionParsed.second; return; } diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp index 33cd8892d7335..a32e796a597de 100644 --- a/src/libsync/propagateuploadv1.cpp +++ b/src/libsync/propagateuploadv1.cpp @@ -218,6 +218,9 @@ void PropagateUploadFileV1::slotPutFinished() QNetworkReply::NetworkError err = job->reply()->error(); if (err != QNetworkReply::NoError) { commonErrorHandling(job); + const auto exceptionParsed = getExceptionFromReply(job->reply()); + _item->_errorExceptionName = exceptionParsed.first; + _item->_errorExceptionMessage = exceptionParsed.second; return; } diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index e4e76a7ff79b7..523f3506a193b 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -314,6 +314,7 @@ void SyncEngine::conflictRecordMaintenance() } _journal->setConflictRecord(record); + account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_Conflict); } } } @@ -1261,6 +1262,7 @@ void SyncEngine::slotSummaryError(const QString &message) void SyncEngine::slotInsufficientLocalStorage() { + account()->reportClientStatus(ClientStatusReportingStatus::DownloadError_No_Free_Space); slotSummaryError( tr("Disk space is low: Downloads that would reduce free space " "below %1 were skipped.") @@ -1269,6 +1271,7 @@ void SyncEngine::slotInsufficientLocalStorage() void SyncEngine::slotInsufficientRemoteStorage() { + account()->reportClientStatus(ClientStatusReportingStatus::UploadError_No_Free_Space); auto msg = tr("There is insufficient space available on the server for some uploads."); if (_uniqueErrors.contains(msg)) return; diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 399546dcd0459..c0d880be6097f 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -287,6 +287,8 @@ class OWNCLOUDSYNC_EXPORT SyncFileItem quint16 _httpErrorCode = 0; RemotePermissions _remotePerm; QString _errorString; // Contains a string only in case of error + QString _errorExceptionName; // Contains a server exception string only in case of error + QString _errorExceptionMessage; // Contains a server exception message string only in case of error QByteArray _responseTimeStamp; QByteArray _requestId; // X-Request-Id of the failed request quint32 _affectedItems = 1; // the number of affected items by the operation on this item. diff --git a/src/libsync/vfs/cfapi/vfs_cfapi.cpp b/src/libsync/vfs/cfapi/vfs_cfapi.cpp index e7905fe6ca7c0..935cd8605fe4f 100644 --- a/src/libsync/vfs/cfapi/vfs_cfapi.cpp +++ b/src/libsync/vfs/cfapi/vfs_cfapi.cpp @@ -464,6 +464,7 @@ void VfsCfApi::onHydrationJobFinished(HydrationJob *job) qCInfo(lcCfApi) << "Hydration job finished" << job->requestId() << job->folderPath() << job->status(); emit hydrationRequestFinished(job->requestId()); if (!job->errorString().isEmpty()) { + params().account->reportClientStatus(ClientStatusReportingStatus::DownloadError_Virtual_File_Hydration_Failure); emit failureHydrating(job->errorCode(), job->statusCode(), job->errorString(), job->folderPath()); } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 829b1d36c0188..44396f29a6caa 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -75,6 +75,7 @@ nextcloud_add_test(SecureFileDrop) nextcloud_add_test(FileTagModel) nextcloud_add_test(SyncConflictsModel) nextcloud_add_test(DateFieldBackend) +nextcloud_add_test(ClientStatusReporting) target_link_libraries(SecureFileDropTest PRIVATE Nextcloud::sync) configure_file(fake2eelocksucceeded.json "${PROJECT_BINARY_DIR}/bin/fake2eelocksucceeded.json" COPYONLY) diff --git a/test/testclientstatusreporting.cpp b/test/testclientstatusreporting.cpp new file mode 100644 index 0000000000000..b018f20336666 --- /dev/null +++ b/test/testclientstatusreporting.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (C) by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#include "account.h" +#include "accountstate.h" +#include "clientstatusreportingcommon.h" +#include "clientstatusreportingdatabase.h" +#include "clientstatusreportingnetwork.h" +#include "syncenginetestutils.h" + +#include +#include + +namespace { +static QByteArray fake200Response = R"({"ocs":{"meta":{"status":"success","statuscode":200},"data":[]}})"; +} + +class TestClientStatusReporting : public QObject +{ + Q_OBJECT + +public: + TestClientStatusReporting() = default; + + QScopedPointer fakeQnam; + OCC::Account *account; + QScopedPointer accountState; + QString dbFilePath; + QVariantMap bodyReceivedAndParsed; + +private slots: + void initTestCase() + { + OCC::ClientStatusReportingNetwork::clientStatusReportingTrySendTimerInterval = 1000; + OCC::ClientStatusReportingNetwork::repordSendIntervalMs = 2000; + + fakeQnam.reset(new FakeQNAM({})); + account = OCC::Account::create().get(); + account->setCredentials(new FakeCredentials{fakeQnam.data()}); + account->setUrl(QUrl(("http://example.de"))); + + accountState.reset(new OCC::AccountState(account->sharedFromThis())); + + const auto databaseId = QStringLiteral("%1@%2").arg(account->davUser(), account->url().toString()); + const auto databaseIdHash = QCryptographicHash::hash(databaseId.toUtf8(), QCryptographicHash::Md5); + dbFilePath = QDir::tempPath() + QStringLiteral("/.tests_userdata_%1.db").arg(QString::fromLatin1(databaseIdHash.left(6).toHex())); + QFile(dbFilePath).remove(); + OCC::ClientStatusReportingDatabase::dbPathForTesting = dbFilePath; + + QVariantMap capabilities; + capabilities[QStringLiteral("security_guard")] = QVariantMap{ + { QStringLiteral("diagnostics"), true } + }; + account->setCapabilities(capabilities); + + fakeQnam->setOverride([this](QNetworkAccessManager::Operation op, const QNetworkRequest &req, QIODevice *device) { + QNetworkReply *reply = nullptr; + const auto reqBody = device->readAll(); + bodyReceivedAndParsed = QJsonDocument::fromJson(reqBody).toVariant().toMap(); + reply = new FakePayloadReply(op, req, fake200Response, fakeQnam.data()); + return reply; + }); + } + + void testReportAndSendStatuses() + { + for (int i = 0; i < 2; ++i) { + // 5 conflicts + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_Conflict); + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_ConflictInvalidCharacters); + account->reportClientStatus(OCC::ClientStatusReportingStatus::DownloadError_Conflict); + account->reportClientStatus(OCC::ClientStatusReportingStatus::DownloadError_ConflictInvalidCharacters); + account->reportClientStatus(OCC::ClientStatusReportingStatus::DownloadError_ConflictCaseClash); + + // 4 problems + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_ServerError); + account->reportClientStatus(OCC::ClientStatusReportingStatus::DownloadError_ServerError); + account->reportClientStatus(OCC::ClientStatusReportingStatus::DownloadError_Virtual_File_Hydration_Failure); + // 3 occurances of case ClientStatusReportingStatus::UploadError_No_Write_Permissions + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_No_Write_Permissions); + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_No_Write_Permissions); + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_No_Write_Permissions); + + // 3 occurances of case ClientStatusReportingStatus::UploadError_Virus_Detected + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_Virus_Detected); + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_Virus_Detected); + account->reportClientStatus(OCC::ClientStatusReportingStatus::UploadError_Virus_Detected); + + // 2 occurances of E2EeError_GeneralError + account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); + account->reportClientStatus(OCC::ClientStatusReportingStatus::E2EeError_GeneralError); + QTest::qWait(OCC::ClientStatusReportingNetwork::clientStatusReportingTrySendTimerInterval + OCC::ClientStatusReportingNetwork::repordSendIntervalMs); + + QVERIFY(!bodyReceivedAndParsed.isEmpty()); + + // we must have 2 e2ee errors + const auto virusDetectedErrorsReceived = bodyReceivedAndParsed.value("virus_detected").toMap(); + QVERIFY(!virusDetectedErrorsReceived.isEmpty()); + QCOMPARE(virusDetectedErrorsReceived.value("count"), 3); + + // we must have 2 e2ee errors + const auto e2eeErrorsReceived = bodyReceivedAndParsed.value("e2e_errors").toMap(); + QVERIFY(!e2eeErrorsReceived.isEmpty()); + QCOMPARE(e2eeErrorsReceived.value("count"), 2); + + // we must have 5 conflicts + const auto conflictsReceived = bodyReceivedAndParsed.value("sync_conflicts").toMap(); + QVERIFY(!conflictsReceived.isEmpty()); + QCOMPARE(conflictsReceived.value("count"), 5); + + // we must have 4 problems + const auto problemsReceived = bodyReceivedAndParsed.value("problems").toMap(); + QVERIFY(!problemsReceived.isEmpty()); + QCOMPARE(problemsReceived.size(), 4); + const auto problemsNoWritePermissions = problemsReceived.value(OCC::clientStatusstatusStringFromNumber(OCC::ClientStatusReportingStatus::UploadError_No_Write_Permissions)).toMap(); + // among those, 3 occurances of case ClientStatusReportingStatus::UploadError_No_Write_Permissions + QCOMPARE(problemsNoWritePermissions.value("count"), 3); + + bodyReceivedAndParsed.clear(); + } + } + + void testNothingReportedAndNothingSent() + { + QTest::qWait(OCC::ClientStatusReportingNetwork::clientStatusReportingTrySendTimerInterval + OCC::ClientStatusReportingNetwork::repordSendIntervalMs); + QVERIFY(bodyReceivedAndParsed.isEmpty()); + } + + void cleanupTestCase() + { + accountState.reset(nullptr); + delete account; + QFile(dbFilePath).remove(); + } +}; + +QTEST_MAIN(TestClientStatusReporting) +#include "testclientstatusreporting.moc" diff --git a/test/testnextcloudpropagator.cpp b/test/testnextcloudpropagator.cpp index 954c665f5f91c..812c5b43c7008 100644 --- a/test/testnextcloudpropagator.cpp +++ b/test/testnextcloudpropagator.cpp @@ -9,6 +9,7 @@ #include "propagatedownload.h" #include "owncloudpropagator_p.h" +#include "syncenginetestutils.h" using namespace OCC; namespace OCC { @@ -76,6 +77,28 @@ private slots: QCOMPARE(parseEtag(test.first), QByteArray(test.second)); } } + + void testParseException() + { + QNetworkRequest request; + request.setUrl(QStringLiteral("http://cloud.example.de/")); + const auto body = QByteArrayLiteral( + "\n" + "\n" + "Sabre\\Exception\\UnsupportedMediaType\n" + "Virus detected!\n" + ""); + const auto reply = new FakeErrorReply(QNetworkAccessManager::PutOperation, + request, + this, + 415, body); + const auto exceptionParsed = OCC::getExceptionFromReply(reply); + // verify parsing succeeded + QVERIFY(!exceptionParsed.first.isEmpty()); + QVERIFY(!exceptionParsed.second.isEmpty()); + // verify buffer is not changed + QCOMPARE(reply->readAll().size(), body.size()); + } }; QTEST_APPLESS_MAIN(TestNextcloudPropagator)