diff --git a/src/config.d b/src/config.d index 2c73c7f14..8bb8b706c 100644 --- a/src/config.d +++ b/src/config.d @@ -76,6 +76,8 @@ final class Config stringValues["application_id"] = ""; // allow for resync to be set via config file boolValues["resync"] = false; + // Treat new 'remote' files as 'on-demand' - that is, do not download them but save that it exists + boolValues["on_demand"] = false; // Determine the users home directory. // Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts diff --git a/src/itemdb.d b/src/itemdb.d index a8e93807f..44f412766 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -13,25 +13,27 @@ enum ItemType { } struct Item { - string driveId; - string id; - string name; - ItemType type; - string eTag; - string cTag; - SysTime mtime; - string parentId; - string crc32Hash; - string sha1Hash; - string quickXorHash; - string remoteDriveId; - string remoteId; + string driveId; // #1 + string id; // #2 + string name; // #3 + ItemType type; // #4 + string eTag; // #5 + string cTag; // #6 + SysTime mtime; // #7 + string parentId; // #8 + string crc32Hash; // #9 + string sha1Hash; // #10 + string quickXorHash; // #11 + string remoteDriveId; // #12 + string remoteId; // #13 + string webUrl; // #14 + string fileSize; // #15 } final class ItemDatabase { // increment this for every change in the db schema - immutable int itemDatabaseVersion = 9; + immutable int itemDatabaseVersion = 10; Database db; string insertItemStmt; @@ -83,12 +85,12 @@ final class ItemDatabase db.exec("PRAGMA auto_vacuum = FULL"); insertItemStmt = " - INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) + INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, webUrl, fileSize) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) "; updateItemStmt = " UPDATE item - SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13 + SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, webUrl = ?14, fileSize = ?15 WHERE driveId = ?1 AND id = ?2 "; selectItemByIdStmt = " @@ -117,6 +119,8 @@ final class ItemDatabase remoteDriveId TEXT, remoteId TEXT, deltaLink TEXT, + webUrl TEXT, + fileSize TEXT, PRIMARY KEY (driveId, id), FOREIGN KEY (driveId, parentId) REFERENCES item (driveId, id) @@ -300,13 +304,15 @@ final class ItemDatabase bind(11, quickXorHash); bind(12, remoteDriveId); bind(13, remoteId); + bind(14, webUrl); + bind(15, fileSize); } } private Item buildItem(Statement.Result result) { assert(!result.empty, "The result must not be empty"); - assert(result.front.length == 14, "The result must have 14 columns"); + assert(result.front.length == 16, "The result must have 16 columns"); Item item = { driveId: result.front[0].dup, id: result.front[1].dup, @@ -319,7 +325,9 @@ final class ItemDatabase sha1Hash: result.front[9].dup, quickXorHash: result.front[10].dup, remoteDriveId: result.front[11].dup, - remoteId: result.front[12].dup + remoteId: result.front[12].dup, + webUrl: result.front[14].dup, + fileSize: result.front[15].dup }; switch (result.front[3]) { case "file": item.type = ItemType.file; break; diff --git a/src/main.d b/src/main.d index db2be57b5..1054ef1de 100644 --- a/src/main.d +++ b/src/main.d @@ -271,7 +271,7 @@ int main(string[] args) if (cfg.getValueBool("dry_run")) { log.log("DRY-RUN Configured. Output below shows what 'would' have occurred."); } - + // Are we able to reach the OneDrive Service bool online = false; @@ -607,6 +607,12 @@ int main(string[] args) // Do we configure to disable the upload validation routine if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation(); + // Do we configure on-demand file access mode .. only and ONLY if --monitor is also in use + if (cfg.getValueBool("monitor") && cfg.getValueBool("on_demand")) { + log.log("IMPORTANT: On-Demand access of files has been enabled."); + sync.setOnDemandAccessEnabled(); + } + // Do we need to validate the syncDir to check for the presence of a '.nosync' file if (cfg.getValueBool("check_nomount")) { // we were asked to check the mounts @@ -687,6 +693,33 @@ int main(string[] args) log.logAndNotify("Initializing monitor ..."); log.log("OneDrive monitor interval (seconds): ", cfg.getValueLong("monitor_interval")); Monitor m = new Monitor(selectiveSync); + + m.onFileOpened = delegate(string path) { + if (sync.getOnDemandAccessEnabled){ + // On-Demand access is enabled + if (sync.validateOnDemandFile(path, syncDir)) { + log.log("[M] On-Demand File requested: ", path); + // Temp remove watch + m.removeFileWatch(path); + // Download replacement file + sync.downloadOnDemandFile(path, syncDir); + // Add watch + m.addFileWatch(path); + } + } + }; + + m.onFileChanged = delegate(string path) { + log.vlog("[M] File changed: ", path); + try { + sync.scanForDifferences(path); + } catch (CurlException e) { + log.vlog("Offline, cannot upload changed item!"); + } catch(Exception e) { + log.logAndNotify("Cannot upload file changes/creation: ", e.msg); + } + }; + m.onDirCreated = delegate(string path) { // Handle .folder creation if skip_dotfiles is enabled if ((cfg.getValueBool("skip_dotfiles")) && (selectiveSync.isDotFile(path))) { @@ -702,16 +735,6 @@ int main(string[] args) } } }; - m.onFileChanged = delegate(string path) { - log.vlog("[M] File changed: ", path); - try { - sync.scanForDifferences(path); - } catch (CurlException e) { - log.vlog("Offline, cannot upload changed item!"); - } catch(Exception e) { - log.logAndNotify("Cannot upload file changes/creation: ", e.msg); - } - }; m.onDelete = delegate(string path) { log.vlog("[M] Item deleted: ", path); try { diff --git a/src/monitor.d b/src/monitor.d index c3cdc5b9c..e955f44a0 100644 --- a/src/monitor.d +++ b/src/monitor.d @@ -2,6 +2,7 @@ import core.sys.linux.sys.inotify; import core.stdc.errno; import core.sys.posix.poll, core.sys.posix.unistd; import std.exception, std.file, std.path, std.regex, std.stdio, std.string, std.algorithm.mutation; +import std.conv; import config; import selective; import util; @@ -9,7 +10,7 @@ static import log; // relevant inotify events private immutable uint32_t mask = IN_CLOSE_WRITE | IN_CREATE | IN_DELETE | - IN_MOVE | IN_IGNORED | IN_Q_OVERFLOW; + IN_MOVE | IN_IGNORED | IN_Q_OVERFLOW | IN_OPEN; class MonitorException: ErrnoException { @@ -38,6 +39,7 @@ final class Monitor private SelectiveSync selectiveSync; void delegate(string path) onDirCreated; + void delegate(string path) onFileOpened; void delegate(string path) onFileChanged; void delegate(string path) onDelete; void delegate(string from, string to) onMove; @@ -54,7 +56,7 @@ final class Monitor this.skip_symlinks = skip_symlinks; this.check_nosync = check_nosync; - assert(onDirCreated && onFileChanged && onDelete && onMove); + assert(onDirCreated && onFileChanged && onDelete && onMove && onFileOpened); fd = inotify_init(); if (fd < 0) throw new MonitorException("inotify_init failed"); if (!buffer) buffer = new void[4096]; @@ -199,10 +201,57 @@ final class Monitor int ret = inotify_rm_watch(fd, wd); if (ret < 0) throw new MonitorException("inotify_rm_watch failed"); wdToDirName.remove(wd); - log.vlog("Monitored directory removed: ", dirname); + log.log("Monitored directory removed: ", dirname); } } } + + // remove a specific path from being watched + void removeFileWatch(const(char)[] path) + { + string parentPath = to!string(dirName(path)); + foreach (wd, dirname; wdToDirName) { + if (dirname.startsWith(parentPath)) { + int ret = inotify_rm_watch(fd, wd); + if (ret < 0) throw new MonitorException("inotify_rm_watch failed"); + wdToDirName.remove(wd); + log.vdebug("Monitor removed for: ", path); + + } + } + } + + // add a specific path to being watched + void addFileWatch(const(char)[] path) + { + string parentPath = to!string(dirName(path)); + int wd = inotify_add_watch(fd, toStringz(parentPath), mask); + if (wd < 0) { + if (errno() == ENOSPC) { + log.log("The user limit on the total number of inotify watches has been reached."); + log.log("To see the current max number of watches run:"); + log.log("sysctl fs.inotify.max_user_watches"); + log.log("To change the current max number of watches to 524288 run:"); + log.log("sudo sysctl fs.inotify.max_user_watches=524288"); + } + if (errno() == 13) { + if ((selectiveSync.getSkipDotfiles()) && (selectiveSync.isDotFile(parentPath))) { + // no misleading output that we could not add a watch due to permission denied + return; + } else { + log.vlog("WARNING: inotify_add_watch failed - permission denied: ", parentPath); + return; + } + } + // Flag any other errors + log.error("ERROR: inotify_add_watch failed: ", parentPath); + return; + } + // Add path to inotify watch - required regardless if a '.folder' or 'folder' + wdToDirName[wd] = buildNormalizedPath(parentPath) ~ "/"; + // log that this is agail being monitored + log.vdebug("Monitor added for: ", path); + } // return the file path from an inotify event private string getPath(const(inotify_event)* event) @@ -284,6 +333,15 @@ final class Monitor } else if ((event.mask & IN_CLOSE_WRITE) && !(event.mask & IN_ISDIR)) { log.vdebug("event IN_CLOSE_WRITE and ...: ", path); if (useCallbacks) onFileChanged(path); + } else if (event.mask & IN_OPEN) { + if ((event.mask & IN_OPEN) && !(event.mask & IN_ISDIR)) { + // open event for a file + log.vdebug("event IN_OPEN detected for: ", path); + if (useCallbacks) onFileOpened(path); + } else { + // open event for a directory + goto skip; + } } else { log.vdebug("event unhandled: ", path); assert(0); diff --git a/src/onedrive.d b/src/onedrive.d index 3da3d4c61..74a165ec0 100644 --- a/src/onedrive.d +++ b/src/onedrive.d @@ -283,7 +283,7 @@ final class OneDriveApi const(char)[] url = deltaLink; if (url == null) { url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl"; } return get(url); } @@ -357,7 +357,7 @@ final class OneDriveApi // string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/"; if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/"; else url = itemByPathUrl ~ encodeComponent(path) ~ ":/"; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl"; return get(url); } @@ -369,7 +369,7 @@ final class OneDriveApi const(char)[] url; // string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/"; url = driveByIdUrl ~ driveId ~ "/items/" ~ id; - url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size"; + url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl"; return get(url); } diff --git a/src/sync.d b/src/sync.d index ef77894ba..dff158eeb 100644 --- a/src/sync.d +++ b/src/sync.d @@ -137,15 +137,22 @@ private Item makeItem(const ref JSONValue driveItem) item.mtime = SysTime.fromISOExtString(driveItem["fileSystemInfo"]["lastModifiedDateTime"].str); } } - + + // Is the driveItem a file? if (isItemFile(driveItem)) { item.type = ItemType.file; - } else if (isItemFolder(driveItem)) { + if ("webUrl" in driveItem) item.webUrl = driveItem["webUrl"].str; + if ("size" in driveItem) item.fileSize = to!string(driveItem["size"].integer); + } + + // Is the driveItem a folder? + if (isItemFolder(driveItem)) { item.type = ItemType.dir; - } else if (isItemRemote(driveItem)) { + } + + // Is the driveItem a remoteItem (Shared Folder)? + if (isItemRemote(driveItem)) { item.type = ItemType.remote; - } else { - // do not throw exception, item will be removed in applyDifferences() } // root and remote items do not have parentReference @@ -168,12 +175,14 @@ private Item makeItem(const ref JSONValue driveItem) log.vlog("The file does not have any hash"); } } - + + // If the item is a remote item, set the remote driveID and remoteID properties if (isItemRemote(driveItem)) { item.remoteDriveId = driveItem["remoteItem"]["parentReference"]["driveId"].str; item.remoteId = driveItem["remoteItem"]["id"].str; } - + + // return the configured item return item; } @@ -240,6 +249,8 @@ final class SyncEngine private bool syncListConfigured = false; // sync_list new folder added, trigger delta scan override private bool oneDriveFullScanTrigger = false; + // Has on-demand access of files been enabled? + private bool onDemandAccessEnabled = false; this(Config cfg, OneDriveApi onedrive, ItemDatabase itemdb, SelectiveSync selectiveSync) { @@ -484,6 +495,19 @@ final class SyncEngine log.vdebug("Setting syncListConfigured = true"); } + // set onDemandAccessEnabled to true + void setOnDemandAccessEnabled() + { + onDemandAccessEnabled = true; + log.vdebug("Setting onDemandAccessEnabled = true"); + } + + // get OnDemandAccessEnabled value + bool getOnDemandAccessEnabled() + { + return onDemandAccessEnabled; + } + // download all new changes from OneDrive void applyDifferences(bool performFullItemScan) { @@ -1878,6 +1902,7 @@ final class SyncEngine write("Downloading file ", path, " ... "); JSONValue fileDetails; + // Get the file details from OneDrive try { fileDetails = onedrive.getFileDetails(item.driveId, item.id); } catch (OneDriveException e) { @@ -1941,122 +1966,138 @@ final class SyncEngine log.vdebug("WARNING: fileDetails['file']['hashes'] is missing - unable to compare file hash after download"); } - try { - onedrive.downloadById(item.driveId, item.id, path, fileSize); - } catch (OneDriveException e) { - log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); - // 408 = Request Time Out - // 429 = Too Many Requests - need to delay - - if (e.httpStatusCode == 408) { - // 408 error handling - request time out - // https://github.com/abraunegg/onedrive/issues/694 - // Back off & retry with incremental delay - int retryCount = 10; - int retryAttempts = 1; - int backoffInterval = 2; - while (retryAttempts < retryCount){ - // retry in 2,4,8,16,32,64,128,256,512,1024 seconds - Thread.sleep(dur!"seconds"(retryAttempts*backoffInterval)); - try { - onedrive.downloadById(item.driveId, item.id, path, fileSize); - // successful download - retryAttempts = retryCount; - } catch (OneDriveException e) { - log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); - if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { - // If another 408 .. - if (e.httpStatusCode == 408) { - // Increment & loop around - log.vdebug("HTTP 408 generated - incrementing retryAttempts"); - retryAttempts++; - } - // If a 429 .. - if (e.httpStatusCode == 429) { - // Increment & loop around - handleOneDriveThrottleRequest(); - log.vdebug("HTTP 429 generated - incrementing retryAttempts"); - retryAttempts++; + if (!onDemandAccessEnabled) { + // Attempt to download the file from OneDrive + try { + onedrive.downloadById(item.driveId, item.id, path, fileSize); + } catch (OneDriveException e) { + log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); + // 408 = Request Time Out + // 429 = Too Many Requests - need to delay + if (e.httpStatusCode == 408) { + // 408 error handling - request time out + // https://github.com/abraunegg/onedrive/issues/694 + // Back off & retry with incremental delay + int retryCount = 10; + int retryAttempts = 1; + int backoffInterval = 2; + while (retryAttempts < retryCount){ + // retry in 2,4,8,16,32,64,128,256,512,1024 seconds + Thread.sleep(dur!"seconds"(retryAttempts*backoffInterval)); + try { + onedrive.downloadById(item.driveId, item.id, path, fileSize); + // successful download + retryAttempts = retryCount; + } catch (OneDriveException e) { + log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { + // If another 408 .. + if (e.httpStatusCode == 408) { + // Increment & loop around + log.vdebug("HTTP 408 generated - incrementing retryAttempts"); + retryAttempts++; + } + // If a 429 .. + if (e.httpStatusCode == 429) { + // Increment & loop around + handleOneDriveThrottleRequest(); + log.vdebug("HTTP 429 generated - incrementing retryAttempts"); + retryAttempts++; + } + } else { + displayOneDriveErrorMessage(e.msg); } - } else { - displayOneDriveErrorMessage(e.msg); } } } - } - - if (e.httpStatusCode == 429) { - // HTTP request returned status code 429 (Too Many Requests) - // https://github.com/abraunegg/onedrive/issues/133 - int retryCount = 10; - int retryAttempts = 1; - while (retryAttempts < retryCount){ - // retry after waiting the timeout value from the 429 HTTP response header Retry-After - handleOneDriveThrottleRequest(); - try { - onedrive.downloadById(item.driveId, item.id, path, fileSize); - // successful download - retryAttempts = retryCount; - } catch (OneDriveException e) { - log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); - if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { - // If another 408 .. - if (e.httpStatusCode == 408) { - // Increment & loop around - log.vdebug("HTTP 408 generated - incrementing retryAttempts"); - retryAttempts++; - } - // If a 429 .. - if (e.httpStatusCode == 429) { - // Increment & loop around - handleOneDriveThrottleRequest(); - log.vdebug("HTTP 429 generated - incrementing retryAttempts"); - retryAttempts++; + + if (e.httpStatusCode == 429) { + // HTTP request returned status code 429 (Too Many Requests) + // https://github.com/abraunegg/onedrive/issues/133 + int retryCount = 10; + int retryAttempts = 1; + while (retryAttempts < retryCount){ + // retry after waiting the timeout value from the 429 HTTP response header Retry-After + handleOneDriveThrottleRequest(); + try { + onedrive.downloadById(item.driveId, item.id, path, fileSize); + // successful download + retryAttempts = retryCount; + } catch (OneDriveException e) { + log.vdebug("onedrive.downloadById(item.driveId, item.id, path, fileSize); generated a OneDriveException"); + if ((e.httpStatusCode == 429) || (e.httpStatusCode == 408)) { + // If another 408 .. + if (e.httpStatusCode == 408) { + // Increment & loop around + log.vdebug("HTTP 408 generated - incrementing retryAttempts"); + retryAttempts++; + } + // If a 429 .. + if (e.httpStatusCode == 429) { + // Increment & loop around + handleOneDriveThrottleRequest(); + log.vdebug("HTTP 429 generated - incrementing retryAttempts"); + retryAttempts++; + } + } else { + displayOneDriveErrorMessage(e.msg); } - } else { - displayOneDriveErrorMessage(e.msg); } } } + } catch (std.exception.ErrnoException e) { + // There was a file system error + // display the error message + displayFileSystemErrorMessage(e.msg); + downloadFailed = true; + return; } - } catch (std.exception.ErrnoException e) { - // There was a file system error - // display the error message - displayFileSystemErrorMessage(e.msg); - downloadFailed = true; - return; + } else { + // On-Demand file access has been configured & in use + std.file.write(path, "OneDrive On-Demand File"); } + // file has to have downloaded in order to set the times / data for the file if (exists(path)) { - // A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on - // does the file hash OneDrive reports match what we have locally? - string quickXorHash = computeQuickXorHash(path); - string sha1Hash = computeSha1Hash(path); - - if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) { - // downloaded matches either size or hash - log.vdebug("Downloaded file matches reported size and or reported file hash"); + if (!onDemandAccessEnabled) { + // A 'file' was downloaded - does what we downloaded = reported fileSize or if there is some sort of funky local disk compression going on + // does the file hash OneDrive reports match what we have locally? + string quickXorHash = computeQuickXorHash(path); + string sha1Hash = computeSha1Hash(path); + + if ((getSize(path) == fileSize) || (OneDriveFileHash == quickXorHash) || (OneDriveFileHash == sha1Hash)) { + // downloaded matches either size or hash + log.vdebug("Downloaded file matches reported size and or reported file hash"); + try { + setTimes(path, item.mtime, item.mtime); + } catch (FileException e) { + // display the error message + displayFileSystemErrorMessage(e.msg); + } + } else { + // size error? + if (getSize(path) != fileSize) { + // downloaded file size does not match + log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); + } + // hash error? + if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) { + // downloaded file hash does not match + log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why."); + } + // we do not want this local file to remain on the local file system + safeRemove(path); + downloadFailed = true; + return; + } + } else { + // file validation will never be successful when using on-demand files try { setTimes(path, item.mtime, item.mtime); } catch (FileException e) { // display the error message displayFileSystemErrorMessage(e.msg); } - } else { - // size error? - if (getSize(path) != fileSize) { - // downloaded file size does not match - log.error("ERROR: File download size mis-match. Increase logging verbosity to determine why."); - } - // hash error? - if ((OneDriveFileHash != quickXorHash) || (OneDriveFileHash != sha1Hash)) { - // downloaded file hash does not match - log.error("ERROR: File download hash mis-match. Increase logging verbosity to determine why."); - } - // we do not want this local file to remain on the local file system - safeRemove(path); - downloadFailed = true; - return; } } else { log.error("ERROR: File failed to download. Increase logging verbosity to determine why."); @@ -4215,22 +4256,66 @@ final class SyncEngine // File exists locally, does it exist in the database // Path needs to be relative to sync_dir path string relativePath = relativePath(localFilePath, syncDir); + string webUrl; Item item; if (itemdb.selectByPath(relativePath, defaultDriveId, item)) { // File is in the local database cache - JSONValue fileDetails; - - try { - fileDetails = onedrive.getFileDetails(item.driveId, item.id); - } catch (OneDriveException e) { - // display what the error is - displayOneDriveErrorMessage(e.msg); - return; + if (item.webUrl != "") { + // use the database entry + webUrl = item.webUrl; + } else { + // value not in database ... query for it + JSONValue fileDetails; + try { + fileDetails = onedrive.getFileDetails(item.driveId, item.id); + } catch (OneDriveException e) { + // display what the error is + displayOneDriveErrorMessage(e.msg); + return; + } + if ((fileDetails.type() == JSONType.object) && ("webUrl" in fileDetails)) { + // Valid JSON object + webUrl = fileDetails["webUrl"].str; + } } - - if ((fileDetails.type() == JSONType.object) && ("webUrl" in fileDetails)) { - // Valid JSON object - writeln(fileDetails["webUrl"].str); + // respond with webUrl + writeln(webUrl); + } else { + // File has not been synced with OneDrive + log.error("File has not been synced with OneDrive: ", localFilePath); + } + } else { + // File does not exist locally + log.error("File not found on local system: ", localFilePath); + } + } + + // Validate on-demand file from OneDrive + bool validateOnDemandFile(string localFilePath, string syncDir) { + // Query if file is valid locally + if (exists(localFilePath)) { + // File exists locally, does it exist in the database + // Path needs to be relative to sync_dir path + string relativePath = relativePath(localFilePath, syncDir); + Item item; + if (itemdb.selectByPath(relativePath, defaultDriveId, item)) { + // File is in the local database cache + // File size + ulong fileSize = 0; + fileSize = getSize(localFilePath); + + // Is the local file technically 'newer' based on UTC timestamp? + SysTime localModifiedTime = timeLastModified(localFilePath).toUTC(); + localModifiedTime.fracSecs = Duration.zero; + item.mtime.fracSecs = Duration.zero; + + if ((fileSize < to!long(item.fileSize)) && (localModifiedTime == item.mtime)) { + // file on disk is smaller, mtime is the same as OneDrive + // high likelihood this is a file that needs to be replaced with the 'real' file + return true; + } else { + // file size or modified time is not what the DB / OneDrive thinks it is .. + log.error("File size or modified time is incorrect - not downloading on-demand file: ", localFilePath); } } else { // File has not been synced with OneDrive @@ -4240,8 +4325,29 @@ final class SyncEngine // File does not exist locally log.error("File not found on local system: ", localFilePath); } + return false; } + + // Download on-demand file from OneDrive + void downloadOnDemandFile(string localFilePath, string syncDir) { + // validation was done with validateOnDemandFile, download file + string relativePath = relativePath(localFilePath, syncDir); + Item item; + if (itemdb.selectByPath(relativePath, defaultDriveId, item)) { + try { + writeln("downloading on demand file: ", relativePath); + onedrive.downloadById(item.driveId, item.id, relativePath, to!long(item.fileSize)); + setTimes(relativePath, item.mtime, item.mtime); + } catch (OneDriveException e) { + displayOneDriveErrorMessage(e.msg); + } + } + } + + + + // Query the OneDrive 'drive' to determine if we are 'in sync' or if there are pending changes void queryDriveForChanges(string path) {