Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Implement On-Demand file access #921

Closed
wants to merge 1 commit into from
Closed

WIP: Implement On-Demand file access #921

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/config.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ final class Config
stringValues["application_id"] = "";
// allow for resync to be set via config file
boolValues["resync"] = false;
// Treat new 'remote' files as 'on-demand' - that is, do not download them but save that it exists
boolValues["on_demand"] = false;

// Determine the users home directory.
// Need to avoid using ~ here as expandTilde() below does not interpret correctly when running under init.d or systemd scripts
Expand Down
46 changes: 27 additions & 19 deletions src/itemdb.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,27 @@ enum ItemType {
}

struct Item {
string driveId;
string id;
string name;
ItemType type;
string eTag;
string cTag;
SysTime mtime;
string parentId;
string crc32Hash;
string sha1Hash;
string quickXorHash;
string remoteDriveId;
string remoteId;
string driveId; // #1
string id; // #2
string name; // #3
ItemType type; // #4
string eTag; // #5
string cTag; // #6
SysTime mtime; // #7
string parentId; // #8
string crc32Hash; // #9
string sha1Hash; // #10
string quickXorHash; // #11
string remoteDriveId; // #12
string remoteId; // #13
string webUrl; // #14
string fileSize; // #15
}

final class ItemDatabase
{
// increment this for every change in the db schema
immutable int itemDatabaseVersion = 9;
immutable int itemDatabaseVersion = 10;

Database db;
string insertItemStmt;
Expand Down Expand Up @@ -83,12 +85,12 @@ final class ItemDatabase
db.exec("PRAGMA auto_vacuum = FULL");

insertItemStmt = "
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)
INSERT OR REPLACE INTO item (driveId, id, name, type, eTag, cTag, mtime, parentId, crc32Hash, sha1Hash, quickXorHash, remoteDriveId, remoteId, webUrl, fileSize)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)
";
updateItemStmt = "
UPDATE item
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13
SET name = ?3, type = ?4, eTag = ?5, cTag = ?6, mtime = ?7, parentId = ?8, crc32Hash = ?9, sha1Hash = ?10, quickXorHash = ?11, remoteDriveId = ?12, remoteId = ?13, webUrl = ?14, fileSize = ?15
WHERE driveId = ?1 AND id = ?2
";
selectItemByIdStmt = "
Expand Down Expand Up @@ -117,6 +119,8 @@ final class ItemDatabase
remoteDriveId TEXT,
remoteId TEXT,
deltaLink TEXT,
webUrl TEXT,
fileSize TEXT,
PRIMARY KEY (driveId, id),
FOREIGN KEY (driveId, parentId)
REFERENCES item (driveId, id)
Expand Down Expand Up @@ -300,13 +304,15 @@ final class ItemDatabase
bind(11, quickXorHash);
bind(12, remoteDriveId);
bind(13, remoteId);
bind(14, webUrl);
bind(15, fileSize);
}
}

private Item buildItem(Statement.Result result)
{
assert(!result.empty, "The result must not be empty");
assert(result.front.length == 14, "The result must have 14 columns");
assert(result.front.length == 16, "The result must have 16 columns");
Item item = {
driveId: result.front[0].dup,
id: result.front[1].dup,
Expand All @@ -319,7 +325,9 @@ final class ItemDatabase
sha1Hash: result.front[9].dup,
quickXorHash: result.front[10].dup,
remoteDriveId: result.front[11].dup,
remoteId: result.front[12].dup
remoteId: result.front[12].dup,
webUrl: result.front[14].dup,
fileSize: result.front[15].dup
};
switch (result.front[3]) {
case "file": item.type = ItemType.file; break;
Expand Down
45 changes: 34 additions & 11 deletions src/main.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ int main(string[] args)
if (cfg.getValueBool("dry_run")) {
log.log("DRY-RUN Configured. Output below shows what 'would' have occurred.");
}

// Are we able to reach the OneDrive Service
bool online = false;

Expand Down Expand Up @@ -607,6 +607,12 @@ int main(string[] args)
// Do we configure to disable the upload validation routine
if (cfg.getValueBool("disable_upload_validation")) sync.setDisableUploadValidation();

// Do we configure on-demand file access mode .. only and ONLY if --monitor is also in use
if (cfg.getValueBool("monitor") && cfg.getValueBool("on_demand")) {
log.log("IMPORTANT: On-Demand access of files has been enabled.");
sync.setOnDemandAccessEnabled();
}

// Do we need to validate the syncDir to check for the presence of a '.nosync' file
if (cfg.getValueBool("check_nomount")) {
// we were asked to check the mounts
Expand Down Expand Up @@ -687,6 +693,33 @@ int main(string[] args)
log.logAndNotify("Initializing monitor ...");
log.log("OneDrive monitor interval (seconds): ", cfg.getValueLong("monitor_interval"));
Monitor m = new Monitor(selectiveSync);

m.onFileOpened = delegate(string path) {
if (sync.getOnDemandAccessEnabled){
// On-Demand access is enabled
if (sync.validateOnDemandFile(path, syncDir)) {
log.log("[M] On-Demand File requested: ", path);
// Temp remove watch
m.removeFileWatch(path);
// Download replacement file
sync.downloadOnDemandFile(path, syncDir);
// Add watch
m.addFileWatch(path);
}
}
};

m.onFileChanged = delegate(string path) {
log.vlog("[M] File changed: ", path);
try {
sync.scanForDifferences(path);
} catch (CurlException e) {
log.vlog("Offline, cannot upload changed item!");
} catch(Exception e) {
log.logAndNotify("Cannot upload file changes/creation: ", e.msg);
}
};

m.onDirCreated = delegate(string path) {
// Handle .folder creation if skip_dotfiles is enabled
if ((cfg.getValueBool("skip_dotfiles")) && (selectiveSync.isDotFile(path))) {
Expand All @@ -702,16 +735,6 @@ int main(string[] args)
}
}
};
m.onFileChanged = delegate(string path) {
log.vlog("[M] File changed: ", path);
try {
sync.scanForDifferences(path);
} catch (CurlException e) {
log.vlog("Offline, cannot upload changed item!");
} catch(Exception e) {
log.logAndNotify("Cannot upload file changes/creation: ", e.msg);
}
};
m.onDelete = delegate(string path) {
log.vlog("[M] Item deleted: ", path);
try {
Expand Down
64 changes: 61 additions & 3 deletions src/monitor.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@ import core.sys.linux.sys.inotify;
import core.stdc.errno;
import core.sys.posix.poll, core.sys.posix.unistd;
import std.exception, std.file, std.path, std.regex, std.stdio, std.string, std.algorithm.mutation;
import std.conv;
import config;
import selective;
import util;
static import log;

// relevant inotify events
private immutable uint32_t mask = IN_CLOSE_WRITE | IN_CREATE | IN_DELETE |
IN_MOVE | IN_IGNORED | IN_Q_OVERFLOW;
IN_MOVE | IN_IGNORED | IN_Q_OVERFLOW | IN_OPEN;

class MonitorException: ErrnoException
{
Expand Down Expand Up @@ -38,6 +39,7 @@ final class Monitor
private SelectiveSync selectiveSync;

void delegate(string path) onDirCreated;
void delegate(string path) onFileOpened;
void delegate(string path) onFileChanged;
void delegate(string path) onDelete;
void delegate(string from, string to) onMove;
Expand All @@ -54,7 +56,7 @@ final class Monitor
this.skip_symlinks = skip_symlinks;
this.check_nosync = check_nosync;

assert(onDirCreated && onFileChanged && onDelete && onMove);
assert(onDirCreated && onFileChanged && onDelete && onMove && onFileOpened);
fd = inotify_init();
if (fd < 0) throw new MonitorException("inotify_init failed");
if (!buffer) buffer = new void[4096];
Expand Down Expand Up @@ -199,10 +201,57 @@ final class Monitor
int ret = inotify_rm_watch(fd, wd);
if (ret < 0) throw new MonitorException("inotify_rm_watch failed");
wdToDirName.remove(wd);
log.vlog("Monitored directory removed: ", dirname);
log.log("Monitored directory removed: ", dirname);
}
}
}

// remove a specific path from being watched
void removeFileWatch(const(char)[] path)
{
string parentPath = to!string(dirName(path));
foreach (wd, dirname; wdToDirName) {
if (dirname.startsWith(parentPath)) {
int ret = inotify_rm_watch(fd, wd);
if (ret < 0) throw new MonitorException("inotify_rm_watch failed");
wdToDirName.remove(wd);
log.vdebug("Monitor removed for: ", path);

}
}
}

// add a specific path to being watched
void addFileWatch(const(char)[] path)
{
string parentPath = to!string(dirName(path));
int wd = inotify_add_watch(fd, toStringz(parentPath), mask);
if (wd < 0) {
if (errno() == ENOSPC) {
log.log("The user limit on the total number of inotify watches has been reached.");
log.log("To see the current max number of watches run:");
log.log("sysctl fs.inotify.max_user_watches");
log.log("To change the current max number of watches to 524288 run:");
log.log("sudo sysctl fs.inotify.max_user_watches=524288");
}
if (errno() == 13) {
if ((selectiveSync.getSkipDotfiles()) && (selectiveSync.isDotFile(parentPath))) {
// no misleading output that we could not add a watch due to permission denied
return;
} else {
log.vlog("WARNING: inotify_add_watch failed - permission denied: ", parentPath);
return;
}
}
// Flag any other errors
log.error("ERROR: inotify_add_watch failed: ", parentPath);
return;
}
// Add path to inotify watch - required regardless if a '.folder' or 'folder'
wdToDirName[wd] = buildNormalizedPath(parentPath) ~ "/";
// log that this is agail being monitored
log.vdebug("Monitor added for: ", path);
}

// return the file path from an inotify event
private string getPath(const(inotify_event)* event)
Expand Down Expand Up @@ -284,6 +333,15 @@ final class Monitor
} else if ((event.mask & IN_CLOSE_WRITE) && !(event.mask & IN_ISDIR)) {
log.vdebug("event IN_CLOSE_WRITE and ...: ", path);
if (useCallbacks) onFileChanged(path);
} else if (event.mask & IN_OPEN) {
if ((event.mask & IN_OPEN) && !(event.mask & IN_ISDIR)) {
// open event for a file
log.vdebug("event IN_OPEN detected for: ", path);
if (useCallbacks) onFileOpened(path);
} else {
// open event for a directory
goto skip;
}
} else {
log.vdebug("event unhandled: ", path);
assert(0);
Expand Down
6 changes: 3 additions & 3 deletions src/onedrive.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ final class OneDriveApi
const(char)[] url = deltaLink;
if (url == null) {
url = driveByIdUrl ~ driveId ~ "/items/" ~ id ~ "/delta";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl";
}
return get(url);
}
Expand Down Expand Up @@ -357,7 +357,7 @@ final class OneDriveApi
// string itemByPathUrl = "https://graph.microsoft.com/v1.0/me/drive/root:/";
if ((path == ".")||(path == "/")) url = driveUrl ~ "/root/";
else url = itemByPathUrl ~ encodeComponent(path) ~ ":/";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl";
return get(url);
}

Expand All @@ -369,7 +369,7 @@ final class OneDriveApi
const(char)[] url;
// string driveByIdUrl = "https://graph.microsoft.com/v1.0/drives/";
url = driveByIdUrl ~ driveId ~ "/items/" ~ id;
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size";
url ~= "?select=id,name,eTag,cTag,deleted,file,folder,root,fileSystemInfo,remoteItem,parentReference,size,webUrl";
return get(url);
}

Expand Down
Loading