From 4d6f27087a0506536447677c832f7b4c495e0802 Mon Sep 17 00:00:00 2001 From: Keksov Date: Wed, 14 Nov 2018 12:46:57 +0300 Subject: [PATCH 1/6] Reuse existsing repo cache --- gitfs/repository.py | 30 +++++++++++++++++++++++++----- requirements.txt | 8 ++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/gitfs/repository.py b/gitfs/repository.py index 9f061104..bc7cd265 100644 --- a/gitfs/repository.py +++ b/gitfs/repository.py @@ -14,14 +14,15 @@ import os +import sys from collections import namedtuple from shutil import rmtree from stat import S_IFDIR, S_IFREG, S_IFLNK -from pygit2 import (clone_repository, Signature, GIT_SORT_TOPOLOGICAL, +from pygit2 import (clone_repository, discover_repository, init_repository, Signature, GIT_SORT_TOPOLOGICAL, GIT_FILEMODE_TREE, GIT_STATUS_CURRENT, GIT_FILEMODE_LINK, GIT_FILEMODE_BLOB, GIT_BRANCH_REMOTE, - GIT_BRANCH_LOCAL, GIT_FILEMODE_BLOB_EXECUTABLE) + GIT_BRANCH_LOCAL, GIT_FILEMODE_BLOB_EXECUTABLE, GIT_REPOSITORY_INIT_NO_REINIT) from six import iteritems from gitfs.cache import CommitCache @@ -185,10 +186,29 @@ def clone(cls, remote_url, path, branch=None, credentials=None): clone. The default is to use the remote's default branch. """ - - repo = clone_repository(remote_url, path, checkout_branch=branch, + existingRepoPath = discover_repository(path) + if existingRepoPath=="": + log.debug("clone_repository %s", path) + + try: + repo = clone_repository(remote_url, path, checkout_branch=branch, callbacks=credentials) - repo.checkout_head() + except Exception, e: + log.error("[Exception] clone_repository failed: %s", str(e)) + sys.exit() + + repo.checkout_head() + log.info("repo cloned") + else: + log.debug("init_repository %s", existingRepoPath) + try: + repo = init_repository(existingRepoPath) + except Exception, e: + log.error("[Exception] init_repository failed: %s", str(e)) + sys.exit() + + log.info("existing repo opened") + return cls(repo) def _is_searched_entry(self, entry_name, searched_entry, path_components): diff --git a/requirements.txt b/requirements.txt index abafda81..752b0045 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -fusepy==2.0.2 -pygit2==0.24.1 -six==1.10.0 +fusepy>=2.0.4 +pygit2==0.26.0 +six>=1.10.0 atomiclong -raven==5.27.0 \ No newline at end of file +raven>=6.1.0 \ No newline at end of file From a6b9b91b8a1476b4772b1a7270a997b613606327 Mon Sep 17 00:00:00 2001 From: Keksov Date: Sun, 18 Nov 2018 11:22:31 +0300 Subject: [PATCH 2/6] Please ignore PR #300 --- gitfs/repository.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/gitfs/repository.py b/gitfs/repository.py index bc7cd265..8811e74e 100644 --- a/gitfs/repository.py +++ b/gitfs/repository.py @@ -186,8 +186,16 @@ def clone(cls, remote_url, path, branch=None, credentials=None): clone. The default is to use the remote's default branch. """ - existingRepoPath = discover_repository(path) - if existingRepoPath=="": + hasExistingRepo = False + try: + existingRepoPath = discover_repository(path) + if existingRepoPath<>"": + hasExistingRepo = True + except Exception, e: + log.debug("[Exception] discover_repository repo not found: %s", str(e)) + pass + + if hasExistingRepo == False: log.debug("clone_repository %s", path) try: @@ -207,7 +215,7 @@ def clone(cls, remote_url, path, branch=None, credentials=None): log.error("[Exception] init_repository failed: %s", str(e)) sys.exit() - log.info("existing repo opened") + log.info("existing repo '%s' opened", existingRepoPath) return cls(repo) From 79f9d26d22041a7c04076c7c6612f62dcf75659b Mon Sep 17 00:00:00 2001 From: Keksov Date: Sun, 18 Nov 2018 14:20:21 +0300 Subject: [PATCH 3/6] Fix in Repository.clone --- gitfs/repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitfs/repository.py b/gitfs/repository.py index 8811e74e..499e07c1 100644 --- a/gitfs/repository.py +++ b/gitfs/repository.py @@ -186,6 +186,7 @@ def clone(cls, remote_url, path, branch=None, credentials=None): clone. The default is to use the remote's default branch. """ + hasExistingRepo = False try: existingRepoPath = discover_repository(path) From b1e49a9579b585f74a1f360bffeb13d5a204715b Mon Sep 17 00:00:00 2001 From: Keksov Date: Sun, 18 Nov 2018 14:25:56 +0300 Subject: [PATCH 4/6] Do not remove repository on unmount --- gitfs/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitfs/router.py b/gitfs/router.py index b874ea79..71e1f97e 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -99,7 +99,7 @@ def destroy(self, path): worker.join() log.debug('Workers stopped') - shutil.rmtree(self.repo_path) + #shutil.rmtree(self.repo_path) log.info('Successfully umounted %s', self.mount_path) def __call__(self, operation, *args): From 3eba00a5c3d521457a00804a0c361d063c857f90 Mon Sep 17 00:00:00 2001 From: Keksov Date: Fri, 23 Nov 2018 11:03:57 +0300 Subject: [PATCH 5/6] Fixing "File too large" error when max_size=0, #293 --- gitfs/views/current.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitfs/views/current.py b/gitfs/views/current.py index 64518595..b9c02696 100644 --- a/gitfs/views/current.py +++ b/gitfs/views/current.py @@ -103,7 +103,7 @@ def write(self, path, buf, offset, fh): is off limit, raise EFBIG error and delete the file. """ - if offset + len(buf) > self.max_size: + if ( self.max_size > 0 ) and ( offset + len(buf) > self.max_size ): raise FuseOSError(errno.EFBIG) result = super(CurrentView, self).write(path, buf, offset, fh) From 41984eb2c1ab55933c408ba606c1d450b081a9c8 Mon Sep 17 00:00:00 2001 From: vegaminer Date: Mon, 26 Nov 2018 22:29:15 +0100 Subject: [PATCH 6/6] fix --- gitfs/cache/gitignore.py | 7 ++- gitfs/repository.py | 37 ++++++++++-- gitfs/router.py | 9 ++- gitfs/utils/decorators/not_in.py | 14 ++++- gitfs/views/current.py | 97 +++++++++++++++++++++++++------- requirements.txt | 8 +-- 6 files changed, 138 insertions(+), 34 deletions(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 92cfa000..83d6ea9a 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -18,7 +18,7 @@ import fnmatch from six import string_types - +from gitfs.log import log class CachedIgnore(object): def __init__(self, ignore=False, submodules=False, exclude=False, @@ -38,6 +38,8 @@ def __init__(self, ignore=False, submodules=False, exclude=False, def update(self): self.items = ['.git', '.git/*', '/.git/*', '*.keep', '*.gitmodules'] + log.debug("[ignore] CachedIgnore.update" ) + self.items += self._parse_ignore_file(self.ignore) self.items += self._parse_ignore_file(self.exclude) @@ -57,6 +59,7 @@ def update(self): def _parse_ignore_file(self, ignore_file): items = [] + log.debug("[ignore] _parse_ignore_file: %s", ignore_file) if ignore_file and os.path.exists(ignore_file): with open(ignore_file) as gitignore: for item in gitignore.readlines(): @@ -77,7 +80,9 @@ def __contains__(self, path): def check_key(self, key): for item in self.items: if self._check_item_and_key(item, key): + log.debug("[ignore] check_key match:%s,%s", item, key) return True + return False def _check_item_and_key(self, item, key): diff --git a/gitfs/repository.py b/gitfs/repository.py index 9f061104..499e07c1 100644 --- a/gitfs/repository.py +++ b/gitfs/repository.py @@ -14,14 +14,15 @@ import os +import sys from collections import namedtuple from shutil import rmtree from stat import S_IFDIR, S_IFREG, S_IFLNK -from pygit2 import (clone_repository, Signature, GIT_SORT_TOPOLOGICAL, +from pygit2 import (clone_repository, discover_repository, init_repository, Signature, GIT_SORT_TOPOLOGICAL, GIT_FILEMODE_TREE, GIT_STATUS_CURRENT, GIT_FILEMODE_LINK, GIT_FILEMODE_BLOB, GIT_BRANCH_REMOTE, - GIT_BRANCH_LOCAL, GIT_FILEMODE_BLOB_EXECUTABLE) + GIT_BRANCH_LOCAL, GIT_FILEMODE_BLOB_EXECUTABLE, GIT_REPOSITORY_INIT_NO_REINIT) from six import iteritems from gitfs.cache import CommitCache @@ -185,10 +186,38 @@ def clone(cls, remote_url, path, branch=None, credentials=None): clone. The default is to use the remote's default branch. """ + + hasExistingRepo = False + try: + existingRepoPath = discover_repository(path) + if existingRepoPath<>"": + hasExistingRepo = True + except Exception, e: + log.debug("[Exception] discover_repository repo not found: %s", str(e)) + pass - repo = clone_repository(remote_url, path, checkout_branch=branch, + if hasExistingRepo == False: + log.debug("clone_repository %s", path) + + try: + repo = clone_repository(remote_url, path, checkout_branch=branch, callbacks=credentials) - repo.checkout_head() + except Exception, e: + log.error("[Exception] clone_repository failed: %s", str(e)) + sys.exit() + + repo.checkout_head() + log.info("repo cloned") + else: + log.debug("init_repository %s", existingRepoPath) + try: + repo = init_repository(existingRepoPath) + except Exception, e: + log.error("[Exception] init_repository failed: %s", str(e)) + sys.exit() + + log.info("existing repo '%s' opened", existingRepoPath) + return cls(repo) def _is_searched_entry(self, entry_name, searched_entry, path_components): diff --git a/gitfs/router.py b/gitfs/router.py index b874ea79..ff8327e9 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -18,6 +18,7 @@ import time import shutil import inspect +import sys from pwd import getpwnam from grp import getgrnam @@ -80,7 +81,11 @@ def __init__(self, remote_url, repo_path, mount_path, self.max_size = kwargs['max_size'] self.max_offset = kwargs['max_offset'] - self.repo.commits.update() + try: + self.repo.commits.update() + except Exception, e: + log.error("[Exception] repo.commits.update failed: %s", str(e)) + sys.exit() self.workers = [] @@ -99,7 +104,7 @@ def destroy(self, path): worker.join() log.debug('Workers stopped') - shutil.rmtree(self.repo_path) + #shutil.rmtree(self.repo_path) log.info('Successfully umounted %s', self.mount_path) def __call__(self, operation, *args): diff --git a/gitfs/utils/decorators/not_in.py b/gitfs/utils/decorators/not_in.py index c48e5399..64b5dbfa 100644 --- a/gitfs/utils/decorators/not_in.py +++ b/gitfs/utils/decorators/not_in.py @@ -20,7 +20,7 @@ from six import string_types from fuse import FuseOSError - +from gitfs.log import log class not_in(object): def __init__(self, look_at, check=None): @@ -34,7 +34,17 @@ def decorated(their_self, *args, **kwargs): if isinstance(self.look_at, string_types): self.look_at = getattr(their_self, self.look_at) - self.check_args(f, args) + #self.check_args(f, args) + + gitignore = False + try: + self.check_args(f, args) + except Exception, e: + gitignore = True + pass + + kwargs["gitignore"] = gitignore + result = f(their_self, *args, **kwargs) return result diff --git a/gitfs/views/current.py b/gitfs/views/current.py index 64518595..f88c4c03 100644 --- a/gitfs/views/current.py +++ b/gitfs/views/current.py @@ -38,10 +38,14 @@ def __init__(self, *args, **kwargs): @write_operation @not_in("ignore", check=["old", "new"]) - def rename(self, old, new): + def rename(self, old, new, gitignore): new = re.sub(self.regex, '', new) result = super(CurrentView, self).rename(old, new) + if gitignore == True: + log.debug("[ignore] rename:%s", name) + return result + message = "Rename {} to {}".format(old, new) self._stage(**{ 'remove': os.path.split(old)[1], @@ -54,9 +58,13 @@ def rename(self, old, new): @write_operation @not_in("ignore", check=["target"]) - def symlink(self, name, target): + def symlink(self, name, target, gitignore): result = os.symlink(target, self.repo._full_path(name)) - + + if gitignore == True: + log.debug("[ignore] symlink:%s", name) + return result + message = "Create symlink to {} for {}".format(target, name) self._stage(add=name, message=message) @@ -65,12 +73,16 @@ def symlink(self, name, target): @write_operation @not_in("ignore", check=["target"]) - def link(self, name, target): + def link(self, name, target, gitignore): if target.startswith('/%s/' % self.current_path): target = target.replace('/%s/' % self.current_path, '/') result = super(CurrentView, self).link(target, name) - + + if gitignore == True: + log.debug("[ignore] link:%s", name) + return result + message = "Create link to {} for {}".format(target, name) self._stage(add=name, message=message) @@ -96,17 +108,22 @@ def getattr(self, path, fh=None): @write_operation @not_in("ignore", check=["path"]) - def write(self, path, buf, offset, fh): + def write(self, path, buf, offset, fh, gitignore): """ We don't like big big files, so we need to be really carefull with them. First we check for offset, then for size. If any of this is off limit, raise EFBIG error and delete the file. """ - if offset + len(buf) > self.max_size: + if ( self.max_size > 0 ) and ( offset + len(buf) > self.max_size ): raise FuseOSError(errno.EFBIG) result = super(CurrentView, self).write(path, buf, offset, fh) + + if gitignore == True: + log.debug("[ignore] write:%s. Wrote %s bytes to %s", path, len(buf), path) + return result + self.dirty[fh] = { 'message': 'Update {}'.format(path), 'stage': True @@ -117,9 +134,13 @@ def write(self, path, buf, offset, fh): @write_operation @not_in("ignore", check=["path"]) - def mkdir(self, path, mode): + def mkdir(self, path, mode, gitignore): result = super(CurrentView, self).mkdir(path, mode) - + + if gitignore == True: + log.debug("[ignore] mkdir:%s", path) + return result + keep_path = "{}/.keep".format(path) full_path = self.repo._full_path(keep_path) if not os.path.exists(keep_path): @@ -141,10 +162,15 @@ def mkdir(self, path, mode): return result - def create(self, path, mode, fi=None): + @not_in("ignore", check=["path"]) + def create(self, path, mode, fi=None, gitignore=False): fh = self.open_for_write(path, os.O_WRONLY | os.O_CREAT) super(CurrentView, self).chmod(path, mode) + if gitignore == True: + log.debug("[ignore] create:%s", path) + return fh + self.dirty[fh] = { 'message': "Created {}".format(path), 'stage': True @@ -155,10 +181,11 @@ def create(self, path, mode, fi=None): @write_operation @not_in("ignore", check=["path"]) - def chmod(self, path, mode): + def chmod(self, path, mode, gitignore): """ Executes chmod on the file at os level and then it commits the change. """ + str_mode = ('%o' % mode)[-4:] if str_mode not in ['0755', '0644']: raise FuseOSError(errno.EINVAL) @@ -168,6 +195,10 @@ def chmod(self, path, mode): if os.path.isdir(self.repo._full_path(path)): return result + if gitignore == True: + log.debug("[ignore] chmod:%s", path) + return result + message = 'Chmod to {} on {}'.format(str_mode, path) self._stage(add=path, message=message) @@ -177,13 +208,17 @@ def chmod(self, path, mode): @write_operation @not_in("ignore", check=["path"]) - def fsync(self, path, fdatasync, fh): + def fsync(self, path, fdatasync, fh, gitignore): """ Each time you fsync, a new commit and push are made """ result = super(CurrentView, self).fsync(path, fdatasync, fh) - + + if gitignore == True: + log.debug("[ignore] fsync:%s", path) + return result + message = 'Fsync {}'.format(path) self._stage(add=path, message=message) @@ -192,10 +227,16 @@ def fsync(self, path, fdatasync, fh): @write_operation @not_in("ignore", check=["path"]) - def open_for_write(self, path, flags): - global writers + def open_for_write(self, path, flags, gitignore): fh = self.open_for_read(path, flags) - writers += 1 + + if gitignore == True: + log.debug("[ignore] open_for_write:%s", path) + return fh + + global writers + writers += 1 + self.dirty[fh] = { 'message': "Opened {} for write".format(path), 'stage': False @@ -207,21 +248,30 @@ def open_for_write(self, path, flags): def open_for_read(self, path, flags): full_path = self.repo._full_path(path) log.info("CurrentView: Open %s for read", path) + return os.open(full_path, flags) def open(self, path, flags): write_mode = flags & (os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_CREAT) if write_mode: + log.debug("[ignore] open.write_mode: %s", path) return self.open_for_write(path, flags) + + log.debug("[ignore] open.read_mode: %s", path) return self.open_for_read(path, flags) - def release(self, path, fh): + @not_in("ignore", check=["path"]) + def release(self, path, fh, gitignore): """ Check for path if something was written to. If so, commit and push the changed to upstream. """ + if gitignore == True: + log.debug("[ignore] release:%s", path) + return os.close(fh) + if fh in self.dirty: message = self.dirty[fh]['message'] should_stage = self.dirty[fh].get('stage', False) @@ -238,7 +288,7 @@ def release(self, path, fh): @write_operation @not_in("ignore", check=["path"]) - def rmdir(self, path): + def rmdir(self, path, gitignore): message = 'Delete the {} directory'.format(path) # Unlink all the files @@ -248,7 +298,8 @@ def rmdir(self, path): deleting_file = os.path.join(root, _file) if os.path.exists(deleting_file): result = super(CurrentView, self).unlink(os.path.join(path, _file)) - self._stage(remove=os.path.join(path, _file), message=message) + if gitignore == False: + self._stage(remove=os.path.join(path, _file), message=message) # Delete the actual directory result = super(CurrentView, self).rmdir("{}/".format(path)) @@ -258,9 +309,13 @@ def rmdir(self, path): @write_operation @not_in("ignore", check=["path"]) - def unlink(self, path): + def unlink(self, path, gitignore): result = super(CurrentView, self).unlink(path) - + + if gitignore == True: + log.debug("[ignore] unlink:%s", path) + return result + message = 'Deleted {}'.format(path) self._stage(remove=path, message=message) diff --git a/requirements.txt b/requirements.txt index abafda81..e4184956 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -fusepy==2.0.2 -pygit2==0.24.1 -six==1.10.0 +fusepy>=2.0.4 +pygit2==0.26.0 +six>=1.10.0 atomiclong -raven==5.27.0 \ No newline at end of file +raven>=6.1.0