Enable skipping sanity checks using --force (#18, #21)

The mentioned issue / pull request made it clear to me that its
very well possible for these (well intentioned) sanity checks
to fail. Nevertheless I'm not inclined to just remove them.
I do believe the changes here and in daa41a2 suitably
accommodate the use case described in issue 18.

README.rst

@@ -206,11 +206,6 @@ intended you have no right to complain ;-).
    that sections in user-specific configuration files override sections by the
    same name in system-wide configuration files. For more details refer to the
    online documentation."
-   "``-u``, ``--use-sudo``","Enable the use of ""sudo"" to rotate backups in directories that are not
-   readable and/or writable for the current user (or the user logged in to a
-   remote system over SSH)."
-   "``-n``, ``--dry-run``","Don't make any changes, just print what would be done. This makes it easy
-   to evaluate the impact of a rotation scheme without losing any backups."
    "``-C``, ``--removal-command=CMD``","Change the command used to remove backups. The value of ``CMD`` defaults to
    ``rm ``-f``R``. This choice was made because it works regardless of whether
    ""backups to be rotated"" are files or directories or a mixture of both.
@@ -219,6 +214,17 @@ intended you have no right to complain ;-).
    represented as regular directory trees that can be deleted at once with a
    single 'rmdir' command (even though according to POSIX semantics this
    command should refuse to remove nonempty directories, but I digress)."
+   "``-u``, ``--use-sudo``","Enable the use of ""sudo"" to rotate backups in directories that are not
+   readable and/or writable for the current user (or the user logged in to a
+   remote system over SSH)."
+   "``-f``, ``--force``","If a sanity check fails an error is reported and the program aborts. You
+   can use ``--force`` to continue with backup rotation instead. Sanity checks
+   are done to ensure that the given DIRECTORY exists, is readable and is
+   writable. If the ``--removal-command`` option is given then the last sanity
+   check (that the given location is writable) is skipped (because custom
+   removal commands imply custom semantics)."
+   "``-n``, ``--dry-run``","Don't make any changes, just print what would be done. This makes it easy
+   to evaluate the impact of a rotation scheme without losing any backups."
    "``-v``, ``--verbose``",Increase logging verbosity (can be repeated).
    "``-q``, ``--quiet``",Decrease logging verbosity (can be repeated).
    "``-h``, ``--help``",Show this message and exit.

rotate_backups/__init__.py

@@ -27,7 +27,7 @@
 from executor.concurrent import CommandPool
 from executor.contexts import RemoteContext, create_context
 from humanfriendly import Timer, coerce_boolean, format_path, parse_path, pluralize
-from humanfriendly.text import compact, concatenate, split
+from humanfriendly.text import concatenate, split
 from natsort import natsort
 from property_manager import (
     PropertyManager,
@@ -294,6 +294,28 @@ def exclude_list(self):
         """
         return []
 
+    @mutable_property
+    def force(self):
+        """
+        :data:`True` to continue if sanity checks fail, :data:`False` to raise an exception.
+
+        Sanity checks are performed before backup rotation starts to ensure
+        that the given location exists, is readable and is writable. If
+        :attr:`removal_command` is customized then the last sanity check (that
+        the given location is writable) is skipped (because custom removal
+        commands imply custom semantics, see also `#18`_). If a sanity check
+        fails an exception is raised, but you can set :attr:`force` to
+        :data:`True` to continue with backup rotation instead (the default is
+        obviously :data:`False`).
+
+        .. seealso:: :func:`Location.ensure_exists()`,
+                     :func:`Location.ensure_readable()` and
+                     :func:`Location.ensure_writable()`
+
+        .. _#18: https://github.com/xolox/python-rotate-backups/issues/18
+        """
+        return False
+
     @cached_property(writable=True)
     def include_list(self):
         """
@@ -477,7 +499,7 @@ class together to implement backup rotation with an easy to use Python
         # https://github.com/xolox/python-rotate-backups/issues/18 for
         # more details about one such use case).
         if not self.dry_run and (self.removal_command == DEFAULT_REMOVAL_COMMAND):
-            location.ensure_writable()
+            location.ensure_writable(self.force)
         most_recent_backup = sorted_backups[-1]
         # Group the backups by the rotation frequencies.
         backups_by_frequency = self.group_backups(sorted_backups)
@@ -563,7 +585,7 @@ def collect_backups(self, location):
         backups = []
         location = coerce_location(location)
         logger.info("Scanning %s for backups ..", location)
-        location.ensure_readable()
+        location.ensure_readable(self.force)
         for entry in natsort(location.context.list_entries(location.directory)):
             match = TIMESTAMP_PATTERN.search(entry)
             if match:
@@ -733,49 +755,105 @@ def key_properties(self):
         """
         return ['ssh_alias', 'directory'] if self.is_remote else ['directory']
 
-    def ensure_exists(self):
-        """Make sure the location exists."""
-        if not self.context.is_directory(self.directory):
-            # This can also happen when we don't have permission to one of the
-            # parent directories so we'll point that out in the error message
-            # when it seems applicable (so as not to confuse users).
-            if self.context.have_superuser_privileges:
-                msg = "The directory %s doesn't exist!"
-                raise ValueError(msg % self)
-            else:
-                raise ValueError(compact("""
-                    The directory {location} isn't accessible, most likely
-                    because it doesn't exist or because of permissions. If
-                    you're sure the directory exists you can use the
-                    --use-sudo option.
-                """, location=self))
-
-    def ensure_readable(self):
-        """Make sure the location exists and is readable."""
-        self.ensure_exists()
-        if not self.context.is_readable(self.directory):
-            if self.context.have_superuser_privileges:
-                msg = "The directory %s isn't readable!"
-                raise ValueError(msg % self)
+    def ensure_exists(self, override=False):
+        """
+        Sanity check that the location exists.
+
+        :param override: :data:`True` to log a message, :data:`False` to raise
+                         an exception (when the sanity check fails).
+        :returns: :data:`True` if the sanity check succeeds,
+                  :data:`False` if it fails (and `override` is :data:`True`).
+        :raises: :exc:`~exceptions.ValueError` when the sanity
+                 check fails and `override` is :data:`False`.
+
+        .. seealso:: :func:`ensure_readable()`, :func:`ensure_writable()` and :func:`add_hints()`
+        """
+        if self.context.is_directory(self.directory):
+            logger.verbose("Confirmed that location exists: %s", self)
+            return True
+        elif override:
+            logger.notice("It seems %s doesn't exist but --force was given so continuing anyway ..", self)
+            return False
+        else:
+            message = "It seems %s doesn't exist or isn't accessible due to filesystem permissions!"
+            raise ValueError(self.add_hints(message % self))
+
+    def ensure_readable(self, override=False):
+        """
+        Sanity check that the location exists and is readable.
+
+        :param override: :data:`True` to log a message, :data:`False` to raise
+                         an exception (when the sanity check fails).
+        :returns: :data:`True` if the sanity check succeeds,
+                  :data:`False` if it fails (and `override` is :data:`True`).
+        :raises: :exc:`~exceptions.ValueError` when the sanity
+                 check fails and `override` is :data:`False`.
+
+        .. seealso:: :func:`ensure_exists()`, :func:`ensure_writable()` and :func:`add_hints()`
+        """
+        # Only sanity check that the location is readable when its
+        # existence has been confirmed, to avoid multiple notices
+        # about the same underlying problem.
+        if self.ensure_exists(override):
+            if self.context.is_readable(self.directory):
+                logger.verbose("Confirmed that location is readable: %s", self)
+                return True
+            elif override:
+                logger.notice("It seems %s isn't readable but --force was given so continuing anyway ..", self)
             else:
-                raise ValueError(compact("""
-                    The directory {location} isn't readable, most likely
-                    because of permissions. Consider using the --use-sudo
-                    option.
-                """, location=self))
-
-    def ensure_writable(self):
-        """Make sure the directory exists and is writable."""
-        self.ensure_exists()
-        if not self.context.is_writable(self.directory):
-            if self.context.have_superuser_privileges:
-                msg = "The directory %s isn't writable!"
-                raise ValueError(msg % self)
+                message = "It seems %s isn't readable!"
+                raise ValueError(self.add_hints(message % self))
+        return False
+
+    def ensure_writable(self, override=False):
+        """
+        Sanity check that the directory exists and is writable.
+
+        :param override: :data:`True` to log a message, :data:`False` to raise
+                         an exception (when the sanity check fails).
+        :returns: :data:`True` if the sanity check succeeds,
+                  :data:`False` if it fails (and `override` is :data:`True`).
+        :raises: :exc:`~exceptions.ValueError` when the sanity
+                 check fails and `override` is :data:`False`.
+
+        .. seealso:: :func:`ensure_exists()`, :func:`ensure_readable()` and :func:`add_hints()`
+        """
+        # Only sanity check that the location is readable when its
+        # existence has been confirmed, to avoid multiple notices
+        # about the same underlying problem.
+        if self.ensure_exists(override):
+            if self.context.is_writable(self.directory):
+                logger.verbose("Confirmed that location is writable: %s", self)
+                return True
+            elif override:
+                logger.notice("It seems %s isn't writable but --force was given so continuing anyway ..", self)
             else:
-                raise ValueError(compact("""
-                    The directory {location} isn't writable, most likely due
-                    to permissions. Consider using the --use-sudo option.
-                """, location=self))
+                message = "It seems %s isn't writable!"
+                raise ValueError(self.add_hints(message % self))
+        return False
+
+    def add_hints(self, message):
+        """
+        Provide hints about failing sanity checks.
+
+        :param message: The message to the user (a string).
+        :returns: The message including hints (a string).
+
+        When superuser privileges aren't being used a hint about the
+        ``--use-sudo`` option will be added (in case a sanity check failed
+        because we don't have permission to one of the parent directories).
+
+        In all cases a hint about the ``--force`` option is added (in case the
+        sanity checks themselves are considered the problem, which is obviously
+        up to the operator to decide).
+
+        .. seealso:: :func:`ensure_exists()`, :func:`ensure_readable()` and :func:`ensure_writable()`
+        """
+        sentences = [message]
+        if not self.context.have_superuser_privileges:
+            sentences.append("If filesystem permissions are the problem consider using the --use-sudo option.")
+        sentences.append("To continue despite this failing sanity check you can use --force.")
+        return " ".join(sentences)
 
     def match(self, location):
         """

rotate_backups/cli.py

@@ -1,7 +1,7 @@
 # rotate-backups: Simple command line interface for backup rotation.
 #
 # Author: Peter Odding <[email protected]>
-# Last Change: August 2, 2018
+# Last Change: February 12, 2020
 # URL: https://github.com/xolox/python-rotate-backups
 
 """
@@ -143,17 +143,6 @@
     same name in system-wide configuration files. For more details refer to the
     online documentation.
 
-  -u, --use-sudo
-
-    Enable the use of `sudo' to rotate backups in directories that are not
-    readable and/or writable for the current user (or the user logged in to a
-    remote system over SSH).
-
-  -n, --dry-run
-
-    Don't make any changes, just print what would be done. This makes it easy
-    to evaluate the impact of a rotation scheme without losing any backups.
-
   -C, --removal-command=CMD
 
     Change the command used to remove backups. The value of CMD defaults to
@@ -165,6 +154,26 @@
     single 'rmdir' command (even though according to POSIX semantics this
     command should refuse to remove nonempty directories, but I digress).
 
+  -u, --use-sudo
+
+    Enable the use of `sudo' to rotate backups in directories that are not
+    readable and/or writable for the current user (or the user logged in to a
+    remote system over SSH).
+
+  -f, --force
+
+    If a sanity check fails an error is reported and the program aborts. You
+    can use --force to continue with backup rotation instead. Sanity checks
+    are done to ensure that the given DIRECTORY exists, is readable and is
+    writable. If the --removal-command option is given then the last sanity
+    check (that the given location is writable) is skipped (because custom
+    removal commands imply custom semantics).
+
+  -n, --dry-run
+
+    Don't make any changes, just print what would be done. This makes it easy
+    to evaluate the impact of a rotation scheme without losing any backups.
+
   -v, --verbose
 
     Increase logging verbosity (can be repeated).
@@ -214,11 +223,11 @@ def main():
     selected_locations = []
     # Parse the command line arguments.
     try:
-        options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [
+        options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:fnvqh', [
             'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',
             'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',
-            'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=',
-            'verbose', 'quiet', 'help',
+            'ionice=', 'config=', 'removal-command=', 'use-sudo', 'force',
+            'dry-run', 'verbose', 'quiet', 'help',
         ])
         for option, value in options:
             if option in ('-M', '--minutely'):
@@ -248,15 +257,17 @@ def main():
                 kw['io_scheduling_class'] = value
             elif option in ('-c', '--config'):
                 kw['config_file'] = parse_path(value)
+            elif option in ('-C', '--removal-command'):
+                removal_command = shlex.split(value)
+                logger.info("Using custom removal command: %s", removal_command)
+                kw['removal_command'] = removal_command
             elif option in ('-u', '--use-sudo'):
                 use_sudo = True
+            elif option in ('-f', '--force'):
+                kw['force'] = True
             elif option in ('-n', '--dry-run'):
                 logger.info("Performing a dry run (because of %s option) ..", option)
                 kw['dry_run'] = True
-            elif option in ('-C', '--removal-command'):
-                removal_command = shlex.split(value)
-                logger.info("Using custom removal command: %s", removal_command)
-                kw['removal_command'] = removal_command
             elif option in ('-v', '--verbose'):
                 coloredlogs.increase_verbosity()
             elif option in ('-q', '--quiet'):

rotate_backups/tests.py

@@ -1,7 +1,7 @@
 # Test suite for the `rotate-backups' Python package.
 #
 # Author: Peter Odding <[email protected]>
-# Last Change: February 11, 2020
+# Last Change: February 12, 2020
 # URL: https://github.com/xolox/python-rotate-backups
 
 """Test suite for the `rotate-backups` package."""
@@ -13,6 +13,7 @@
 import os
 
 # External dependencies.
+from executor import ExternalCommandFailed
 from executor.contexts import RemoteContext
 from humanfriendly.testing import TemporaryDirectory, TestCase, run_cli, touch
 from six.moves import configparser
@@ -393,6 +394,15 @@ def test_removal_command(self):
             commands = program.rotate_backups(root, prepare=True)
             assert any(cmd.command_line[0] == 'rmdir' for cmd in commands)
 
+    def test_force(self):
+        """Test that sanity checks can be overridden."""
+        with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root:
+            for date in '2019-03-05', '2019-03-06':
+                os.mkdir(os.path.join(root, date))
+            with readonly_directory(root):
+                program = RotateBackups(force=True, rotation_scheme=dict(monthly='always'))
+                self.assertRaises(ExternalCommandFailed, program.rotate_backups, root)
+
     def test_ensure_writable(self):
         """Test that ensure_writable() complains when the location isn't writable."""
         with TemporaryDirectory(prefix='rotate-backups-', suffix='-test-suite') as root: