Refactoring for testing

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+omit =
+    munininfluxdb/test/*
+

.gitignore

@@ -1,8 +1,9 @@
 *.pyc
 *~
-venv/
-munininfluxdb/test*
 .idea/
-local/
+/*.egg-info
 data/
 lib/
+local/
+venv/
+/.cache

README.md

@@ -57,10 +57,14 @@ Installation & Usage
 3. Run ```import``` command: 
 
   ```
-  $ sudo ./muninflux import
+  $ sudo muninflux import
   ```
 
-4. A cron job will be automatically added after installation to refresh data from munin every 5 minutes (Munin default)
+4. Install the CRON job to refresh data from munin every 5 minutes (Munin default)
+
+  ```
+  $ sudo muninflux cron install
+  ```
 
 ### Some more details
 

munininfluxdb/commands/cron.py

@@ -0,0 +1,110 @@
+from __future__ import print_function
+import os
+import pwd
+import sys
+
+from munininfluxdb.utils import Symbol, absolute_executable
+
+
+# Cron job comment is used to uninstall and must not be manually deleted from the crontab
+CRON_COMMENT = 'Update InfluxDB with fresh values from Munin'
+NAME = 'cron'
+DESCRIPTION = 'Installs or uninstalls the CRON job'
+
+
+def get_cron_user():
+    try:
+        pwd.getpwnam('munin')
+    except KeyError:
+        output = 'root'
+    else:
+        output = 'munin'
+    return output
+
+
+def uninstall_cron(cron_adapter):
+    """
+    Creates a function which uses *cron_adapter* to remove an entry from the
+    CRONtab.
+
+    See :py:mod:`munininfluxdb.external.cron` for an example of an adapter.
+    """
+    def fun(args):
+        """
+        Main function for the "cron uninstall" command.
+
+        :param args: The result from parsing CLI arguments.
+        """
+        if os.geteuid() != 0:
+            print("It seems you are not root, please run \"muninflux cron uninstall\" again with root privileges")
+            sys.exit(1)
+
+        user = args.user or get_cron_user()
+        nb = cron_adapter.remove_by_comment(user, CRON_COMMENT)
+
+        if nb:
+            print("{0} Cron job uninstalled for user {1} ({2} entries deleted)".format(Symbol.OK_GREEN, user, nb))
+        else:
+            print("No matching job found (searching comment \"{1}\" in crontab for user {2})".format(Symbol.WARN_YELLOW,
+                                                                                                 CRON_COMMENT, user))
+    return fun
+
+
+def install_cron(cron_adapter):
+    """
+    Creates a function which uses *cron_adapter* to add an entry to the CRONtab.
+
+    See :py:mod:`munininfluxdb.external.cron` for an example of an adapter.
+    """
+    def fun(args):
+        """
+        Main function for the "cron install" command.
+
+        :param args: The result from parsing CLI arguments.
+        :return: Whether the operation was successful or not.
+        :rtype: bool
+        """
+        script_path = absolute_executable()
+        cmd = '%s fetch' % script_path
+
+        if os.geteuid() != 0:
+            print("It seems you are not root, please run \"%s cron install\" again with root privileges")
+            sys.exit(1)
+
+        user = args.user or get_cron_user()
+        success = cron_adapter.add_with_comment(user, cmd, args.period, CRON_COMMENT)
+
+        print("{0} Cron job installed for user {1}".format(Symbol.OK_GREEN, user))
+        return success
+    return fun
+
+
+def setup(parser, injections):
+    """
+    Sets up CLI argument parsing.
+
+    The argument *injections* should be a dictionary containing a key 'cron'
+    mapping to a cron adapter. For an example cron adapter see
+    ``munininfluxdb/external/cron.py``
+
+    :param parser: The argument parser for this subcommand.
+    :param injections: A dictionary containing the key ``'cron'`` mapping to an
+        implementation of a CRON adapter. See
+        :py:mod:`munininfluxdb.external.cron` for an example.
+    """
+    parser.add_argument('-u', '--user', default='', metavar='USER',
+                        help='The CRON user')
+
+    subparsers = parser.add_subparsers(title='CRON commands')
+    install_parser = subparsers.add_parser(
+        'install', description='Installs the CRON job')
+    uninstall_parser = subparsers.add_parser(
+        'uninstall', description='Uninstalls the CRON job')
+
+    install_parser.add_argument(
+        '-p', '--period', default=5, type=int,
+        help="sets the period in minutes between each fetch in the cron job (default: %(default)dmin)")
+
+    cron_adapter = injections['cron']
+    install_parser.set_defaults(func=install_cron(cron_adapter))
+    uninstall_parser.set_defaults(func=uninstall_cron(cron_adapter))

munininfluxdb/commands/dump.py

@@ -0,0 +1,80 @@
+import logging
+
+from munininfluxdb import munin
+from munininfluxdb import rrd
+from munininfluxdb.settings import Settings, Defaults
+from munininfluxdb.utils import Symbol
+
+
+LOG = logging.getLogger(__name__)
+NAME = 'dump'
+DESCRIPTION = """
+The 'dump' command writes out the munin RRD files to XML. These XML files can
+then be used by the 'load' command to import them into influxdb.
+"""
+
+
+def retrieve_munin_configuration(settings):
+    """
+    """
+    print("Exploring Munin structure")
+
+    try:
+        settings = munin.discover_from_datafile(settings)
+    except Exception as e:
+        LOG.debug('Traceback:', exc_info=True)
+        print("  {0} Could not process datafile ({1}), will read www and RRD cache instead".format(Symbol.NOK_RED, settings.paths['datafile']))
+
+        # read /var/cache/munin/www to check what's currently displayed on the dashboard
+        settings = munin.discover_from_www(settings)
+        settings = rrd.discover_from_rrd(settings, insert_missing=False)
+    else:
+        print("  {0} Found {1}: extracted {2} measurement units".format(Symbol.OK_GREEN, settings.paths['datafile'],
+                                                                        settings.nb_fields))
+
+    # for each host, find the /var/lib/munin/<host> directory and check if node name and plugin conf match RRD files
+    try:
+        rrd.check_rrd_files(settings)
+    except Exception as e:
+        print("  {0} {1}".format(Symbol.NOK_RED, e))
+    else:
+        print("  {0} Found {1} RRD files".format(Symbol.OK_GREEN, settings.nb_rrd_files))
+
+    return settings
+
+
+def main(args):
+    settings = Settings(args)
+    settings = retrieve_munin_configuration(settings)
+
+    # export RRD files as XML for (much) easier parsing (but takes much more time)
+    print("\nExporting RRD databases:".format(settings.nb_rrd_files))
+    nb_xml = rrd.export_to_xml(settings)
+    print("  {0} Exported {1} RRD files to XML ({2})".format(Symbol.OK_GREEN, nb_xml, settings.paths['xml']))
+
+
+def setup(parser, injections):
+    """
+    Sets up CLI argument parsing.
+
+    The argument *injections* is currently unused in this command and is a
+    placeholder for the future.
+
+    :param parser: The argument parser for this subcommand.
+    """
+    parser.add_argument('--xml-temp-path', default=Defaults.MUNIN_XML_FOLDER,
+                        help='set path where to store result of RRD exported files (default: %(default)s)')
+    parser.add_argument('--keep-temp', action='store_true',
+                        help='instruct to retain temporary files (mostly RRD\'s XML) after generation')
+    parser.add_argument('-v', '--verbose', type=int, default=1,
+                        help='set verbosity level (0: quiet, 1: default, 2: debug)')
+
+    # Munin
+    munargs = parser.add_argument_group('Munin parameters')
+    munargs.add_argument('--munin-path', default=Defaults.MUNIN_VAR_FOLDER,
+                         help='path to main Munin folder (default: %(default)s)')
+    munargs.add_argument('--www', '--munin-www-path', default=Defaults.MUNIN_WWW_FOLDER,
+                         help='path to main Munin folder (default: %(default)s)')
+    munargs.add_argument('--rrd', '--munin-rrd-path', default=Defaults.MUNIN_RRD_FOLDER,
+                         help='path to main Munin folder (default: %(default)s)')
+    parser.set_defaults(func=main)

bin/fetch.py → munininfluxdb/commands/fetch.py

@@ -1,31 +1,22 @@
 #!/usr/bin/env python
 from __future__ import print_function
-import pwd
 import json
 import os
 import sys
-import argparse
 from collections import defaultdict
 
 from munininfluxdb.utils import Symbol
 from munininfluxdb.settings import Defaults
 
 import influxdb
+import storable
 
-try:
-    import storable
-except ImportError:
-    from vendor import storable
+NAME = 'fetch'
+DESCRIPTION = """'fetch' command grabs fresh data gathered by a still running Munin installation and send it to InfluxDB.
 
-try:
-    pwd.getpwnam('munin')
-except KeyError:
-    CRON_USER = 'root'
-else:
-    CRON_USER = 'munin'
+Currently, Munin needs to be still running to update the data in '/var/lib/munin/state-*' files.
+"""
 
-# Cron job comment is used to uninstall and must not be manually deleted from the crontab
-CRON_COMMENT = 'Update InfluxDB with fresh values from Munin'
 
 def pack_values(config, values):
     suffix = ":{0}".format(Defaults.DEFAULT_RRD_INDEX)
@@ -66,7 +57,9 @@ def read_state_file(filename):
     assert 'spoolfetch' in data and 'value' in data
     return data['value'], data['spoolfetch']
 
-def main(config_filename=Defaults.FETCH_CONFIG):
+def main(args):
+    config_filename = args.config or Defaults.FETCH_CONFIG
+
     config = None
     with open(config_filename) as f:
         config = json.load(f)
@@ -113,68 +106,16 @@ def main(config_filename=Defaults.FETCH_CONFIG):
         json.dump(config, f)
         print("{0} Updated configuration: {1}".format(Symbol.OK_GREEN, f.name))
 
-def uninstall_cron():
-    if os.geteuid() != 0:
-        print("It seems you are not root, please run \"muninflux fetch --uninstall-cron\" again with root privileges".format(sys.argv[0]))
-        sys.exit(1)
-
-    try:
-        import crontab
-    except ImportError:
-        from vendor import crontab
-
-    cron = crontab.CronTab(user=CRON_USER)
-    jobs = list(cron.find_comment(CRON_COMMENT))
-    cron.remove(*jobs)
-    cron.write()
-
-    return len(jobs)
-
-def install_cron(script_file, period):
-    if os.geteuid() != 0:
-        print("It seems you are not root, please run \"muninflux fetch --install-cron\" again with root privileges".format(sys.argv[0]))
-        sys.exit(1)
-
-    try:
-        import crontab
-    except ImportError:
-        from vendor import crontab
-
-    cron = crontab.CronTab(user=CRON_USER)
-    job = cron.new(command=script_file, user=CRON_USER, comment=CRON_COMMENT)
-    job.minute.every(period)
 
-    if job.is_valid() and job.is_enabled():
-        cron.write()
+def setup(parser, injections):
+    """
+    Sets up CLI argument parsing.
 
-    return job.is_valid() and job.is_enabled()
+    The argument *injections* is currently unused in this command and is a
+    placeholder for the future.
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="""
-    'fetch' command grabs fresh data gathered by a still running Munin installation and send it to InfluxDB.
-
-    Currently, Munin needs to be still running to update the data in '/var/lib/munin/state-*' files.
-    """)
+    :param parser: The argument parser for this subcommand.
+    """
     parser.add_argument('--config', default=Defaults.FETCH_CONFIG,
                         help='overrides the default configuration file (default: %(default)s)')
-    cronargs = parser.add_argument_group('cron job management')
-    cronargs.add_argument('--install-cron', dest='script_path',
-                        help='install a cron job to updated InfluxDB with fresh data from Munin every <period> minutes')
-    cronargs.add_argument('-p', '--period', default=5, type=int,
-                        help="sets the period in minutes between each fetch in the cron job (default: %(default)min)")
-    cronargs.add_argument('--uninstall-cron', action='store_true',
-                        help='uninstall the fetch cron job (any matching the initial comment actually)')
-    args = parser.parse_args()
-
-    if args.script_path:
-        install_cron(args.script_path, args.period)
-        print("{0} Cron job installed for user {1}".format(Symbol.OK_GREEN, CRON_USER))
-    elif args.uninstall_cron:
-        nb = uninstall_cron()
-        if nb:
-            print("{0} Cron job uninstalled for user {1} ({2} entries deleted)".format(Symbol.OK_GREEN, CRON_USER, nb))
-        else:
-            print("No matching job found (searching comment \"{1}\" in crontab for user {2})".format(Symbol.WARN_YELLOW,
-                                                                                                     CRON_COMMENT, CRON_USER))
-    else:
-        main(args.config)
+    parser.set_defaults(func=main)