Enable email-comments

.gitignore

@@ -52,3 +52,4 @@ buildout_current.cfg
 .sass-cache
 src/adhocracy/static/stylesheets/adhocracy.css
 src/adhocracy/tests/loadtests/data  
+/.project

buildouts/adhocracy.cfg

@@ -24,6 +24,10 @@ auto-checkout = *
 # supervisor config to start adhocracy 
 adhocracy_worker-supervisor = 
     40 adhocracy_worker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_worker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy worker -c ${buildout:directory}/etc/adhocracy.ini]
+
+adhocracy_ecworker-supervisor = 
+    40 adhocracy_ecworker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_ecworker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy ecworker -c ${buildout:directory}/etc/adhocracy.ini]
+
 adhocracy-supervisor =
     45 adhocracy (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [serve ${buildout:directory}/etc/adhocracy.ini]
 # parts in this buildout file to be installed
@@ -170,4 +174,5 @@ scripts = adhocpy
 environment = LD_LIBRARY_PATH="${buildout:directory}/python/python-2.7/lib/"
 programs +=
     ${buildout:adhocracy_worker-supervisor}
+    ${buildout:adhocracy_ecworker-supervisor}
     ${buildout:adhocracy-supervisor}

etc/adhocracy.ini.in

@@ -102,6 +102,17 @@ use = egg:adhocracy
 full_stack = true
 static_files = true
 
+# INSTALL: To enable set email_src to 'local' or 'imap', IMAP-SSL is essential.
+# If 'imap_account = user' fails, try complete address: '[email protected]'.
+# Set local_user to determine the mbox/Maildir-folders you want to watch.
+email_src = none
+local_user = user
+imap_domain = imap.emailprovider.tld
+imap_account = user
+imap_password = password
+imap_directory = inbox
+imap_port = 993
+
 cache_dir = ${parts.buildout.directory}/var/data
 beaker.session.key = adhocracy_state
 {% python
@@ -465,4 +476,4 @@ format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
 # Custom OVERRIDE adhocracy settings 
 ############################################################## 
 
-${parts.adhocracy['settings_override']} 
+${parts.adhocracy['settings_override']}

etc/test.ini.in

@@ -57,7 +57,13 @@ beaker.cache.data_dir = ${parts.buildout.directory}/var/testdata/cache
 beaker.session.data_dir = ${parts.buildout.directory}/var/testdata/sessions
 
 # Turn on all the bells and whistles
-adhocracy.crypto.secret = geheim!
+{% python
+    import random;
+    def randomhash(length):
+        return hex(random.SystemRandom().getrandbits(length))
+%}
+
+adhocracy.crypto.secret = {% if parts.adhocracy.secret == 'autogenerated' %}${randomhash(256)}{% end %}{% if parts.adhocracy.secret != 'autogenerated' %}${parts.adhocracy.secret}{% end %}
 adhocracy.track_outgoing_links = True
 
 adhocracy.login_type = openid,username+password,email+password,shibboleth

setup.py

@@ -58,8 +58,7 @@
         "python-openid>=2.2.4",
         "python-memcached>=1.45",
         "sunburnt==0.6",
-        #"Pillow", # use the adhocracy buildout or install system packages
-                   # (python-imaging) for this dependency
+        "Pillow",
         "Markdown>=2.3",
         "lxml>=2.2.6",
         "Mako>=0.7.3",
@@ -74,6 +73,8 @@
         "setuptools_git >= 0.3",
         "ipaddress>=1.0.3",
         "pytz",
+        "pyinotify>=0.9.4",
+        "imaplib2>=2.28.1",
     ],
     setup_requires=["setuptools>=0.6c6",  # fix OS X 10.5.7
                     "PasteScript",
@@ -93,7 +94,7 @@
                  'pytest-pep8',
                  'cssselect',
                  'decorator',
-                 'pep8']
+                 'pep8',]
     },
     package_data={'adhocracy': ['i18n/*/LC_MESSAGES/*.mo'],
                   '': ['RELEASE-VERSION'],
@@ -111,6 +112,7 @@
         ],
         'paste.paster_command': [
             'worker = adhocracy.lib.cli:Worker',
+            'ecworker = adhocracy.lib.cli:EmailCommentWorker',
             'timer = adhocracy.lib.cli:Timer',
             'index = adhocracy.lib.cli:Index'
         ],
@@ -119,6 +121,7 @@
         ],
         'fanstatic.libraries': [
             'stylesheets = adhocracy.static:stylesheets_library',
+            'yaml = adhocracy.static:yaml_library',
             'autocomplete = adhocracy.static:autocomplete_library',
             'placeholder = adhocracy.static:placeholder_library',
             'jquerytools = adhocracy.static:jquerytools_library',

src/adhocracy/controllers/comment.py

@@ -104,35 +104,21 @@ def create(self, format='html'):
 
         topic = self.form_result.get('topic')
         reply = self.form_result.get('reply')
-
-        if reply:
-            require.comment.reply(reply)
-        else:
-            require.comment.create_on(topic)
-
         variant = self.form_result.get('variant')
+        wiki = self.form_result.get('wiki')
+        sentiment = self.form_result.get('sentiment')
+        text = self.form_result.get('text')
+
         if hasattr(topic, 'variants') and not variant in topic.variants:
             return ret_abort(_("Comment topic has no variant %s") % variant,
                              code=400)
 
-        comment = model.Comment.create(
-            self.form_result.get('text'),
-            c.user, topic,
-            reply=reply,
-            wiki=self.form_result.get('wiki'),
-            variant=variant,
-            sentiment=self.form_result.get('sentiment'),
-            with_vote=can.user.vote())
-
-        # watch comments by default!
-        model.Watch.create(c.user, comment)
-        model.meta.Session.commit()
-        #watchlist.check_watch(comment)
-        event.emit(event.T_COMMENT_CREATE, c.user, instance=c.instance,
-                   topics=[topic], comment=comment, topic=topic,
-                   rev=comment.latest)
+        comment = _create(c.user, text, reply, topic, variant,
+                wiki, c.instance, sentiment)
+
         if len(request.params.get('ret_url', '')):
             redirect(request.params.get('ret_url') + "#c" + str(comment.id))
+
         if format != 'html':
             return ret_success(entity=comment, format=format)
         return ret_success(entity=comment, format='fwd')
@@ -297,3 +283,39 @@ def reply_form(self, id):
         topic = parent.topic
         variant = getattr(topic, 'variant', None)
         return self._render_ajax_create_form(parent, topic, variant)
+
+
+@guard.comment.create()
+def _create(user, text, reply, topic, variant, wiki, instance, sentiment=None):
+    '''
+    extracted from controllers.commentCommentController for usage of
+    emailcomments
+    '''
+    with_vote = can.user.vote()
+
+    if reply:
+        require.comment.reply(reply)
+    else:
+        require.comment.create_on(topic)
+
+    comment = model.Comment.create(
+        text,
+        user, topic,
+        reply=reply,
+        wiki=wiki,
+        variant=variant,
+        sentiment=sentiment,
+        with_vote=with_vote)
+    # watch comments by default!
+
+    model.Watch.create(user, comment)
+
+    model.meta.Session.commit()
+
+    #watchlist.check_watch(comment)
+
+    event.emit(event.T_COMMENT_CREATE, user, instance=instance,
+            topics=[topic], comment=comment, topic=topic,
+            rev=comment.latest)
+
+    return comment

src/adhocracy/lib/cli.py

@@ -17,14 +17,16 @@
 from adhocracy.lib import search
 from adhocracy.lib import queue
 
+from adhocracy.lib.emailcomments import mail_watch
+
 
 log = getLogger(__name__)
 
 
 class AdhocracyCommand(Command):
     parser = Command.standard_parser(verbose=True)
     parser.add_option('-c', '--config', dest='config',
-                      default='etc/adhocracy.ini', help='Config file to use.')
+            default='etc/adhocracy.ini', help='Config file to use.')
     default_verbosity = 1
     group_name = 'adhocracy'
 
@@ -195,6 +197,18 @@ def command(self):
         worker.work()
 
 
+class EmailCommentWorker(AdhocracyCommand):
+    '''Run emailcomment background jobs.'''
+    summary = __doc__.split('\n')[0]
+    usage = __doc__
+    max_args = None
+    min_args = None
+
+    def command(self):
+        self._load_config()
+        mail_watch()
+
+
 class Index(AdhocracyCommand):
     """Re-create Adhocracy's search index."""
     summary = __doc__.split('\n')[0]
@@ -278,11 +292,11 @@ def start(self, actions, classes, instances):
         print ('Starting.\n'
                '  Actions: %s\n'
                '  Content Types: %s\n'
-               '  Instances: %s\n' % (
+               '  Instances: %s\n') % (
                    self.printable(actions),
                    self.printable(classes,
                                   print_=lambda x: x.__name__.lower()),
-                   self.printable(instances, print_=lambda x: x.key)))
+                   self.printable(instances, print_=lambda x: x.key))
 
         if self.DROP in actions:
             p_instances = instances if instances else [None]
@@ -313,8 +327,8 @@ def usage(self):
         indexed_classes = sorted(self.indexed_classes.keys())
         content_types = '\n          '.join(indexed_classes)
         usage += (
-            'index (INDEX|DROP|DROP_ALL|ALL) [<entity>, ...] [-I <instance>, '
-            '...] -c <inifile>'
+            'index (INDEX|DROP|DROP_ALL|ALL) [<entity>, ...] [-I <instance>, ...]'
+            ' -c <inifile>'
             '\n\n'
             '  DROP_ALL:\n'
             '      Remove all documents from solr.\n'

src/adhocracy/lib/emailcomments/__init__.py

@@ -0,0 +1,58 @@
+import os
+from pylons import config
+import getpass
+import logging
+import Queue
+from sqlalchemy.orm import sessionmaker, scoped_session
+from adhocracy.model import meta
+
+log = logging.getLogger(__name__)
+
+
+def mail_watch():
+    '''
+    This is a script executed by the adhocracy emailcommentworker
+    Its target is to listen to new mails (Maildir and mbox or IMAP)
+    and identify if they are comment replies. If so the jobs
+    will be executed in a queue.
+    '''
+    esrc = config.get("email_src")
+
+    if esrc == "imap" or esrc == "local":
+        ecq = Queue.Queue()
+
+        Session = sessionmaker(bind=meta.engine, autoflush=True)
+        meta.Session = scoped_session(Session)
+
+        from adhocracy.lib.emailcomments import parseincoming
+        if esrc == "imap":
+            from adhocracy.lib.emailcomments import imap
+            idler = imap.Idler(ecq)
+            idler.start()
+        elif esrc == "local":
+            from adhocracy.lib.emailcomments import localwatch
+            username = config.get("local_user")
+            if not username:
+                log.error("no user set in config")
+                log.info("emailcomments are disabled")
+                return
+            if username:
+                path_md = os.path.join("/home", username, "Maildir")
+                path_mb = os.path.join("/var/mail", username)
+                if os.path.exists("/usr/include/linux/inotify.h"):
+                    if util.create_filesystem(path_md):
+                        localwatch.watch_new_mail(path_md, path_mb, ecq)
+                    else:
+                        return
+                else:
+                    log.error("kernel module inotify is not installed")
+                    log.info("emailcomments are disabled")
+                    return
+            else:
+                log.error("user cannot be determined")
+                log.info("emailcomments are disabled")
+                return
+
+        while True:
+            message = ecq.get()
+            parseincoming.handle_inc_mail(message)

src/adhocracy/lib/emailcomments/imap.py

@@ -0,0 +1,66 @@
+import imaplib2
+import email
+from email.parser import HeaderParser
+import os
+import logging
+import threading
+import time
+from pylons import config
+from adhocracy.lib.emailcomments import util
+from adhocracy.lib import queue
+
+# cfg
+IMAP_DOMAIN = config.get('imap_domain')
+IMAP_USERNAME = config.get('imap_account')
+IMAP_PASSWORD = config.get('imap_password')
+COMMENTS_DIR = config.get('imap_directory')
+IMAP_PORT = config.get('imap_port')
+MAX_INTERVAL = 60 * 64  # max. reconnect-interval (seconds - 60 * 2^n possible)
+
+log = logging.getLogger(__name__)
+
+
+class Idler(threading.Thread):
+
+    def __init__(self, ecq):
+        threading.Thread.__init__(self)
+        self.ecq = ecq
+
+    def run(self):
+        '''(re)connects to IMAP and waits for new mail'''
+        recon_interval = 30
+        while True:
+            try:
+                self.conn = imaplib2.IMAP4_SSL(IMAP_DOMAIN, IMAP_PORT)
+                self.conn.login(IMAP_USERNAME, IMAP_PASSWORD)
+                self.conn.select(COMMENTS_DIR)
+            except Exception as e:
+                log.info("IMAP-connection could not be established because:")
+                log.info(e)
+                if recon_interval < MAX_INTERVAL:
+                    recon_interval = recon_interval * 2
+                minutes = recon_interval / 60
+                log.info("reconnect in {0} minute(s)".format(minutes))
+                time.sleep(recon_interval)
+                continue
+
+            recon_interval = 30
+
+            while True:
+                try:
+                    self.dosync()
+                    self.conn.idle()
+                except self.conn.abort:
+                    log.info("IMAP-connection lost, reconnecting...")
+
+    def dosync(self):
+        '''executed if a new mail arrives'''
+        typ, data = self.conn.search(None, 'UNSEEN')
+        if data[0]:
+            for num in data[0].split():
+                typ, data = self.conn.fetch(num, '(RFC822)')
+                header_data = data[0][1]
+                parser = HeaderParser()
+                message = parser.parsestr(header_data)
+                log.info("new IMAP mail")
+                self.ecq.put(message)