diff --git a/.gitignore b/.gitignore index 59ef84693..13197dc5c 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ buildout_current.cfg .sass-cache src/adhocracy/static/stylesheets/adhocracy.css src/adhocracy/tests/loadtests/data +/.project diff --git a/buildouts/adhocracy.cfg b/buildouts/adhocracy.cfg index fb48bcf6e..bef4c54bd 100644 --- a/buildouts/adhocracy.cfg +++ b/buildouts/adhocracy.cfg @@ -24,6 +24,10 @@ auto-checkout = * # supervisor config to start adhocracy adhocracy_worker-supervisor = 40 adhocracy_worker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_worker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy worker -c ${buildout:directory}/etc/adhocracy.ini] + +adhocracy_ecworker-supervisor = + 40 adhocracy_ecworker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_ecworker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy ecworker -c ${buildout:directory}/etc/adhocracy.ini] + adhocracy-supervisor = 45 adhocracy (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [serve ${buildout:directory}/etc/adhocracy.ini] # parts in this buildout file to be installed @@ -170,4 +174,5 @@ scripts = adhocpy environment = LD_LIBRARY_PATH="${buildout:directory}/python/python-2.7/lib/" programs += ${buildout:adhocracy_worker-supervisor} + ${buildout:adhocracy_ecworker-supervisor} ${buildout:adhocracy-supervisor} diff --git a/etc/adhocracy.ini.in b/etc/adhocracy.ini.in index 7e3efef02..daf71a5b2 100644 --- a/etc/adhocracy.ini.in +++ b/etc/adhocracy.ini.in @@ -102,6 +102,17 @@ use = egg:adhocracy full_stack = true static_files = true +# INSTALL: To enable set email_src to 'local' or 'imap', IMAP-SSL is essential. +# If 'imap_account = user' fails, try complete address: 'user@emailprovider.tld'. +# Set local_user to determine the mbox/Maildir-folders you want to watch. +email_src = none +local_user = user +imap_domain = imap.emailprovider.tld +imap_account = user +imap_password = password +imap_directory = inbox +imap_port = 993 + cache_dir = ${parts.buildout.directory}/var/data beaker.session.key = adhocracy_state {% python @@ -465,4 +476,4 @@ format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s # Custom OVERRIDE adhocracy settings ############################################################## -${parts.adhocracy['settings_override']} +${parts.adhocracy['settings_override']} diff --git a/etc/test.ini.in b/etc/test.ini.in index b5c4df5ef..dbff89a7b 100644 --- a/etc/test.ini.in +++ b/etc/test.ini.in @@ -57,7 +57,13 @@ beaker.cache.data_dir = ${parts.buildout.directory}/var/testdata/cache beaker.session.data_dir = ${parts.buildout.directory}/var/testdata/sessions # Turn on all the bells and whistles -adhocracy.crypto.secret = geheim! +{% python + import random; + def randomhash(length): + return hex(random.SystemRandom().getrandbits(length)) +%} + +adhocracy.crypto.secret = {% if parts.adhocracy.secret == 'autogenerated' %}${randomhash(256)}{% end %}{% if parts.adhocracy.secret != 'autogenerated' %}${parts.adhocracy.secret}{% end %} adhocracy.track_outgoing_links = True adhocracy.login_type = openid,username+password,email+password,shibboleth diff --git a/python/buildout.python b/python/buildout.python deleted file mode 160000 index 44b473393..000000000 --- a/python/buildout.python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 44b4733937827f0765419ff935100aa385341dd3 diff --git a/setup.py b/setup.py index a34eac64c..c4d9bb5b3 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,7 @@ "python-openid>=2.2.4", "python-memcached>=1.45", "sunburnt==0.6", - #"Pillow", # use the adhocracy buildout or install system packages - # (python-imaging) for this dependency + "Pillow", "Markdown>=2.3", "lxml>=2.2.6", "Mako>=0.7.3", @@ -74,6 +73,8 @@ "setuptools_git >= 0.3", "ipaddress>=1.0.3", "pytz", + "pyinotify>=0.9.4", + "imaplib2>=2.28.1", ], setup_requires=["setuptools>=0.6c6", # fix OS X 10.5.7 "PasteScript", @@ -93,7 +94,7 @@ 'pytest-pep8', 'cssselect', 'decorator', - 'pep8'] + 'pep8',] }, package_data={'adhocracy': ['i18n/*/LC_MESSAGES/*.mo'], '': ['RELEASE-VERSION'], @@ -111,6 +112,7 @@ ], 'paste.paster_command': [ 'worker = adhocracy.lib.cli:Worker', + 'ecworker = adhocracy.lib.cli:EmailCommentWorker', 'timer = adhocracy.lib.cli:Timer', 'index = adhocracy.lib.cli:Index' ], @@ -119,6 +121,7 @@ ], 'fanstatic.libraries': [ 'stylesheets = adhocracy.static:stylesheets_library', + 'yaml = adhocracy.static:yaml_library', 'autocomplete = adhocracy.static:autocomplete_library', 'placeholder = adhocracy.static:placeholder_library', 'jquerytools = adhocracy.static:jquerytools_library', diff --git a/src/adhocracy/controllers/comment.py b/src/adhocracy/controllers/comment.py index abf908126..21711bcbc 100644 --- a/src/adhocracy/controllers/comment.py +++ b/src/adhocracy/controllers/comment.py @@ -104,35 +104,21 @@ def create(self, format='html'): topic = self.form_result.get('topic') reply = self.form_result.get('reply') - - if reply: - require.comment.reply(reply) - else: - require.comment.create_on(topic) - variant = self.form_result.get('variant') + wiki = self.form_result.get('wiki') + sentiment = self.form_result.get('sentiment') + text = self.form_result.get('text') + if hasattr(topic, 'variants') and not variant in topic.variants: return ret_abort(_("Comment topic has no variant %s") % variant, code=400) - comment = model.Comment.create( - self.form_result.get('text'), - c.user, topic, - reply=reply, - wiki=self.form_result.get('wiki'), - variant=variant, - sentiment=self.form_result.get('sentiment'), - with_vote=can.user.vote()) - - # watch comments by default! - model.Watch.create(c.user, comment) - model.meta.Session.commit() - #watchlist.check_watch(comment) - event.emit(event.T_COMMENT_CREATE, c.user, instance=c.instance, - topics=[topic], comment=comment, topic=topic, - rev=comment.latest) + comment = _create(c.user, text, reply, topic, variant, + wiki, c.instance, sentiment) + if len(request.params.get('ret_url', '')): redirect(request.params.get('ret_url') + "#c" + str(comment.id)) + if format != 'html': return ret_success(entity=comment, format=format) return ret_success(entity=comment, format='fwd') @@ -297,3 +283,39 @@ def reply_form(self, id): topic = parent.topic variant = getattr(topic, 'variant', None) return self._render_ajax_create_form(parent, topic, variant) + + +@guard.comment.create() +def _create(user, text, reply, topic, variant, wiki, instance, sentiment=None): + ''' + extracted from controllers.commentCommentController for usage of + emailcomments + ''' + with_vote = can.user.vote() + + if reply: + require.comment.reply(reply) + else: + require.comment.create_on(topic) + + comment = model.Comment.create( + text, + user, topic, + reply=reply, + wiki=wiki, + variant=variant, + sentiment=sentiment, + with_vote=with_vote) + # watch comments by default! + + model.Watch.create(user, comment) + + model.meta.Session.commit() + + #watchlist.check_watch(comment) + + event.emit(event.T_COMMENT_CREATE, user, instance=instance, + topics=[topic], comment=comment, topic=topic, + rev=comment.latest) + + return comment diff --git a/src/adhocracy/lib/cli.py b/src/adhocracy/lib/cli.py index 496cc9595..f0ee4b125 100644 --- a/src/adhocracy/lib/cli.py +++ b/src/adhocracy/lib/cli.py @@ -17,6 +17,8 @@ from adhocracy.lib import search from adhocracy.lib import queue +from adhocracy.lib.emailcomments import mail_watch + log = getLogger(__name__) @@ -24,7 +26,7 @@ class AdhocracyCommand(Command): parser = Command.standard_parser(verbose=True) parser.add_option('-c', '--config', dest='config', - default='etc/adhocracy.ini', help='Config file to use.') + default='etc/adhocracy.ini', help='Config file to use.') default_verbosity = 1 group_name = 'adhocracy' @@ -195,6 +197,18 @@ def command(self): worker.work() +class EmailCommentWorker(AdhocracyCommand): + '''Run emailcomment background jobs.''' + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = None + min_args = None + + def command(self): + self._load_config() + mail_watch() + + class Index(AdhocracyCommand): """Re-create Adhocracy's search index.""" summary = __doc__.split('\n')[0] @@ -278,11 +292,11 @@ def start(self, actions, classes, instances): print ('Starting.\n' ' Actions: %s\n' ' Content Types: %s\n' - ' Instances: %s\n' % ( + ' Instances: %s\n') % ( self.printable(actions), self.printable(classes, print_=lambda x: x.__name__.lower()), - self.printable(instances, print_=lambda x: x.key))) + self.printable(instances, print_=lambda x: x.key)) if self.DROP in actions: p_instances = instances if instances else [None] @@ -313,8 +327,8 @@ def usage(self): indexed_classes = sorted(self.indexed_classes.keys()) content_types = '\n '.join(indexed_classes) usage += ( - 'index (INDEX|DROP|DROP_ALL|ALL) [, ...] [-I , ' - '...] -c ' + 'index (INDEX|DROP|DROP_ALL|ALL) [, ...] [-I , ...]' + ' -c ' '\n\n' ' DROP_ALL:\n' ' Remove all documents from solr.\n' diff --git a/src/adhocracy/lib/emailcomments/__init__.py b/src/adhocracy/lib/emailcomments/__init__.py new file mode 100644 index 000000000..9029ff444 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/__init__.py @@ -0,0 +1,58 @@ +import os +from pylons import config +import getpass +import logging +import Queue +from sqlalchemy.orm import sessionmaker, scoped_session +from adhocracy.model import meta + +log = logging.getLogger(__name__) + + +def mail_watch(): + ''' + This is a script executed by the adhocracy emailcommentworker + Its target is to listen to new mails (Maildir and mbox or IMAP) + and identify if they are comment replies. If so the jobs + will be executed in a queue. + ''' + esrc = config.get("email_src") + + if esrc == "imap" or esrc == "local": + ecq = Queue.Queue() + + Session = sessionmaker(bind=meta.engine, autoflush=True) + meta.Session = scoped_session(Session) + + from adhocracy.lib.emailcomments import parseincoming + if esrc == "imap": + from adhocracy.lib.emailcomments import imap + idler = imap.Idler(ecq) + idler.start() + elif esrc == "local": + from adhocracy.lib.emailcomments import localwatch + username = config.get("local_user") + if not username: + log.error("no user set in config") + log.info("emailcomments are disabled") + return + if username: + path_md = os.path.join("/home", username, "Maildir") + path_mb = os.path.join("/var/mail", username) + if os.path.exists("/usr/include/linux/inotify.h"): + if util.create_filesystem(path_md): + localwatch.watch_new_mail(path_md, path_mb, ecq) + else: + return + else: + log.error("kernel module inotify is not installed") + log.info("emailcomments are disabled") + return + else: + log.error("user cannot be determined") + log.info("emailcomments are disabled") + return + + while True: + message = ecq.get() + parseincoming.handle_inc_mail(message) diff --git a/src/adhocracy/lib/emailcomments/imap.py b/src/adhocracy/lib/emailcomments/imap.py new file mode 100644 index 000000000..fa2668657 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/imap.py @@ -0,0 +1,68 @@ +import imaplib2 +import email +from email.parser import HeaderParser +import os +import logging +import threading +import time +from pylons import config +from adhocracy.lib.emailcomments import util +from adhocracy.lib import queue + +# cfg +IMAP_DOMAIN = config.get('imap_domain') +IMAP_USERNAME = config.get('imap_account') +IMAP_PASSWORD = config.get('imap_password') +COMMENTS_DIR = config.get('imap_directory') +IMAP_PORT = config.get('imap_port') +MAX_INTERVAL = 60 * 64 # max. reconnect-interval (seconds - 60 * 2^n possible) + +log = logging.getLogger(__name__) + + +class Idler(threading.Thread): + + def __init__(self, ecq): + threading.Thread.__init__(self) + self.ecq = ecq + + def run(self): + '''(re)connects to IMAP and waits for new mail''' + recon_interval = 30 + while True: + try: + self.conn = imaplib2.IMAP4_SSL(IMAP_DOMAIN, IMAP_PORT) + self.conn.login(IMAP_USERNAME, IMAP_PASSWORD) + self.conn.select(COMMENTS_DIR) + except Exception as e: + log.info("IMAP-connection could not be established because:") + log.info(e) + if recon_interval < MAX_INTERVAL: + recon_interval = recon_interval * 2 + minutes = recon_interval / 60 + log.info("reconnect in {0} minute(s)".format(minutes)) + time.sleep(recon_interval) + continue + + recon_interval = 30 + log.info("IMAP-connection established") + + while True: + try: + self.dosync() + self.conn.idle() + except self.conn.abort: + log.info("IMAP-connection lost, reconnecting...") + break + + def dosync(self): + '''executed if a new mail arrives''' + typ, data = self.conn.search(None, 'UNSEEN') + if data[0]: + for num in data[0].split(): + typ, data = self.conn.fetch(num, '(RFC822)') + header_data = data[0][1] + parser = HeaderParser() + message = parser.parsestr(header_data) + log.info("new IMAP mail") + self.ecq.put(message) diff --git a/src/adhocracy/lib/emailcomments/localwatch.py b/src/adhocracy/lib/emailcomments/localwatch.py new file mode 100644 index 000000000..6dc0dcfa0 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/localwatch.py @@ -0,0 +1,74 @@ +import pyinotify +import logging +import os +import time +from datetime import datetime +import mailbox +import email +import shutil +from adhocracy.lib.emailcomments import util + +log = logging.getLogger(__name__) + + +def maildir(path, ecq): + '''gets payload and recipient of a new Maildir mail''' + time.sleep(1) # pause - else mail isn't yet in tmp + log.info("new mail in Maildir") + + for filename in os.listdir(os.path.join(path, "new")): + file_path = os.path.join(path, "new", filename) + mail = open(file_path, "r") + parser = email.Parser.Parser() + message = parser.parse(mail) + mail.close() + ecq.put(message) + util.move_overwrite(file_path, + os.path.join(path, "cur", filename)) + + +def mbox(path, ecq): + '''gets payload and recipient of a new mbox mail''' + time.sleep(1) # pause - MTA could do something yet + log.info("new mail in mbox") + + tmp_mb = os.path.join("/var/tmp", str(datetime.now())) + shutil.copy(path, tmp_mb) + + mbox = mailbox.mbox(tmp_mb) + + # process latest email + try: + message = mbox.get_message(mbox.keys()[-1]) + except Exception as e: + log.error("mbox empty, or header malformed:") + log.error(e) + os.remove(tmp_mb) + return + ecq.put(message) + os.remove(tmp_mb) + + +def watch_new_mail(path_md, path_mb, ecq): + ''' + Uses pyinotify to watch for new local emails. + If a new email arrives it will handled by the ec-Queue. + ''' + wm = pyinotify.WatchManager() + mask_md = pyinotify.IN_CREATE # watch for new files in maildir + mask_mb = pyinotify.IN_CLOSE_WRITE # watch for changed mbox + + class PTmp(pyinotify.ProcessEvent): + + def process_IN_CREATE(self, event): + maildir(path_md, ecq) + + def process_IN_CLOSE_WRITE(self, event): + mbox(path_mb, ecq) + + notifier = pyinotify.ThreadedNotifier(wm, PTmp()) + notifier.start() + + wdd = wm.add_watch(path_md, mask_md, rec=True) + wm.add_watch(path_mb, mask_mb, rec=False) + wm.rm_watch(wdd[os.path.join(path_md, "cur")], rec=True) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py new file mode 100644 index 000000000..9317f96ea --- /dev/null +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -0,0 +1,166 @@ +import logging +import hashlib +import re +from adhocracy.model import User, Comment +from adhocracy.lib.emailcomments import setupcomment +from adhocracy.lib.emailcomments import util +from pylons import config + + +log = logging.getLogger(__name__) + + +def parse_multipart(message, a, rec_dep): + '''saves contents of multipart messages to a list''' + content_list = [] + for part in message.get_payload(): + if not part["Content-Type"]: + continue + if part.is_multipart(): + if "multipart/alternative" in part["Content-Type"]: + content_list = parse_multipart(part, "a", rec_dep) + rec_dep += 1 + else: + content_list = parse_multipart(part, "na", rec_dep) + else: + if "image" in part["Content-Type"]: + content_list.append((rec_dep, a, part["Content-Type"], + part.get_payload(decode=False))) + else: + content_list.append((rec_dep, a, part["Content-Type"], + unicode(part.get_payload(decode=True), + encoding="utf8", errors="replace"))) + return content_list + + +def save_image(image, content_type): + ''' + Helper function for later use of saving attached images in database + and return markdown-code. + Markdown safe_mode must be set to False, else an image with + alt-text will be rendered but looks like plain text. + ''' + data_uri = image.replace("\n", "") + img_type, img_name = util.content_type_reader(content_type) + img_mkd = """ \n![{0}](data:image/{1};base64,{2} \"{0}\") \n""" + img_mkd = img_mkd.format(img_name, img_type, data_uri) + return img_mkd + + +def get_usable_content(content_list): + ''' + Extracts all readable contents of list and prefers HTML over plaintext if + a multipart/alternative content is present. + Images will be saved via data URL. + ''' + text = "" + alt_set = [] + image_list = [] + temp_alt_text = "" + for rec_dep, alt_version, content_type, payload in content_list: + if "na" in alt_version: + text = text + temp_alt_text + temp_alt_text = "" + if "text/plain" in content_type: + text = text + payload + elif "text/html" in content_type: + text = text + util.html_to_markdown(payload) + elif "image" in content_type: + image_list.append(save_image(payload, content_type)) + else: + if not rec_dep in alt_set: + if "text/plain" in content_type: + temp_alt_text = payload + elif "text/html" in content_type: + alt_set.append(rec_dep) + temp_alt_text = util.html_to_markdown(payload) + text = text + temp_alt_text + return text, image_list + + +def parse_payload(message): + ''' + Extracts Payload of emails. The payload of a message will be decoded + if the content-transfer-encoding is base64 or quoted-printable. + If other encodings (7bit/8bit, bogus base64) are detected no decoding + happens. + Multipart messages are supported; the payloads will be concatenated if + readable (html/plain text). + If a multipart message is "alternative" HTML will be prefered over plain + text. + ''' + if message.is_multipart(): + if "multipart/alternative" in message["Content-Type"]: + content_list = parse_multipart(message, "a", 0) + else: + content_list = parse_multipart(message, "na", 0) + text, image_list = get_usable_content(content_list) + text = util.remove_notification(text) + text = util.delete_signatures(text) + for item in image_list: + text = text + item + else: + if "text/plain" in message["Content-Type"]: + text = message.get_payload(decode=True) + elif "text/html" in message["Content-Type"]: + text = message.get_payload(decode=True) + text = util.html_to_markdown(text) + else: + text = "" + text = util.remove_notification(text) + text = util.delete_signatures(text) + text = util.delete_debris(text) + text, sentiment = util.get_sentiment(text) + return (text, sentiment) + + +def parse_local_part(recipient): + '''parse local part of email address to determine if it's a comment''' + result = util.strip_local_part(recipient) + if not result: + return None + + sec_token = result.group("sectoken") + + secrets = config.get("adhocracy.crypto.secret") + + comp_str = result.group("userid") + result.group("commentid") + comp_str = hashlib.sha1(comp_str + secrets).hexdigest() + + if comp_str != sec_token: + return None + + user_obj = User.find(result.group("userid")) + comment_obj = Comment.find(result.group("commentid")) + + if user_obj is None or comment_obj is None: + return None + return (user_obj, comment_obj) + + +def handle_inc_mail(message): + '''Check if incoming email is a comment - if so call commentcreate.''' + recipient = re.sub(r"(.|\n|\r|\r\n)*?<|>", u"", message["To"]) + + objs = parse_local_part(recipient) + + if not objs: + log.info("but email is not a comment-reply") + to_user = User.find_by_email(re.sub(r"(.|\n|\r|\r\n)*?<|>", u"", + message["From"])) + util.error_mail_to_user(400, to_user, None) + return + + user_obj, comment_obj = objs + + out = "user {0} replied to comment {1}" + log.info(out.format(user_obj.id, comment_obj.id)) + log.info("try to write comment-reply to database") + + text, sentiment = parse_payload(message) + + if len(text) > 3: + setupcomment.comment(user_obj, comment_obj, text, sentiment) + else: + log.error("cannot reply: readable text is shorter than 4 characters!") + util.error_mail_to_user(411, user_obj, comment_obj) diff --git a/src/adhocracy/lib/emailcomments/permissions.py b/src/adhocracy/lib/emailcomments/permissions.py new file mode 100644 index 000000000..c0ccf3fc6 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/permissions.py @@ -0,0 +1,31 @@ +from repoze.what.middleware import AuthorizationMetadata +from repoze.what.plugins.sql.adapters import SqlPermissionsAdapter +from adhocracy import model +from adhocracy.lib.auth.authorization import InstanceGroupSourceAdapter +import logging + +log = logging.getLogger(__name__) + + +def setup_perm(environ, user): + ''' + Set permissions for replying user. This is a short authorization + for emailcomments. It contains some functions of + lib/auth/authentication: setup_auth. + Though the adapters must be passed from here to AuthorizationMetadata + directly to setup authorization for the user. Logging in is neither + required nor useful so a modified quick-auth comes in handy. + ''' + + identity = {'repoze.who.userid': str(user.user_name)} + + groupadapter = InstanceGroupSourceAdapter() + permissionadapter = SqlPermissionsAdapter(model.Permission, + model.Group, + model.meta.Session) + group_adapters = {'sql_auth': groupadapter} + permission_adapters = {'sql_auth': permissionadapter} + + get_perm = AuthorizationMetadata(group_adapters=group_adapters, + permission_adapters=permission_adapters) + AuthorizationMetadata.add_metadata(get_perm, environ, identity) diff --git a/src/adhocracy/lib/emailcomments/setupcomment.py b/src/adhocracy/lib/emailcomments/setupcomment.py new file mode 100644 index 000000000..1732070bb --- /dev/null +++ b/src/adhocracy/lib/emailcomments/setupcomment.py @@ -0,0 +1,71 @@ +import logging + +import pylons +from pylons import tmpl_context as c +from pylons import request +from pylons.util import ContextObj, PylonsContext + +from adhocracy.model import instance_filter +from adhocracy.controllers import comment as Comment + +from adhocracy.lib.emailcomments import permissions, util +from adhocracy.lib.search import index + +from webob import Request + +log = logging.getLogger(__name__) + + +def setup_req(): + '''create request''' + request = Request.blank("http://localhost") + pylons.request._push_object(request) + + +def setup_c(user_obj): + ''' + setup the tmpl_context: + sunburnt_connection: solr + user, instance: repoze / permissions + ''' + c = ContextObj() + py_obj = PylonsContext() + py_obj.tmpl_context = c + pylons.tmpl_context._push_object(c) + c = pylons.tmpl_context + + c.instance = instance_filter.get_instance() + c.user = user_obj + c.sunburnt_connection = index.make_connection() + + +def comment(user_obj, comment_obj, text, sentiment): + ''' + If comment does not already exist (inbox sync): + Setup of template-ontext, request and permissions for instance/user. + Finally teardown instance, request-environment and template-context. + ''' + if util.comment_exists(text, user_obj, comment_obj.id): + return + instance_filter.setup_thread(comment_obj.topic.instance) + setup_req() + setup_c(user_obj) + permissions.setup_perm(request.environ, c.user) + try: + Comment._create(user=c.user, + reply=comment_obj, + topic=comment_obj.topic, + text=text, + wiki=0, + variant=comment_obj.variant, + instance=c.instance, + sentiment=sentiment) + except Exception as e: + log.info("error in creating comment: {0}".format(e)) + util.error_mail_to_user(401, user_obj, comment_obj) + finally: + instance_filter.teardown_thread() + request.environ = {} + del c.user + del c.instance + del c.sunburnt_connection diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py new file mode 100644 index 000000000..d208d0cfd --- /dev/null +++ b/src/adhocracy/lib/emailcomments/util.py @@ -0,0 +1,307 @@ +import os +import shutil +import logging +import re +from pylons.i18n import _ +from pylons import config +from BeautifulSoup import BeautifulSoup +from datetime import datetime + +log = logging.getLogger(__name__) + + +def comment_exists(text, user, reply_id): + ''' + Tries to determine if a comment already exists. This can be useful if + more than one worker watches the same Inbox and an email is fetched + multiple times. The last comments made by the user in a specific time + are checked on the same topic and the text itself. + If a user replied the same to the same topic in the last two minutes + the reply will be ignored. + ''' + now = datetime.utcnow() + + i = 0 + for x in user.comments: + time_diff = now - user.comments[i].create_time + if time_diff.seconds < 120 and time_diff.days == 0: + cmp_text = user.comments[i].revisions[0].text + cmp_reply_id = user.comments[i].reply_id + if reply_id == cmp_reply_id: + if cmp_text == text: + return True + else: + return False + i += 1 + + return True + + +def content_type_reader(content_type): + img_type = re.search(r"image/([^;]*);?", content_type).group(1) + try: + img_name = re.search("name=([^;]*)", + content_type).group(1).rpartition(".")[0] + except: + img_name = u"{0}-image".format(img_type) + return (img_type, img_name) + + +def error_mail_to_user(errcode, to_user, orig_comm): + '''receives error codes if posting fails and send notification to user''' + if errcode == 411: + body = _(u"Error: Your reply must be longer than 4 characters.\r\n") + if errcode == 400: + body = _(u"Error: The email address you replied to is invalid.\r\n") + if errcode == 401: + body = _(u"Error: You lack permissions to post replies via email.\r\n") + + if orig_comm: + br = u"\r\n\r\n" + body = body + _(u"original comment:") + br + orig_comm.latest.text + body = body + br + _(u"by: ") + orig_comm.creator.name + br + else: + body = body + _(u"Original Comment cannot be determined.\r\n") + try: + mail.to_mail(to_user.user_name, + to_user.email, + _(u"Email comment failed!"), + body, + headers={}, + decorate_body=True, + email_from=config.get('error_email_from'), + name_from="EC-Error") + except: + log.error("could not send error mail to user: user undeterminable") + return + log.info("sent error mail to user") + + +def strip_local_part(recipient): + '''get userid, commentid, security token from local part''' + pattern = re.compile(r"""\Asubs\. + (?P[1-9][0-9]*)- + (?P[1-9][0-9]*)\. + (?P[a-fA-F0-9]{40})@""", re.VERBOSE) + result = pattern.match(recipient) + return result + + +def remove_notification(text): + '''removes notification if user didn't''' + text = re.sub(ur">*\s*_{33}(\r|\n|\r\n|.)*_{33}", u"", text) + return text + + +def move_overwrite(src, dst): + '''overwrite if a file with same name exists''' + if os.path.exists(dst): + os.remove(dst) + shutil.move(src, dst.rpartition("/")[0]) + + +def get_sentiment(text): + ''' + User can vote via first line of his email: + (Vote: )1 or +1 or 0 or -1 or + or - + The Vote will be extracted and the line will be deleted if present. + Also all leading newlines after vote-line will be deleted. + ''' + pattern = re.compile(ur"""\A(?:[vV][oO][tT][eE]:?\s*)? + (?P0|1|\+1?|-1?)\s* + (?:\n|\r|\r\n)+ + (?P(.|\n|\r|\r\n)*) + """, re.VERBOSE) + result = pattern.match(text) + if result: + text = result.group("text") + sentiment = result.group("sentiment") + if "+" in sentiment or sentiment == "1": + sentiment = 1 + elif "-" in sentiment: + sentiment = -1 + else: + sentiment = 0 + else: + sentiment = None + + return text, sentiment + + +def html_to_markdown(text): + ''' + Basic HTML-Parsing for markdown-output: + Use regex to replace tags for use with markdown. + + tag replacements: +
: + replaced by " \n" + text or text: + replaced by "**text**" + text or text: + replaced by "*text*" + text and
text
: + replaced by "\n> text\n" with spaces for breaks + text: + replaced by "{x times #} text" +
  • text
    • : + replaced by "* text " + descr: + replaced by "[descr](http://link "http://link")" (http added if + required) + alttext: + replaced by "![alttext](http://link "http://link")" (http added if + required) +
  • text
    • : + replaced by "* text " + This is a very basic list parser and returns always an unordered list + without indentations. Deletes before. + and : + replaced by " \n" + other tags: + replaced by "" + ''' + def h_repl(matchobj): + if matchobj: + h = u"" + for i in range(int(matchobj.group(1))): + h = h + u"#" + return h + u" " + else: + return "" + + def a_repl(matchobj): + if matchobj: + try: + href = re.search(u"href=[\"]?([^\">]*)[\"]?", + matchobj.group(0)).group(1) + alt = re.search(u">(.*?)<", + matchobj.group(0)).group(1) + if not "http" in href: + href = u"http://" + href + link = u"[{0}]({1} \"{2}\")".format(alt, href, href) + return link + except: + return "" + else: + return "" + + def img_repl(matchobj): + if matchobj: + try: + src = re.search(u"src=[\"]?([^\">]*)[\"]?", + matchobj.group(0)).group(1) + alt = re.search(u"alt=[\"]?([^\">]*)[\"]?", + matchobj.group(0)).group(1) + if not "http" in src: + src = u"http://" + src + link = u"![{0}]({1} \"{2}\")".format(alt, src, src) + return link + except: + return "" + else: + return "" + + def li_repl(matchobj): + if matchobj: + li = u"* " + matchobj.group(1) + u" \n" + return li + else: + return "" + + def parse_soup(text): + soup = BeautifulSoup(text) + blacklist = ["script", "style"] + keep_attrs = ["a", "img"] + + for tag in soup.findAll(): + if tag.name.lower() in blacklist: + tag.extract() + if not tag.name.lower() in keep_attrs: + tag.attrs = {} + return unicode(soup) + + replacements = [(ur"(\n|\r|\r\n)[\s]*|\t[\s]*", u""), + (ur"\s{2,}", u" "), + (ur"
    ", u" \n"), + (ur"([\s]*)|()", u"**"), + (ur"([\s]*)|()", u"**"), + (ur"([\s]*)|()", u"*"), + (ur"([\s]*)|()", u"*"), + (ur"|", u" > "), + (ur"|", u" \n\n"), + (ur"", h_repl), + (ur"", u" \n"), + (u"", a_repl), + (ur"", img_repl), + (ur"", u""), + (ur"
  • [\s]*([^<|\\n]*)", li_repl), + (ur"()|()", u" \n"), + (ur"<.*?>", u"")] + + text = parse_soup(text) + + for i, j in replacements: + text = re.sub(i, j, text, flags=re.IGNORECASE) + + return text + + +def delete_debris(text): + ''' + Deletes all ending and leading lines, also shrinks number of lines + to two if more than three are concatenated at once. + ''' + replacements = [ur"(\n|\r|\s)*\Z", + ur"\A(\n|\r|\s)*"] + + for item in replacements: + text = re.sub(item, r"", text) + + text = re.sub(ur"((\n|\r\s*|\r\n\s*){3,})", u"\n\n", text) + + return text + + +def delete_signatures(text): + ''' + Deletes PGP or other signatures: + If text begins with a sequence of 2 "-" or "_" at the beginning + of a new line the rest of the text will be stripped(RFC3676). + If text contains a reference to the PGP-signed message, the line + will be deleted before signatures because of containing "-"s. + Order is crucial: for further replacements add something to the end + of the list. + ''' + pgp_debris = ur"(.*?(\r|\n|\r\n))*?" + pgp_start = ur"-----BEGIN PGP SIGNED MESSAGE-----" + pgp_hash = ur"((\r|\n|\r\n)hash:\s?\w*(\n|\r|\r\n|\s)*)?" + p = re.compile(pgp_debris + pgp_start + pgp_hash, re.IGNORECASE) + + replacements = [p, + ur"(\r|\n|\r\n)[-_]{2,}(.|\n|\r|\r\n)*"] + + for item in replacements: + text = re.sub(item, u"", text) + + return text + + +def create_filesystem(path_md): + '''create directories if possible and required''' + try: + if not os.path.exists(path_md): + os.makedirs(path_md) + elif not os.path.exists(os.path.join(path_md, "new")): + os.makedirs(os.path.join(path_md, "new")) + elif not os.path.exists(os.path.join(path_md, "cur")): + os.makedirs(os.path.join(path_md, "cur")) + elif not os.path.exists(os.path.join(path_md, "tmp")): + os.makedirs(os.path.join(path_md, "tmp")) + elif not os.path.exists("/var/mail"): + os.makedirs("/var/mail") + return True + except IOError: + log.error("you have no permission to create Maildir or mbox") + log.info("emailcomments are disabled") + return False diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 2003d1fbe..b543d7451 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -1,8 +1,8 @@ import logging - +import hashlib from pylons import config from webhelpers import text - +from pylons.i18n import _ from adhocracy.lib import mail, microblog TWITTER_LENGTH = 140 @@ -40,6 +40,16 @@ def twitter_sink(pipeline): def mail_sink(pipeline): + """ + Generates email-adress for comment-replies in the form of + subscription.userid-commentid.security-token@domain.tld + if the notification is about a new comment. + HTML will be activated. If a user can not display HTML-emails + the multipart/alternative plaintext-part will display the + default notification.body. Otherwise an HTML-email is only + a beautified version. + This feature is not yet completed. + """ for notification in pipeline: if notification.user.is_email_activated() and \ notification.priority >= notification.user.email_priority: @@ -49,10 +59,63 @@ def mail_sink(pipeline): log.debug("mail to %s: %s" % (notification.user.email, notification.subject)) - mail.to_user(notification.user, - notification.subject, - notification.body, - headers=headers) + notification_body = notification.body + + if str(notification.event.event) == "t_comment_create" or \ + str(notification.event.event) == "n_comment_reply": + + secrets = config.get("adhocracy.crypto.secret") + email_from = config.get("adhocracy.email.from") + + email_domain = email_from.rpartition("@") + email_domain = email_domain[1] + email_domain[2] + + user_id = str(notification.user.id) + + comment_id = notification.link.split("#c")[-1] + + sec_token = user_id + comment_id + secrets + sec_token = hashlib.sha1(sec_token).hexdigest() + + seq = (u" ") + reply_to = "".join(seq) + reply_msg = _(u"\"Reply to leave a comment\"") + reply_to = reply_msg + reply_to + + headers['In-Reply-To'] = reply_to + headers['Reply-To'] = reply_to + + boundary = (u"_________________________________\r\n") + + notification_body = (_(u"Comment by replying to this email." + u"\r\nWrite your answer above the upper line or under the" + u" bottom line." + u"\r\n\r\nYou can use the first line of your reply" + u" to vote. Type vote 1 for a positive vote," + u" vote 0 for a neutral vote and vote -1 for a" + u" negative vote.\r\n\r\n") + notification_body) + # Mention images? (Don't work with markdown safe_mode yet) + + notification_body = boundary + (_(u"Hi %s,") % + notification.user.name + + u"\r\n%s\r\n\r\n" % notification_body + + _(u"Cheers,\r\n\r\n" + u" the %s Team\r\n") % + config.get('adhocracy.site.name')) + u"\r\n" + boundary + + html = False + # html deactivated due to email-comment parsing/voting + decorate_body = False + else: + html = False + + mail.to_user(notification.user, + notification.subject, + notification_body, + headers=headers, + html=html, + decorate_body=decorate_body) else: yield notification diff --git a/src/adhocracy/lib/mail.py b/src/adhocracy/lib/mail.py index 3d6ac474c..906ff7422 100644 --- a/src/adhocracy/lib/mail.py +++ b/src/adhocracy/lib/mail.py @@ -1,6 +1,7 @@ import email from email.header import Header from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart import logging import smtplib from time import time @@ -23,7 +24,7 @@ def send(email_from, to_email, message): def to_mail(to_name, to_email, subject, body, headers={}, decorate_body=True, - email_from=None, name_from=None): + email_from=None, name_from=None, html=False): try: if email_from is None: email_from = config.get('adhocracy.email.from') @@ -36,8 +37,17 @@ def to_mail(to_name, to_email, subject, body, headers={}, decorate_body=True, _(u"Cheers,\r\n\r\n" u" the %s Team\r\n") % config.get('adhocracy.site.name')) + if html: # deactivated in mail sinks due to emailcomment-parsing + msg = MIMEMultipart("alternative") + part_plain = MIMEText(body.encode(ENCODING), 'plain', + ENCODING) + html_mail = "html_template" + part_html = MIMEText(html_mail, 'html', ENCODING) - msg = MIMEText(body.encode(ENCODING), 'plain', ENCODING) + msg.attach(part_plain) + msg.attach(part_html) + else: + msg = MIMEText(body.encode(ENCODING), 'plain', ENCODING) for k, v in headers.items(): msg[k] = v @@ -56,9 +66,9 @@ def to_mail(to_name, to_email, subject, body, headers={}, decorate_body=True, def to_user(to_user, subject, body, headers={}, decorate_body=True, - email_from=None, name_from=None): + email_from=None, name_from=None, html=False): return to_mail(to_user.name, to_user.email, subject, body, headers, - decorate_body, email_from, name_from) + decorate_body, email_from, name_from, html) def send_activation_link(user): diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py new file mode 100644 index 000000000..637c4b280 --- /dev/null +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -0,0 +1,213 @@ +from adhocracy.tests import TestController +import adhocracy.lib.emailcomments.util +import adhocracy.lib.emailcomments.parseincoming +import adhocracy.lib.text.render +from adhocracy.tests import testtools + +class EmailParsingTest(TestController): + def test_render(self): + '''safe_mode must be False''' + r = adhocracy.lib.text.render + self.assertEqual(r(u"![alt](http://test.com/test.jpg \"title\")"), + u"""

    \"alt\"

    """) + self.assertEqual(r(u"![alt](data:image/png;base64,img \"title\")"), + u"""

    \"alt\"

    """) + + + def test_delete_signatures(self): + ds = adhocracy.lib.emailcomments.util.delete_signatures + self.assertEqual(ds(u'foobar\n__barfoo'), u'foobar') + self.assertEqual(ds(u'foobar\n--barfoo'), u'foobar') + self.assertEqual(ds(u'foobar\n __barfoo'), u'foobar\n __barfoo') + self.assertEqual(ds(u'foobar\n_barfoo'), u'foobar\n_barfoo') + self.assertEqual(ds(u'foobar\n-----BEGIN PGP SIGNED MESSAGE-----\n'), u'\n') + self.assertEqual(ds(u'foobar\n-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\nbarfoo'), u'barfoo') + self.assertEqual(ds(u'foobar\nHash: SHA512\n\nbarfoo'), u'foobar\nHash: SHA512\n\nbarfoo') + self.assertEqual(ds(u""" +some debris... +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +This is the Payload of +the PGP-signed message. + +-----BEGIN PGP SIGNATURE----- +iQA/AwUBONpOg40d+PaAQUTlEQIc5ACdGkKSzpOrsT0Gvj3jH9NXD8ZP2IcAn0vj +/BHT+qQCtPCtCwO1aQ3Xk/NL +=1CZt +-----END PGP SIGNATURE----- + +__________________ +provider signature"""), + +u"""This is the Payload of +the PGP-signed message. +""") + + + def test_html_to_markdown(self): + rp = adhocracy.lib.emailcomments.util.html_to_markdown + self.assertEqual(rp(u'
    '), u' \n') + self.assertEqual(rp(u'\n\t
    \t\t\n\n
    '), u' \n \n') + self.assertEqual(rp(u'foobar
    \n\nbarfoo'), u'foobar \nbarfoo') + self.assertEqual(rp(u' foobar'), u'**foobar**') + self.assertEqual(rp(u'

    foobar

    '), u'## foobar \n') + self.assertEqual(rp(u'

    foobar

    '), u'## foobar \n') + self.assertEqual(rp(u'foobar
    \r\n\n\rbarfoo'), u'foobar \nbarfoo') + self.assertEqual(rp(u""" + + + + + + + + +
      +
    • foo +
    • bar
      • +
    +

    bla

    + foo
    + + bar
    +
    + barbar
    + foobar +
    bar
    + barfoo +
    +
    +
    +
    + + + + +"""), + +u"""* foo +* bar + +### bla +![foo](http://foo.bar/bar.gif "http://foo.bar/bar.gif") + + +*bar***bar** + > foobar + + > bar + +[barfoo](http://foobar.de "http://foobar.de") + + + +""") + + + def test_get_sentiment(self): + gs = adhocracy.lib.emailcomments.util.get_sentiment + self.assertEqual(gs(u'vote:+\n'), (u'', 1)) + self.assertEqual(gs(u'vote:+\r\n'), (u'', 1)) + self.assertEqual(gs(u'vote:+\r'), (u'', 1)) + self.assertEqual(gs(u'Vote: 1\nfoo bar'), (u'foo bar', 1)) + self.assertEqual(gs(u'Vote: +1\r\nfoo\nbar'), (u'foo\nbar', 1)) + self.assertEqual(gs(u'0\na'), (u'a', 0)) + self.assertEqual(gs(u'-\nfoo'), (u'foo', -1)) + self.assertEqual(gs(u'anything'), (u'anything', None)) + self.assertEqual(gs(u'Vote: -1\n'), (u'', -1)) + self.assertEqual(gs(u'Vote -1\n'), (u'', -1)) + self.assertEqual(gs(u'2010\n'), (u'2010\n', None)) + self.assertEqual(gs(u'010\n'), (u'010\n', None)) + self.assertEqual(gs(u"""vote + +The Message"""), +(u"""The Message""", 1)) + + + def test_content_type_reader(self): + ctr = adhocracy.lib.emailcomments.util.content_type_reader + self.assertEqual(ctr(u'image/png'), (u'png', u'png-image')) + self.assertEqual(ctr(u'image/png; name=foobar.png'), (u'png', u'foobar')) + self.assertEqual(ctr(u'image/png; name=bar.foobar.png'), (u'png', u'bar.foobar')) + + + def test_delete_debris(self): + dd = adhocracy.lib.emailcomments.util.delete_debris + self.assertEqual(dd(u'foobar\n\n\n\n'), u'foobar') + self.assertEqual(dd(u'\n\nfoobar\n\n'), u'foobar') + self.assertEqual(dd(u'\n foobar\n '), u'foobar') + self.assertEqual(dd(u' \nfoobar \n'), u'foobar') + self.assertEqual(dd(u' \nfoobar \n'), u'foobar') + self.assertEqual(dd(u'\r\n \n\nfoobar \r\n \n\n'), u'foobar') + self.assertEqual(dd(u'foo\n\n\n\n\nbar'), u'foo\n\nbar') + self.assertEqual(dd(u' \n foo\r\n \n \n \n\r\nbar \n '), u'foo\n\nbar') + + + def test_parse_local_part(self): + '''test for admin''' + from pylons import config + secrets = config.get("adhocracy.crypto.secret") + + comment = testtools.tt_make_comment() + + reply_id = unicode(comment.id) + + import hashlib + sec_token = hashlib.sha1(u"1" + reply_id + secrets).hexdigest() + + test_string = (u'subs.1-{0}.{1}@domain.tld').format(reply_id, + sec_token) + + fail_string = (u'subs.1-{0}.MOCKSECRET@domain.tld').format(reply_id) + + from adhocracy.model import User + user = User.find(1) + + plp = adhocracy.lib.emailcomments.parseincoming.parse_local_part + self.assertEqual(plp(u'subs.50-50.foobar@domain.tld'), None) + self.assertEqual(plp(u'subs.1-1.feb340279618fb47d6c0feb340279618fb47d6c0@domain.tld'),None) + self.assertEqual(plp(test_string), (user, comment)) + self.assertEqual(plp(fail_string), None) + + + def test_remove_notification(self): + rm = adhocracy.lib.emailcomments.util.remove_notification + self.assertEqual(rm(u"""first new line +_________________________________ +some text here +some text there +_________________________________ +some new line"""), u"""first new line +some new line""") + self.assertEqual(rm(u"""first new line +>_________________________________ +>some text here +>some text there +>_________________________________ +some new line"""), u"""first new line + +some new line""") + self.assertEqual(rm(u"""first new line +> _________________________________ +> some text here +> some text there +> _________________________________ +some new line"""), u"""first new line + +some new line""") + self.assertEqual(rm(u"""first new line +foobar> _________________________________ +> some text here +> some text there +> _________________________________ +some new line"""), u"""first new line +foobar +some new line""") diff --git a/src/adhocracy/tests/testtools.py b/src/adhocracy/tests/testtools.py index c5317358d..07c0f9012 100644 --- a/src/adhocracy/tests/testtools.py +++ b/src/adhocracy/tests/testtools.py @@ -97,6 +97,20 @@ def tt_make_user(name=None, instance_group=None): return user +def tt_make_comment(proposal=None, creator=None): + if proposal is None: + proposal = tt_make_proposal() + if creator is None: + creator = tt_make_user("DeKue") + + comment = model.Comment.create( + tt_make_str(), + creator, + proposal) + + return comment + + def tt_drop_db(): ''' drop the data tables and if exists the migrate_version table