From 417f60a6bf98806ccd76a74270d5a78f4bc9a016 Mon Sep 17 00:00:00 2001 From: dekue Date: Thu, 1 Aug 2013 22:34:30 +0200 Subject: [PATCH 01/21] update --- adhocracy | 1 + 1 file changed, 1 insertion(+) create mode 160000 adhocracy diff --git a/adhocracy b/adhocracy new file mode 160000 index 000000000..872a5ccd6 --- /dev/null +++ b/adhocracy @@ -0,0 +1 @@ +Subproject commit 872a5ccd61ec8e8774c466c35227e75b7f663e53 From 29c8bc0e453e7902ff41913fcf42bcde67deeeda Mon Sep 17 00:00:00 2001 From: Dekue Date: Thu, 1 Aug 2013 23:35:21 +0200 Subject: [PATCH 02/21] update --- .gitignore | 1 + python/buildout.python | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 160000 python/buildout.python diff --git a/.gitignore b/.gitignore index 59ef84693..13197dc5c 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ buildout_current.cfg .sass-cache src/adhocracy/static/stylesheets/adhocracy.css src/adhocracy/tests/loadtests/data +/.project diff --git a/python/buildout.python b/python/buildout.python deleted file mode 160000 index 44b473393..000000000 --- a/python/buildout.python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 44b4733937827f0765419ff935100aa385341dd3 From 66b06e2130181ed470fae02e05643609cb963e16 Mon Sep 17 00:00:00 2001 From: Dekue Date: Fri, 2 Aug 2013 00:45:24 +0200 Subject: [PATCH 03/21] enables comments via email --- etc/adhocracy.ini.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/etc/adhocracy.ini.in b/etc/adhocracy.ini.in index 7e3efef02..3c39a035d 100644 --- a/etc/adhocracy.ini.in +++ b/etc/adhocracy.ini.in @@ -102,6 +102,17 @@ use = egg:adhocracy full_stack = true static_files = true +# INSTALL: To enable set email_src to 'local' or 'imap', IMAP-SSL is essential. +# If 'imap_account = user' fails, try complete address: 'user@emailprovider.tld'. +# Set local_user to determine the standard-mbox/Maildir-folders you want to watch. +email_src = None +local_user = None +imap_domain = imap.emailprovider.tld +imap_account = user +imap_password = password +imap_directory = inbox +imap_port = 993 + cache_dir = ${parts.buildout.directory}/var/data beaker.session.key = adhocracy_state {% python From 929cb46383d5188306a51ebc298edcb8ada650b5 Mon Sep 17 00:00:00 2001 From: Dekue Date: Fri, 2 Aug 2013 00:46:17 +0200 Subject: [PATCH 04/21] enables comments via email --- adhocracy | 10 +- buildouts/adhocracy.cfg | 5 +- etc/test.ini.in | 7 + setup.py | 3 + src/adhocracy/controllers/comment.py | 66 +++-- src/adhocracy/lib/emailcomments/__init__.py | 58 ++++ src/adhocracy/lib/emailcomments/imap.py | 66 +++++ src/adhocracy/lib/emailcomments/localwatch.py | 82 ++++++ .../lib/emailcomments/parseincoming.py | 161 ++++++++++ .../lib/emailcomments/permissions.py | 31 ++ .../lib/emailcomments/setupcomment.py | 68 +++++ src/adhocracy/lib/emailcomments/util.py | 276 ++++++++++++++++++ src/adhocracy/lib/event/notification/sinks.py | 49 +++- src/adhocracy/lib/mail.py | 18 +- src/adhocracy/tests/lib/test_email_parsing.py | 178 +++++++++++ src/adhocracy/tests/testtools.py | 14 + 16 files changed, 1058 insertions(+), 34 deletions(-) mode change 160000 => 100644 adhocracy create mode 100644 src/adhocracy/lib/emailcomments/__init__.py create mode 100644 src/adhocracy/lib/emailcomments/imap.py create mode 100644 src/adhocracy/lib/emailcomments/localwatch.py create mode 100644 src/adhocracy/lib/emailcomments/parseincoming.py create mode 100644 src/adhocracy/lib/emailcomments/permissions.py create mode 100644 src/adhocracy/lib/emailcomments/setupcomment.py create mode 100644 src/adhocracy/lib/emailcomments/util.py create mode 100644 src/adhocracy/tests/lib/test_email_parsing.py diff --git a/adhocracy b/adhocracy deleted file mode 160000 index 872a5ccd6..000000000 --- a/adhocracy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 872a5ccd61ec8e8774c466c35227e75b7f663e53 diff --git a/adhocracy b/adhocracy new file mode 100644 index 000000000..8b5c7f68d --- /dev/null +++ b/adhocracy @@ -0,0 +1,9 @@ +tree e981b8f97ef54808aae69bfb8d145108b8062c8a +parent ce1d855668eb1a938cece2b5bfc3666c1fee82f0 +author Nicolas Dietrich 1375302727 +0200 +committer Nicolas Dietrich 1375302727 +0200 + +Remove unneeded file again + +templates/user/edit.html has been accidently readded in +03f382dac7e0248f221a94f51854c8f79191d961. diff --git a/buildouts/adhocracy.cfg b/buildouts/adhocracy.cfg index fb48bcf6e..d730ae3fc 100644 --- a/buildouts/adhocracy.cfg +++ b/buildouts/adhocracy.cfg @@ -24,6 +24,8 @@ auto-checkout = * # supervisor config to start adhocracy adhocracy_worker-supervisor = 40 adhocracy_worker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_worker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy worker -c ${buildout:directory}/etc/adhocracy.ini] +adhocracy_ecworker-supervisor = + 40 adhocracy_ecworker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_ecworker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy ecworker -c ${buildout:directory}/etc/adhocracy.ini] adhocracy-supervisor = 45 adhocracy (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [serve ${buildout:directory}/etc/adhocracy.ini] # parts in this buildout file to be installed @@ -49,7 +51,7 @@ supervisor = 5010 ############################################################################## # Adhocracy settings -############################################################################## +####################################### ${buildout:adhocracy_ecworker-supervisor}####################################### [adhocracy] #start adhocracy in debug mode @@ -170,4 +172,5 @@ scripts = adhocpy environment = LD_LIBRARY_PATH="${buildout:directory}/python/python-2.7/lib/" programs += ${buildout:adhocracy_worker-supervisor} + ${buildout:adhocracy_ecworker-supervisor} ${buildout:adhocracy-supervisor} diff --git a/etc/test.ini.in b/etc/test.ini.in index b5c4df5ef..9df9b6005 100644 --- a/etc/test.ini.in +++ b/etc/test.ini.in @@ -47,6 +47,13 @@ memcached.server = 127.0.0.1:${parts.ports.memcached} #solr adhocracy.solr.url = http://${parts.solr.host}:${parts.solr.port}/solr +{% python + import random; + def randomhash(length): + return hex(random.SystemRandom().getrandbits(length)) +%} +#adhocracy session secret +adhocracy.session.secret = {% if parts.adhocracy.secret == 'autogenerated' %}${randomhash(256)}{% end %}{% if parts.adhocracy.secret != 'autogenerated' %}${parts.adhocracy.secret}{% end %} # beaker beaker.session.key = adhocracy_state diff --git a/setup.py b/setup.py index a34eac64c..751b05a07 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,8 @@ "setuptools_git >= 0.3", "ipaddress>=1.0.3", "pytz", + "pyinotify>=0.9.4", + "imaplib2>=2.28.1", ], setup_requires=["setuptools>=0.6c6", # fix OS X 10.5.7 "PasteScript", @@ -111,6 +113,7 @@ ], 'paste.paster_command': [ 'worker = adhocracy.lib.cli:Worker', + 'ecworker = adhocracy.lib.cli:EmailCommentWorker', 'timer = adhocracy.lib.cli:Timer', 'index = adhocracy.lib.cli:Index' ], diff --git a/src/adhocracy/controllers/comment.py b/src/adhocracy/controllers/comment.py index abf908126..21711bcbc 100644 --- a/src/adhocracy/controllers/comment.py +++ b/src/adhocracy/controllers/comment.py @@ -104,35 +104,21 @@ def create(self, format='html'): topic = self.form_result.get('topic') reply = self.form_result.get('reply') - - if reply: - require.comment.reply(reply) - else: - require.comment.create_on(topic) - variant = self.form_result.get('variant') + wiki = self.form_result.get('wiki') + sentiment = self.form_result.get('sentiment') + text = self.form_result.get('text') + if hasattr(topic, 'variants') and not variant in topic.variants: return ret_abort(_("Comment topic has no variant %s") % variant, code=400) - comment = model.Comment.create( - self.form_result.get('text'), - c.user, topic, - reply=reply, - wiki=self.form_result.get('wiki'), - variant=variant, - sentiment=self.form_result.get('sentiment'), - with_vote=can.user.vote()) - - # watch comments by default! - model.Watch.create(c.user, comment) - model.meta.Session.commit() - #watchlist.check_watch(comment) - event.emit(event.T_COMMENT_CREATE, c.user, instance=c.instance, - topics=[topic], comment=comment, topic=topic, - rev=comment.latest) + comment = _create(c.user, text, reply, topic, variant, + wiki, c.instance, sentiment) + if len(request.params.get('ret_url', '')): redirect(request.params.get('ret_url') + "#c" + str(comment.id)) + if format != 'html': return ret_success(entity=comment, format=format) return ret_success(entity=comment, format='fwd') @@ -297,3 +283,39 @@ def reply_form(self, id): topic = parent.topic variant = getattr(topic, 'variant', None) return self._render_ajax_create_form(parent, topic, variant) + + +@guard.comment.create() +def _create(user, text, reply, topic, variant, wiki, instance, sentiment=None): + ''' + extracted from controllers.commentCommentController for usage of + emailcomments + ''' + with_vote = can.user.vote() + + if reply: + require.comment.reply(reply) + else: + require.comment.create_on(topic) + + comment = model.Comment.create( + text, + user, topic, + reply=reply, + wiki=wiki, + variant=variant, + sentiment=sentiment, + with_vote=with_vote) + # watch comments by default! + + model.Watch.create(user, comment) + + model.meta.Session.commit() + + #watchlist.check_watch(comment) + + event.emit(event.T_COMMENT_CREATE, user, instance=instance, + topics=[topic], comment=comment, topic=topic, + rev=comment.latest) + + return comment diff --git a/src/adhocracy/lib/emailcomments/__init__.py b/src/adhocracy/lib/emailcomments/__init__.py new file mode 100644 index 000000000..9029ff444 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/__init__.py @@ -0,0 +1,58 @@ +import os +from pylons import config +import getpass +import logging +import Queue +from sqlalchemy.orm import sessionmaker, scoped_session +from adhocracy.model import meta + +log = logging.getLogger(__name__) + + +def mail_watch(): + ''' + This is a script executed by the adhocracy emailcommentworker + Its target is to listen to new mails (Maildir and mbox or IMAP) + and identify if they are comment replies. If so the jobs + will be executed in a queue. + ''' + esrc = config.get("email_src") + + if esrc == "imap" or esrc == "local": + ecq = Queue.Queue() + + Session = sessionmaker(bind=meta.engine, autoflush=True) + meta.Session = scoped_session(Session) + + from adhocracy.lib.emailcomments import parseincoming + if esrc == "imap": + from adhocracy.lib.emailcomments import imap + idler = imap.Idler(ecq) + idler.start() + elif esrc == "local": + from adhocracy.lib.emailcomments import localwatch + username = config.get("local_user") + if not username: + log.error("no user set in config") + log.info("emailcomments are disabled") + return + if username: + path_md = os.path.join("/home", username, "Maildir") + path_mb = os.path.join("/var/mail", username) + if os.path.exists("/usr/include/linux/inotify.h"): + if util.create_filesystem(path_md): + localwatch.watch_new_mail(path_md, path_mb, ecq) + else: + return + else: + log.error("kernel module inotify is not installed") + log.info("emailcomments are disabled") + return + else: + log.error("user cannot be determined") + log.info("emailcomments are disabled") + return + + while True: + message = ecq.get() + parseincoming.handle_inc_mail(message) diff --git a/src/adhocracy/lib/emailcomments/imap.py b/src/adhocracy/lib/emailcomments/imap.py new file mode 100644 index 000000000..399cd7edd --- /dev/null +++ b/src/adhocracy/lib/emailcomments/imap.py @@ -0,0 +1,66 @@ +import imaplib2 +import email +from email.parser import HeaderParser +import os +import logging +import threading +import time +from pylons import config +from adhocracy.lib.emailcomments import util +from adhocracy.lib import queue + +# cfg +IMAP_DOMAIN = config.get('imap_domain') +IMAP_USERNAME = config.get('imap_account') +IMAP_PASSWORD = config.get('imap_password') +COMMENTS_DIR = config.get('imap_directory') +IMAP_PORT = config.get('imap_port') +MAX_INTERVAL = 60 * 64 # max. reconnect-interval (seconds - 60 * 2^n possible) + +log = logging.getLogger(__name__) + + +class Idler(threading.Thread): + + def __init__(self, ecq): + threading.Thread.__init__(self) + self.ecq = ecq + + def run(self): + '''(re)connects to IMAP and waits for new mail''' + recon_interval = 30 + while True: + try: + self.conn = imaplib2.IMAP4_SSL(IMAP_DOMAIN, IMAP_PORT) + self.conn.login(IMAP_USERNAME, IMAP_PASSWORD) + self.conn.select(COMMENTS_DIR) + except Exception as e: + log.info("IMAP-connection could not be established because:") + log.info(e) + if recon_interval < MAX_INTERVAL: + recon_interval = recon_interval * 2 + minutes = recon_interval / 60 + log.info("reconnect in {0} minute(s)".format(minutes)) + time.sleep(recon_interval) + continue + + recon_interval = 30 + + while True: + try: + self.dosync() + self.conn.idle() + except self.conn.abort: + log.info("IMAP-connection lost, reconnecting...") + + def dosync(self): + '''executed if a new mail arrives''' + typ, data = self.conn.search(None, 'UNSEEN') + if data[0]: + for num in data[0].split(): + typ, data = self.conn.fetch(num, '(RFC822)') + header_data = data[0][1] + parser = HeaderParser() + message = parser.parsestr(header_data) + log.info("new IMAP mail") + self.ecq.put(message) diff --git a/src/adhocracy/lib/emailcomments/localwatch.py b/src/adhocracy/lib/emailcomments/localwatch.py new file mode 100644 index 000000000..53c56b336 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/localwatch.py @@ -0,0 +1,82 @@ +import threading +import pyinotify +import logging +import os +import time +from datetime import datetime +import mailbox +import email +import shutil +from adhocracy.lib.emailcomments import util + +log = logging.getLogger(__name__) + + +def maildir(path, ecq): + '''gets payload and recipient of a new Maildir mail''' + lockmd = threading.Lock() + time.sleep(1) # pause - else mail isn't yet recognized in FS + log.info("new mail in Maildir") + + lockmd.acquire() + try: + for filename in os.listdir(os.path.join(path, "new")): + file_path = os.path.join(path, "new", filename) + mail = open(file_path, "r") + parser = email.Parser.Parser() + message = parser.parse(mail) + mail.close() + ecq.put(message) + util.move_overwrite(file_path, + os.path.join(path, "cur", filename)) + finally: + lockmd.release() + + +def mbox(path, ecq): + '''gets payload and recipient of a new mbox mail''' + time.sleep(1) # pause - else mail isn't yet recognized in FS + log.info("new mail in mbox") + + tmp_mb = os.path.join("/var/tmp", str(datetime.now())) + shutil.copy(path, tmp_mb) + + mbox = mailbox.mbox(tmp_mb) + + # process latest email + try: + message = mbox.get_message(mbox.keys()[-1]) + except Exception as e: + log.error("mbox empty, or header malformed:") + log.error(e) + os.remove(tmp_mb) + return + ecq.put(message) + os.remove(tmp_mb) + + +def watch_new_mail(path_md, path_mb, ecq): + ''' + Uses pyinotify to watch for new local emails. + If a new email arrives it will handled by the ec-Queue. + ''' + wm = pyinotify.WatchManager() + mask_md = pyinotify.IN_CREATE # watch for new files in maildir + mask_mb = pyinotify.IN_CLOSE_WRITE # watch for changed mbox + + class PTmp(pyinotify.ProcessEvent): + + path_decide = None + + def process_IN_CREATE(self, event): + maildir(path_md, ecq) + + def process_IN_CLOSE_WRITE(self, event): + mbox(path_mb, ecq) + + notifier = pyinotify.ThreadedNotifier(wm, PTmp()) + notifier.start() + + wdd = wm.add_watch(path_md, mask_md, rec=True) + wm.add_watch(path_mb, mask_mb, rec=False) + wm.rm_watch(wdd[os.path.join(path_md, "cur")], rec=True) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py new file mode 100644 index 000000000..dfb9ff9ae --- /dev/null +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -0,0 +1,161 @@ +import logging +import hashlib +import re +from adhocracy.model import User, Comment +from adhocracy.lib.emailcomments import setupcomment +from adhocracy.lib.emailcomments import util +from pylons import config + + +log = logging.getLogger(__name__) + + +def parse_multipart(message, a, rec_dep): + '''saves contents of multipart messages to a list''' + content_list = [] + for part in message.get_payload(): + if not part["Content-Type"]: + continue + if part.is_multipart(): + if "multipart/alternative" in part["Content-Type"]: + content_list = parse_multipart(part, "a", rec_dep) + rec_dep += 1 + else: + content_list = parse_multipart(part, "na", rec_dep) + else: + if "image" in part["Content-Type"]: + content_list.append((rec_dep, a, part["Content-Type"], + part.get_payload(decode=False))) + else: + content_list.append((rec_dep, a, part["Content-Type"], + unicode(part.get_payload(decode=True), + encoding="utf8", errors="replace"))) + return content_list + + +def save_image(image, content_type): + ''' + Helper function for later use of saving attached images in database + and return markdown-code. + Markdown safe_mode must be set to False, else an image with + alt-text will be rendered but looks like plain text. + ''' + data_uri = image.replace("\n", "") + img_type, img_name = util.content_type_reader(content_type) + img_mkd = """ \n![{0}](data:image/{1};base64,{2} \"{0}\") \n""" + img_mkd = img_mkd.format(img_name, img_type, data_uri) + return img_mkd + + +def get_usable_content(content_list): + ''' + Extracts all readable contents of list and prefers HTML over plaintext if + a multipart/alternative content is present. + Images will be saved via data URL. + ''' + text = "" + alt_set = [] + temp_alt_text = "" + for rec_dep, alt_version, content_type, payload in content_list: + if "na" in alt_version: + text = text + temp_alt_text + temp_alt_text = "" + if "text/plain" in content_type: + text = text + payload + elif "text/html" in content_type: + text = text + util.html_to_markdown(payload) + elif "image" in content_type: + text = text + save_image(payload, content_type) + else: + if not rec_dep in alt_set: + if "text/plain" in content_type: + temp_alt_text = payload + elif "text/html" in content_type: + alt_set.append(rec_dep) + temp_alt_text = util.html_to_markdown(payload) + text = text + temp_alt_text + return text + + +def parse_payload(message): + ''' + Extracts Payload of emails. The payload of a message will be decoded + if the content-transfer-encoding is base64 or quoted-printable. + If other encodings (7bit/8bit, bogus base64) are detected no decoding + happens. + Multipart messages are supported; the payloads will be concatenated if + readable (html/plain text). + If a multipart message is "alternative" HTML will be prefered over plain + text. + ''' + if message.is_multipart(): + if "multipart/alternative" in message["Content-Type"]: + content_list = parse_multipart(message, "a", 0) + else: + content_list = parse_multipart(message, "na", 0) + text = get_usable_content(content_list) + text = util.delete_signatures(text) + else: + if "text/plain" in message["Content-Type"]: + text = message.get_payload(decode=True) + elif "text/html" in message["Content-Type"]: + text = message.get_payload(decode=True) + text = util.html_to_markdown(text) + else: + text = "" + text = util.delete_signatures(text) + text, sentiment = util.get_sentiment(text) + text = util.delete_debris(text) + return (text, sentiment) + + +def parse_local_part(recipient): + '''parse local part of email address to determine if it's a comment''' + result = util.strip_local_part(recipient) + if not result: + return None + + sec_token = result.group("sectoken") + + secrets = config.get("adhocracy.session.secret") + + comp_str = result.group("userid") + result.group("commentid") + comp_str = hashlib.sha1(comp_str + secrets).hexdigest() + + if comp_str != sec_token: + return None + + user_obj = User.find(result.group("userid")) + comment_obj = Comment.find(result.group("commentid")) + + if user_obj is None or comment_obj is None: + return None + return (user_obj, comment_obj) + + +def handle_inc_mail(message): + '''Check if incoming email is a comment - if so call commentcreate.''' + recipient = re.sub(r"(.|\n|\r|\r\n)*?<|>", u"", message["To"]) + + objs = parse_local_part(recipient) + + if not objs: + log.info("but email is not a comment-reply") + to_user = User.find_by_email(re.sub(r"(.|\n|\r|\r\n)*?<|>", u"", + message["From"])) + util.error_mail_to_user(400, to_user, None) + return + + user_obj, comment_obj = objs + + out = "user {0} replied to comment {1}" + log.info(out.format(user_obj.id, comment_obj.id)) + log.info("try to write comment-reply to database") + + text, sentiment = parse_payload(message) + + if len(text) > 3: + setupcomment.comment(user_obj, comment_obj, text, sentiment) + else: + log.error("cannot reply: readable text is shorter than 4 characters!") + util.error_mail_to_user(411, user_obj, comment_obj) diff --git a/src/adhocracy/lib/emailcomments/permissions.py b/src/adhocracy/lib/emailcomments/permissions.py new file mode 100644 index 000000000..c0ccf3fc6 --- /dev/null +++ b/src/adhocracy/lib/emailcomments/permissions.py @@ -0,0 +1,31 @@ +from repoze.what.middleware import AuthorizationMetadata +from repoze.what.plugins.sql.adapters import SqlPermissionsAdapter +from adhocracy import model +from adhocracy.lib.auth.authorization import InstanceGroupSourceAdapter +import logging + +log = logging.getLogger(__name__) + + +def setup_perm(environ, user): + ''' + Set permissions for replying user. This is a short authorization + for emailcomments. It contains some functions of + lib/auth/authentication: setup_auth. + Though the adapters must be passed from here to AuthorizationMetadata + directly to setup authorization for the user. Logging in is neither + required nor useful so a modified quick-auth comes in handy. + ''' + + identity = {'repoze.who.userid': str(user.user_name)} + + groupadapter = InstanceGroupSourceAdapter() + permissionadapter = SqlPermissionsAdapter(model.Permission, + model.Group, + model.meta.Session) + group_adapters = {'sql_auth': groupadapter} + permission_adapters = {'sql_auth': permissionadapter} + + get_perm = AuthorizationMetadata(group_adapters=group_adapters, + permission_adapters=permission_adapters) + AuthorizationMetadata.add_metadata(get_perm, environ, identity) diff --git a/src/adhocracy/lib/emailcomments/setupcomment.py b/src/adhocracy/lib/emailcomments/setupcomment.py new file mode 100644 index 000000000..77cfe103a --- /dev/null +++ b/src/adhocracy/lib/emailcomments/setupcomment.py @@ -0,0 +1,68 @@ +import logging + +import pylons +from pylons import tmpl_context as c +from pylons import request +from pylons.util import ContextObj, PylonsContext + +from adhocracy.model import instance_filter +from adhocracy.controllers import comment as Comment + +from adhocracy.lib.emailcomments import permissions, util +from adhocracy.lib.search import index + +from webob import Request + +log = logging.getLogger(__name__) + + +def setup_req(): + '''create request''' + request = Request.blank("http://localhost") + pylons.request._push_object(request) + + +def setup_c(user_obj): + ''' + setup the tmpl_context: + sunburnt_connection: solr + user, instance: repoze / permissions + ''' + c = ContextObj() + py_obj = PylonsContext() + py_obj.tmpl_context = c + pylons.tmpl_context._push_object(c) + c = pylons.tmpl_context + + c.instance = instance_filter.get_instance() + c.user = user_obj + c.sunburnt_connection = index.make_connection() + + +def comment(user_obj, comment_obj, text, sentiment): + ''' + Setup of template-ontext, request and permissions for instance/user. + Finally teardown instance, request-environment and template-context. + ''' + instance_filter.setup_thread(comment_obj.topic.instance) + setup_req() + setup_c(user_obj) + permissions.setup_perm(request.environ, c.user) + try: + Comment._create(user=c.user, + reply=comment_obj, + topic=comment_obj.topic, + text=text, + wiki=0, + variant=comment_obj.variant, + instance=c.instance, + sentiment=sentiment) + except Exception as e: + log.info("error in creating comment: {0}".format(e)) + util.error_mail_to_user(401, user_obj, comment_obj) + finally: + instance_filter.teardown_thread() + request.environ = {} + del c.user + del c.instance + del c.sunburnt_connection diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py new file mode 100644 index 000000000..f0ea2adec --- /dev/null +++ b/src/adhocracy/lib/emailcomments/util.py @@ -0,0 +1,276 @@ +import os +import shutil +import logging +import re +from pylons.i18n import _ +from pylons import config +from BeautifulSoup import BeautifulSoup +import markdown + +log = logging.getLogger(__name__) + + +def content_type_reader(content_type): + img_type = re.search(r"image/([^;]*);?", content_type).group(1) + try: + img_name = re.search("name=([^;]*)", + content_type).group(1).rpartition(".")[0] + except: + img_name = u"{0}-image".format(img_type) + return (img_type, img_name) + + +def error_mail_to_user(errcode, to_user, orig_comm): + '''receives error codes if posting fails and send notification to user''' + if errcode == 411: + body = _(u"Error: Your reply must be longer than 4 characters.\r\n") + if errcode == 400: + body = _(u"Error: The email address you replied to is invalid.\r\n") + if errcode == 401: + body = _(u"Error: You lack permissions to post replies via email.\r\n") + + if orig_comm: + br = u"\r\n\r\n" + body = body + _(u"original comment:") + br + orig_comm.latest.text + body = body + br + _(u"by: ") + orig_comm.creator.name + br + else: + body = body + _(u"Original Comment cannot be determined.\r\n") + try: + mail.to_mail(to_user.user_name, + to_user.email, + _(u"Email comment failed!"), + body, + headers={}, + decorate_body=True, + email_from=config.get('error_email_from'), + name_from="EC-Error") + except: + log.error("could not send error mail to user: user undeterminable") + return + log.info("sent error mail to user") + + +def strip_local_part(recipient): + '''get userid, commentid, security token from local part''' + pattern = re.compile(r"""\Asubs\. + (?P[1-9][0-9]*)- + (?P[1-9][0-9]*)\. + (?P[a-fA-F0-9]{40})@""", re.VERBOSE) + result = pattern.match(recipient) + return result + + +def move_overwrite(src, dst): + '''overwrite if a file with same name exists''' + if os.path.exists(dst): + os.remove(dst) + shutil.move(src, dst.rpartition("/")[0]) + + +def get_sentiment(text): + ''' + User can vote via first line of his email: + (Vote: )1 or +1 or 0 or -1 or + or - + The Vote will be extracted and the line will be deleted if present. + Also all leading newlines after vote-line will be deleted. + ''' + pattern = re.compile(ur"""\A(?:[vV]ote:?\s*)? + (?P0|1|\+1?|-1?) + (?:\n|\r|\r\n)+ + (?P(.|\n|\r|\r\n)*) + """, re.VERBOSE) + result = pattern.match(text) + + if result: + text = result.group("text") + sentiment = result.group("sentiment") + if "+" in sentiment or sentiment == "1": + sentiment = 1 + elif "-" in sentiment: + sentiment = -1 + else: + sentiment = 0 + else: + sentiment = None + + return text, sentiment + + +def html_to_markdown(text): + ''' + Basic HTML-Parsing for markdown-output: + Use regex to replace tags for use with markdown. + + tag replacements: +
: + replaced by " \n" + text or text: + replaced by "**text**" + text or text: + replaced by "*text*" + text and
text
: + replaced by "\n> text\n" with spaces for breaks + text: + replaced by "{x times #} text" +
  • text
    • : + replaced by "* text " + descr: + replaced by "[descr](http://link "http://link")" (http added if + required) + alttext: + replaced by "![alttext](http://link "http://link")" (http added if + required) +
  • text
    • : + replaced by "* text " + This is a very basic list parser and returns always an unordered list + without indentations. Deletes before. + and : + replaced by " \n" + other tags: + replaced by "" + ''' + def h_repl(matchobj): + if matchobj: + h = u"" + for i in range(int(matchobj.group(1))): + h = h + u"#" + return h + u" " + else: + return "" + + def a_repl(matchobj): + if matchobj: + try: + href = re.search(u"href=[\"]?([^\">]*)[\"]?", + matchobj.group(0)).group(1) + alt = re.search(u">(.*?)<", + matchobj.group(0)).group(1) + if not "http" in href: + href = u"http://" + href + link = u"[{0}]({1} \"{2}\")".format(alt, href, href) + return link + except: + return "" + else: + return "" + + def img_repl(matchobj): + if matchobj: + try: + src = re.search(u"src=[\"]?([^\">]*)[\"]?", + matchobj.group(0)).group(1) + alt = re.search(u"alt=[\"]?([^\">]*)[\"]?", + matchobj.group(0)).group(1) + if not "http" in src: + src = u"http://" + src + link = u"![{0}]({1} \"{2}\")".format(alt, src, src) + return link + except: + return "" + else: + return "" + + def li_repl(matchobj): + if matchobj: + li = u"* " + matchobj.group(1) + u" \n" + return li + else: + return "" + + def parse_soup(text): + soup = BeautifulSoup(text) + blacklist = ["script", "style"] + keep_attrs = ["a", "img"] + + for tag in soup.findAll(): + if tag.name.lower() in blacklist: + tag.extract() + if not tag.name.lower() in keep_attrs: + tag.attrs = {} + return unicode(soup) + + replacements = [(ur"(\n|\r|\r\n)[\s]*|\t[\s]*", u""), + (ur"\s{2,}", u" "), + (ur">\s*", u">"), + (ur"\s*<", u"<"), + (ur"
    ", u" \n"), + (ur"([\s]*)|()", u"**"), + (ur"([\s]*)|()", u"**"), + (ur"([\s]*)|()", u"*"), + (ur"([\s]*)|()", u"*"), + (ur"|", u" > "), + (ur"|", u" \n\n"), + (ur"", h_repl), + (u"", a_repl), + (ur"", img_repl), + (ur"", u""), + (ur"
  • [\s]*([^<|\\n]*)", li_repl), + (ur"()|()", u" \n"), + (ur"<.*?>", u"")] + + text = parse_soup(text) + + for i, j in replacements: + text = re.sub(i, j, text, flags=re.IGNORECASE) + + return text + + +def delete_debris(text): + ''' + Deletes all ending and leading lines, also shrinks number of lines + to two if more than three are concatenated at once. + ''' + replacements = [ur"(\n|\r|\s)*\Z", + ur"\A(\n|\r|\s)*"] + + for item in replacements: + text = re.sub(item, r"", text) + + text = re.sub(ur"((\n|\r\s*|\r\n\s*){3,})", u"\n\n", text) + + return text + + +def delete_signatures(text): + ''' + Deletes PGP or other signatures: + If text begins with a sequence of 2 "-" or "_" at the beginning + of a new line the rest of the text will be stripped(RFC3676). + If text contains a reference to the PGP-signed message, the line + will be deleted before signatures because of containing "-"s. + Order is crucial: for further replacements add something to the end + of the list. + ''' + pgp_debris = ur"(.*?(\r|\n|\r\n))*?" + pgp_start = ur"-----BEGIN PGP SIGNED MESSAGE-----" + pgp_hash = ur"((\r|\n|\r\n)hash:\s?\w*(\n|\r|\r\n|\s)*)?" + p = re.compile(pgp_debris + pgp_start + pgp_hash, re.IGNORECASE) + + replacements = [p, + ur"(\r|\n|\r\n)[-_]{2,}(.|\n|\r|\r\n)*"] + + for item in replacements: + text = re.sub(item, u"", text) + + return text + + +def create_filesystem(path_md): + '''create directories if possible and required''' + try: + if not os.path.exists(path_md): + os.makedirs(path_md) + elif not os.path.exists(os.path.join(path_md, "new")): + os.makedirs(os.path.join(path_md, "new")) + elif not os.path.exists(os.path.join(path_md, "cur")): + os.makedirs(os.path.join(path_md, "cur")) + elif not os.path.exists(os.path.join(path_md, "tmp")): + os.makedirs(os.path.join(path_md, "tmp")) + elif not os.path.exists("/var/mail"): + os.makedirs("/var/mail") + return True + except IOError: + log.error("you have no permission to create Maildir or mbox") + log.info("emailcomments are disabled") + return False diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 2003d1fbe..30af23458 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -1,8 +1,8 @@ import logging - +import hashlib from pylons import config from webhelpers import text - +from pylons.i18n import _ from adhocracy.lib import mail, microblog TWITTER_LENGTH = 140 @@ -40,6 +40,16 @@ def twitter_sink(pipeline): def mail_sink(pipeline): + """ + Generates email-adress for comment-replies in the form of + subscription.userid-commentid.security-token@domain.tld + if the notification is about a new comment. + HTML will be activated. If a user can not display HTML-emails + the multipart/alternative plaintext-part will display the + default notification.body. Otherwise an HTML-email is only + a beautified version. + This feature is not yet completed. + """ for notification in pipeline: if notification.user.is_email_activated() and \ notification.priority >= notification.user.email_priority: @@ -49,10 +59,37 @@ def mail_sink(pipeline): log.debug("mail to %s: %s" % (notification.user.email, notification.subject)) - mail.to_user(notification.user, - notification.subject, - notification.body, - headers=headers) + if str(notification.event.event) == "t_comment_create" or \ + str(notification.event.event) == "n_comment_reply": + html = False # deactivated due to parsing + secrets = config.get("adhocracy.session.secret") + email_from = config.get("adhocracy.email.from") + + email_domain = email_from.rpartition("@") + email_domain = email_domain[1] + email_domain[2] + + user_id = str(notification.user.id) + + comment_id = notification.link.split("#c")[-1] + + sec_token = user_id + comment_id + secrets + sec_token = hashlib.sha1(sec_token).hexdigest() + + seq = (u" ") + reply_to = "".join(seq) + reply_msg = _(u"\"Reply to leave a comment\"") + reply_to = reply_msg + reply_to + + headers['In-Reply-To'] = reply_to + else: + html = False + + mail.to_user(notification.user, + notification.subject, + notification.body, + headers=headers, + html=html) else: yield notification diff --git a/src/adhocracy/lib/mail.py b/src/adhocracy/lib/mail.py index 3d6ac474c..906ff7422 100644 --- a/src/adhocracy/lib/mail.py +++ b/src/adhocracy/lib/mail.py @@ -1,6 +1,7 @@ import email from email.header import Header from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart import logging import smtplib from time import time @@ -23,7 +24,7 @@ def send(email_from, to_email, message): def to_mail(to_name, to_email, subject, body, headers={}, decorate_body=True, - email_from=None, name_from=None): + email_from=None, name_from=None, html=False): try: if email_from is None: email_from = config.get('adhocracy.email.from') @@ -36,8 +37,17 @@ def to_mail(to_name, to_email, subject, body, headers={}, decorate_body=True, _(u"Cheers,\r\n\r\n" u" the %s Team\r\n") % config.get('adhocracy.site.name')) + if html: # deactivated in mail sinks due to emailcomment-parsing + msg = MIMEMultipart("alternative") + part_plain = MIMEText(body.encode(ENCODING), 'plain', + ENCODING) + html_mail = "html_template" + part_html = MIMEText(html_mail, 'html', ENCODING) - msg = MIMEText(body.encode(ENCODING), 'plain', ENCODING) + msg.attach(part_plain) + msg.attach(part_html) + else: + msg = MIMEText(body.encode(ENCODING), 'plain', ENCODING) for k, v in headers.items(): msg[k] = v @@ -56,9 +66,9 @@ def to_mail(to_name, to_email, subject, body, headers={}, decorate_body=True, def to_user(to_user, subject, body, headers={}, decorate_body=True, - email_from=None, name_from=None): + email_from=None, name_from=None, html=False): return to_mail(to_user.name, to_user.email, subject, body, headers, - decorate_body, email_from, name_from) + decorate_body, email_from, name_from, html) def send_activation_link(user): diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py new file mode 100644 index 000000000..83a40babe --- /dev/null +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -0,0 +1,178 @@ +from adhocracy.tests import TestController +import adhocracy.lib.emailcomments.util +import adhocracy.lib.emailcomments.parseincoming +import adhocracy.lib.text.render +from adhocracy.tests import testtools + +class EmailParsingTest(TestController): + def test_render(self): + '''safe_mode must be False''' + r = adhocracy.lib.text.render + self.assertEqual(r(u"![alt](http://test.com/test.jpg \"title\")"), + u"""

    \"alt\"

    """) + self.assertEqual(r(u"![alt](data:image/png;base64,img \"title\")"), + u"""

    \"alt\"

    """) + + + def test_delete_signatures(self): + ds = adhocracy.lib.emailcomments.util.delete_signatures + self.assertEqual(ds(u'foobar\n__barfoo'), u'foobar') + self.assertEqual(ds(u'foobar\n--barfoo'), u'foobar') + self.assertEqual(ds(u'foobar\n __barfoo'), u'foobar\n __barfoo') + self.assertEqual(ds(u'foobar\n_barfoo'), u'foobar\n_barfoo') + self.assertEqual(ds(u'foobar\n-----BEGIN PGP SIGNED MESSAGE-----\n'), u'\n') + self.assertEqual(ds(u'foobar\n-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\nbarfoo'), u'barfoo') + self.assertEqual(ds(u'foobar\nHash: SHA512\n\nbarfoo'), u'foobar\nHash: SHA512\n\nbarfoo') + self.assertEqual(ds(u""" +some debris... +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +This is the Payload of +the PGP-signed message. + +-----BEGIN PGP SIGNATURE----- +iQA/AwUBONpOg40d+PaAQUTlEQIc5ACdGkKSzpOrsT0Gvj3jH9NXD8ZP2IcAn0vj +/BHT+qQCtPCtCwO1aQ3Xk/NL +=1CZt +-----END PGP SIGNATURE----- + +__________________ +provider signature"""), + +u"""This is the Payload of +the PGP-signed message. +""") + + + def test_html_to_markdown(self): + rp = adhocracy.lib.emailcomments.util.html_to_markdown + self.assertEqual(rp(u'
    '), u' \n') + self.assertEqual(rp(u'\n\t
    \t\t\n\n
    '), u' \n \n') + self.assertEqual(rp(u'foobar
    \n\nbarfoo'), u'foobar \nbarfoo') + self.assertEqual(rp(u' foobar'), u'**foobar**') + self.assertEqual(rp(u'

    foobar

    '), u'## foobar') + self.assertEqual(rp(u'

    foobar

    '), u'## foobar') + self.assertEqual(rp(u'foobar
    \r\n\n\rbarfoo'), u'foobar \nbarfoo') + self.assertEqual(rp(u""" + + + + + + + + +
      +
    • foo +
    • bar
      • +
    +

    bla


    + foo
    + + bar
    +
    + barbar
    + foobar +
    bar
    + barfoo +
    +
    +
    +
    + + + + +"""), + +u"""* foo +* bar + +### bla +![foo](http://foo.bar/bar.gif "http://foo.bar/bar.gif") + + +*bar***bar** + > foobar + + > bar + +[barfoo](http://foobar.de "http://foobar.de") + + + +""") + + + def test_get_sentiment(self): + gs = adhocracy.lib.emailcomments.util.get_sentiment + self.assertEqual(gs(u'vote:+\n'), (u'', 1)) + self.assertEqual(gs(u'vote:+\r\n'), (u'', 1)) + self.assertEqual(gs(u'vote:+\r'), (u'', 1)) + self.assertEqual(gs(u'Vote: 1\nfoo bar'), (u'foo bar', 1)) + self.assertEqual(gs(u'Vote: +1\r\nfoo\nbar'), (u'foo\nbar', 1)) + self.assertEqual(gs(u'0\na'), (u'a', 0)) + self.assertEqual(gs(u'-\nfoo'), (u'foo', -1)) + self.assertEqual(gs(u'anything'), (u'anything', None)) + self.assertEqual(gs(u'Vote: -1\n'), (u'', -1)) + self.assertEqual(gs(u'Vote -1\n'), (u'', -1)) + self.assertEqual(gs(u'2010\n'), (u'2010\n', None)) + self.assertEqual(gs(u'010\n'), (u'010\n', None)) + self.assertEqual(gs(u"""vote + +The Message"""), +(u"""The Message""", 1)) + + + def test_content_type_reader(self): + ctr = adhocracy.lib.emailcomments.util.content_type_reader + self.assertEqual(ctr(u'image/png'), (u'png', u'png-image')) + self.assertEqual(ctr(u'image/png; name=foobar.png'), (u'png', u'foobar')) + self.assertEqual(ctr(u'image/png; name=bar.foobar.png'), (u'png', u'bar.foobar')) + + + def test_delete_debris(self): + dd = adhocracy.lib.emailcomments.util.delete_debris + self.assertEqual(dd(u'foobar\n\n\n\n'), u'foobar') + self.assertEqual(dd(u'\n\nfoobar\n\n'), u'foobar') + self.assertEqual(dd(u'\n foobar\n '), u'foobar') + self.assertEqual(dd(u' \nfoobar \n'), u'foobar') + self.assertEqual(dd(u' \nfoobar \n'), u'foobar') + self.assertEqual(dd(u'\r\n \n\nfoobar \r\n \n\n'), u'foobar') + self.assertEqual(dd(u'foo\n\n\n\n\nbar'), u'foo\n\nbar') + self.assertEqual(dd(u' \n foo\r\n \n \n \n\r\nbar \n '), u'foo\n\nbar') + + + def test_parse_local_part(self): + '''test for admin''' + from pylons import config + secrets = config.get("adhocracy.session.secret") + + comment = testtools.tt_make_comment() + + reply_id = unicode(comment.id) + + import hashlib + sec_token = hashlib.sha1(u"1" + reply_id + secrets).hexdigest() + + test_string = (u'subs.1-{0}.{1}@domain.tld').format(reply_id, + sec_token) + + fail_string = (u'subs.1-{0}.MOCKSECRET@domain.tld').format(reply_id) + + from adhocracy.model import User + user = User.find(1) + + plp = adhocracy.lib.emailcomments.parseincoming.parse_local_part + self.assertEqual(plp(u'subs.50-50.foobar@domain.tld'), None) + self.assertEqual(plp(u'subs.1-1.feb340279618fb47d6c0feb340279618fb47d6c0@domain.tld'),None) + self.assertEqual(plp(test_string), (user, comment)) + self.assertEqual(plp(fail_string), None) diff --git a/src/adhocracy/tests/testtools.py b/src/adhocracy/tests/testtools.py index c5317358d..07c0f9012 100644 --- a/src/adhocracy/tests/testtools.py +++ b/src/adhocracy/tests/testtools.py @@ -97,6 +97,20 @@ def tt_make_user(name=None, instance_group=None): return user +def tt_make_comment(proposal=None, creator=None): + if proposal is None: + proposal = tt_make_proposal() + if creator is None: + creator = tt_make_user("DeKue") + + comment = model.Comment.create( + tt_make_str(), + creator, + proposal) + + return comment + + def tt_drop_db(): ''' drop the data tables and if exists the migrate_version table From b7f80f340c04a68c391e26765370b05832b1aa70 Mon Sep 17 00:00:00 2001 From: Denis Date: Fri, 2 Aug 2013 00:47:59 +0200 Subject: [PATCH 05/21] Delete adhocracy --- adhocracy | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 adhocracy diff --git a/adhocracy b/adhocracy deleted file mode 100644 index 8b5c7f68d..000000000 --- a/adhocracy +++ /dev/null @@ -1,9 +0,0 @@ -tree e981b8f97ef54808aae69bfb8d145108b8062c8a -parent ce1d855668eb1a938cece2b5bfc3666c1fee82f0 -author Nicolas Dietrich 1375302727 +0200 -committer Nicolas Dietrich 1375302727 +0200 - -Remove unneeded file again - -templates/user/edit.html has been accidently readded in -03f382dac7e0248f221a94f51854c8f79191d961. From 99fef80bc3afa76f8c67932bc4a155b2d0828df7 Mon Sep 17 00:00:00 2001 From: Dekue Date: Sun, 4 Aug 2013 17:07:56 +0200 Subject: [PATCH 06/21] sinks-update for better view of notification emails about new comments --- src/adhocracy/lib/event/notification/sinks.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 30af23458..c851f9b2b 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -60,9 +60,11 @@ def mail_sink(pipeline): log.debug("mail to %s: %s" % (notification.user.email, notification.subject)) + notification_body = notification.body + if str(notification.event.event) == "t_comment_create" or \ str(notification.event.event) == "n_comment_reply": - html = False # deactivated due to parsing + secrets = config.get("adhocracy.session.secret") email_from = config.get("adhocracy.email.from") @@ -83,13 +85,36 @@ def mail_sink(pipeline): reply_to = reply_msg + reply_to headers['In-Reply-To'] = reply_to + + vote_line = (u"vote 0\r\n\r\n" + _(u"Type your answer here.") + + u"\r\n\r\n_________________________\r\n") + + notification_body = (_(u"comment by replying to this email.") + + u"\r\n" + _(u"Write your answer above the upper line.") + + u"\r\n\r\n" + _(u"You can use the first line of your reply") + + u" " + _(u"to vote. Type vote 1 for a positive vote,") + + u" " + _(u"vote 0 for a neutral vote and vote -1 for a") + + u" " + _(u"negative vote\r\n\r\n") + notification_body) + + notification_body = vote_line + (_(u"Hi %s,") % + notification.user.name + + u"\r\n%s\r\n\r\n" % notification_body + + _(u"Cheers,\r\n\r\n" + u" the %s Team\r\n") % + config.get('adhocracy.site.name')) + + html = False + # html deactivated due to email-comment parsing/voting + decorate_body = False else: html = False + decorate_body = True mail.to_user(notification.user, notification.subject, - notification.body, + notification_body, headers=headers, - html=html) + html=html, + decorate_body=decorate_body) else: yield notification From 114f2432f348089762ee628fe050006b295c19f6 Mon Sep 17 00:00:00 2001 From: Denis Date: Mon, 5 Aug 2013 23:16:57 +0200 Subject: [PATCH 07/21] Update sinks.py Updated sinks for better translation possibilities. --- src/adhocracy/lib/event/notification/sinks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index c851f9b2b..81b78742e 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -89,12 +89,12 @@ def mail_sink(pipeline): vote_line = (u"vote 0\r\n\r\n" + _(u"Type your answer here.") + u"\r\n\r\n_________________________\r\n") - notification_body = (_(u"comment by replying to this email.") + - u"\r\n" + _(u"Write your answer above the upper line.") + - u"\r\n\r\n" + _(u"You can use the first line of your reply") + - u" " + _(u"to vote. Type vote 1 for a positive vote,") + - u" " + _(u"vote 0 for a neutral vote and vote -1 for a") + - u" " + _(u"negative vote\r\n\r\n") + notification_body) + notification_body = _((u"comment by replying to this email." + u"\r\nWrite your answer above the upper line." + u"\r\n\r\nYou can use the first line of your reply" + u" to vote. Type vote 1 for a positive vote," + u" vote 0 for a neutral vote and vote -1 for a" + u" negative vote.\r\n\r\n") + notification_body) notification_body = vote_line + (_(u"Hi %s,") % notification.user.name + From 29e829a0f3cbc28ae853b6c4f7e06b87f81a8fa8 Mon Sep 17 00:00:00 2001 From: Denis Date: Mon, 5 Aug 2013 23:17:26 +0200 Subject: [PATCH 08/21] Update sinks.py --- src/adhocracy/lib/event/notification/sinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 81b78742e..03bb06451 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -89,7 +89,7 @@ def mail_sink(pipeline): vote_line = (u"vote 0\r\n\r\n" + _(u"Type your answer here.") + u"\r\n\r\n_________________________\r\n") - notification_body = _((u"comment by replying to this email." + notification_body = (_(u"comment by replying to this email." u"\r\nWrite your answer above the upper line." u"\r\n\r\nYou can use the first line of your reply" u" to vote. Type vote 1 for a positive vote," From 974783dbf048574a874a28a40cb8010c17d1b0c3 Mon Sep 17 00:00:00 2001 From: Denis Date: Wed, 7 Aug 2013 00:27:23 +0200 Subject: [PATCH 09/21] Update localwatch.py removed thread lock --- src/adhocracy/lib/emailcomments/localwatch.py | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/localwatch.py b/src/adhocracy/lib/emailcomments/localwatch.py index 53c56b336..603ed66e2 100644 --- a/src/adhocracy/lib/emailcomments/localwatch.py +++ b/src/adhocracy/lib/emailcomments/localwatch.py @@ -1,4 +1,3 @@ -import threading import pyinotify import logging import os @@ -14,28 +13,23 @@ def maildir(path, ecq): '''gets payload and recipient of a new Maildir mail''' - lockmd = threading.Lock() - time.sleep(1) # pause - else mail isn't yet recognized in FS + time.sleep(1) # pause - else mail isn't yet in tmp log.info("new mail in Maildir") - lockmd.acquire() - try: - for filename in os.listdir(os.path.join(path, "new")): - file_path = os.path.join(path, "new", filename) - mail = open(file_path, "r") - parser = email.Parser.Parser() - message = parser.parse(mail) - mail.close() - ecq.put(message) - util.move_overwrite(file_path, - os.path.join(path, "cur", filename)) - finally: - lockmd.release() + for filename in os.listdir(os.path.join(path, "new")): + file_path = os.path.join(path, "new", filename) + mail = open(file_path, "r") + parser = email.Parser.Parser() + message = parser.parse(mail) + mail.close() + ecq.put(message) + util.move_overwrite(file_path, + os.path.join(path, "cur", filename)) def mbox(path, ecq): '''gets payload and recipient of a new mbox mail''' - time.sleep(1) # pause - else mail isn't yet recognized in FS + time.sleep(1) # pause - MTA could do something yet log.info("new mail in mbox") tmp_mb = os.path.join("/var/tmp", str(datetime.now())) From f1b53130e3a332e29f75de8124d0aea12b5eda39 Mon Sep 17 00:00:00 2001 From: Denis Date: Wed, 7 Aug 2013 01:38:47 +0200 Subject: [PATCH 10/21] Update util.py Updated debris - function. If more than three breaks follow each other they will be no longer shrinked to two breaks. --- src/adhocracy/lib/emailcomments/util.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index f0ea2adec..24f53697b 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -217,18 +217,13 @@ def parse_soup(text): def delete_debris(text): - ''' - Deletes all ending and leading lines, also shrinks number of lines - to two if more than three are concatenated at once. - ''' + '''deletes all ending and leading lines''' replacements = [ur"(\n|\r|\s)*\Z", ur"\A(\n|\r|\s)*"] for item in replacements: text = re.sub(item, r"", text) - text = re.sub(ur"((\n|\r\s*|\r\n\s*){3,})", u"\n\n", text) - return text From 8e94887fda8c89154f68d826ee9e5e22655b9e12 Mon Sep 17 00:00:00 2001 From: Dekue Date: Wed, 7 Aug 2013 01:46:30 +0200 Subject: [PATCH 11/21] Deleted Ddebris function completely. Had no use. --- src/adhocracy/lib/emailcomments/parseincoming.py | 1 - src/adhocracy/lib/emailcomments/util.py | 11 ----------- src/adhocracy/tests/lib/test_email_parsing.py | 12 ------------ 3 files changed, 24 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index dfb9ff9ae..1381bcc0b 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -105,7 +105,6 @@ def parse_payload(message): text = "" text = util.delete_signatures(text) text, sentiment = util.get_sentiment(text) - text = util.delete_debris(text) return (text, sentiment) diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index 24f53697b..68e504a9e 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -216,17 +216,6 @@ def parse_soup(text): return text -def delete_debris(text): - '''deletes all ending and leading lines''' - replacements = [ur"(\n|\r|\s)*\Z", - ur"\A(\n|\r|\s)*"] - - for item in replacements: - text = re.sub(item, r"", text) - - return text - - def delete_signatures(text): ''' Deletes PGP or other signatures: diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py index 83a40babe..06c0f26b5 100644 --- a/src/adhocracy/tests/lib/test_email_parsing.py +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -139,18 +139,6 @@ def test_content_type_reader(self): self.assertEqual(ctr(u'image/png; name=bar.foobar.png'), (u'png', u'bar.foobar')) - def test_delete_debris(self): - dd = adhocracy.lib.emailcomments.util.delete_debris - self.assertEqual(dd(u'foobar\n\n\n\n'), u'foobar') - self.assertEqual(dd(u'\n\nfoobar\n\n'), u'foobar') - self.assertEqual(dd(u'\n foobar\n '), u'foobar') - self.assertEqual(dd(u' \nfoobar \n'), u'foobar') - self.assertEqual(dd(u' \nfoobar \n'), u'foobar') - self.assertEqual(dd(u'\r\n \n\nfoobar \r\n \n\n'), u'foobar') - self.assertEqual(dd(u'foo\n\n\n\n\nbar'), u'foo\n\nbar') - self.assertEqual(dd(u' \n foo\r\n \n \n \n\r\nbar \n '), u'foo\n\nbar') - - def test_parse_local_part(self): '''test for admin''' from pylons import config From 305a6ebccac86164332214c92f1b694b902ef796 Mon Sep 17 00:00:00 2001 From: Denis Date: Fri, 9 Aug 2013 05:05:26 +0200 Subject: [PATCH 12/21] Updated notification info for email-comment replies --- src/adhocracy/lib/event/notification/sinks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 03bb06451..7dceef3bc 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -102,6 +102,9 @@ def mail_sink(pipeline): _(u"Cheers,\r\n\r\n" u" the %s Team\r\n") % config.get('adhocracy.site.name')) + + notification_body = notification_body + u"\n\n\n" + (_("Please" + u" write your answer above the upper line.")) html = False # html deactivated due to email-comment parsing/voting From 1109a8e5d8738ec76ca40670f4c379768cdc9569 Mon Sep 17 00:00:00 2001 From: Dekue Date: Fri, 9 Aug 2013 22:12:10 +0200 Subject: [PATCH 13/21] Checks if the same comment was posted by the same user to the same topic in the last two minutes. This could happen if the same inbox is watched multiple times so the program gets multiple parse commands for the same email. --- .../lib/emailcomments/setupcomment.py | 3 ++ src/adhocracy/lib/emailcomments/util.py | 41 ++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/adhocracy/lib/emailcomments/setupcomment.py b/src/adhocracy/lib/emailcomments/setupcomment.py index 77cfe103a..402a7ed12 100644 --- a/src/adhocracy/lib/emailcomments/setupcomment.py +++ b/src/adhocracy/lib/emailcomments/setupcomment.py @@ -41,9 +41,12 @@ def setup_c(user_obj): def comment(user_obj, comment_obj, text, sentiment): ''' + If comment does not already exist (inbox sync): Setup of template-ontext, request and permissions for instance/user. Finally teardown instance, request-environment and template-context. ''' + if not util.comment_exists(text, user_obj, comment_obj.id): + return instance_filter.setup_thread(comment_obj.topic.instance) setup_req() setup_c(user_obj) diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index 68e504a9e..cd86ef81d 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -2,14 +2,42 @@ import shutil import logging import re +import markdown from pylons.i18n import _ from pylons import config from BeautifulSoup import BeautifulSoup -import markdown +from datetime import datetime log = logging.getLogger(__name__) +def comment_exists(text, user, reply_id): + ''' + Tries to determine if a comment already exists. This can be useful if + more than one worker watches the same Inbox and an email is fetched + multiple times. The last comments made by the user in a specific time + are checked on the same topic and the text itself. + If a user replied the same to the same topic in the last two minutes + the reply will be ignored. + ''' + now = datetime.utcnow() + + i = 0 + for x in user.comments: + time_diff = now - user.comments[i].create_time + if time_diff.seconds < 120 and time_diff.days == 0: + cmp_text = user.comments[i].revisions[0].text + cmp_reply_id = user.comments[i].reply_id + if reply_id == cmp_reply_id: + if cmp_text == text: + return False + else: + return True + i += 1 + + return True + + def content_type_reader(content_type): img_type = re.search(r"image/([^;]*);?", content_type).group(1) try: @@ -216,6 +244,17 @@ def parse_soup(text): return text +def delete_debris(text): + '''deletes all ending and leading lines''' + replacements = [ur"(\n|\r|\s)*\Z", + ur"\A(\n|\r|\s)*"] + + for item in replacements: + text = re.sub(item, r"", text) + + return text + + def delete_signatures(text): ''' Deletes PGP or other signatures: From deffe82402da8dd8e340e8bfeffafa94257cc42e Mon Sep 17 00:00:00 2001 From: Dekue Date: Sat, 10 Aug 2013 05:00:55 +0200 Subject: [PATCH 14/21] updated secret-usage; reintegrated derbis function vor comparison between revisions in DB and recently parsed emails --- buildouts/adhocracy.cfg | 4 +++- etc/adhocracy.ini.in | 8 +++---- etc/test.ini.in | 15 ++++++------ setup.py | 6 ++--- src/adhocracy/lib/cli.py | 24 +++++++++++++++---- .../lib/emailcomments/parseincoming.py | 3 ++- src/adhocracy/lib/emailcomments/util.py | 7 +++++- src/adhocracy/lib/event/notification/sinks.py | 4 ++-- src/adhocracy/tests/lib/test_email_parsing.py | 14 ++++++++++- 9 files changed, 59 insertions(+), 26 deletions(-) diff --git a/buildouts/adhocracy.cfg b/buildouts/adhocracy.cfg index d730ae3fc..bef4c54bd 100644 --- a/buildouts/adhocracy.cfg +++ b/buildouts/adhocracy.cfg @@ -24,8 +24,10 @@ auto-checkout = * # supervisor config to start adhocracy adhocracy_worker-supervisor = 40 adhocracy_worker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_worker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy worker -c ${buildout:directory}/etc/adhocracy.ini] + adhocracy_ecworker-supervisor = 40 adhocracy_ecworker (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy_ecworker.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [--plugin=adhocracy ecworker -c ${buildout:directory}/etc/adhocracy.ini] + adhocracy-supervisor = 45 adhocracy (environment=${supervisor:environment} redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/paster [serve ${buildout:directory}/etc/adhocracy.ini] # parts in this buildout file to be installed @@ -51,7 +53,7 @@ supervisor = 5010 ############################################################################## # Adhocracy settings -####################################### ${buildout:adhocracy_ecworker-supervisor}####################################### +############################################################################## [adhocracy] #start adhocracy in debug mode diff --git a/etc/adhocracy.ini.in b/etc/adhocracy.ini.in index 3c39a035d..daf71a5b2 100644 --- a/etc/adhocracy.ini.in +++ b/etc/adhocracy.ini.in @@ -104,9 +104,9 @@ static_files = true # INSTALL: To enable set email_src to 'local' or 'imap', IMAP-SSL is essential. # If 'imap_account = user' fails, try complete address: 'user@emailprovider.tld'. -# Set local_user to determine the standard-mbox/Maildir-folders you want to watch. -email_src = None -local_user = None +# Set local_user to determine the mbox/Maildir-folders you want to watch. +email_src = none +local_user = user imap_domain = imap.emailprovider.tld imap_account = user imap_password = password @@ -476,4 +476,4 @@ format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s # Custom OVERRIDE adhocracy settings ############################################################## -${parts.adhocracy['settings_override']} +${parts.adhocracy['settings_override']} diff --git a/etc/test.ini.in b/etc/test.ini.in index 9df9b6005..dbff89a7b 100644 --- a/etc/test.ini.in +++ b/etc/test.ini.in @@ -47,13 +47,6 @@ memcached.server = 127.0.0.1:${parts.ports.memcached} #solr adhocracy.solr.url = http://${parts.solr.host}:${parts.solr.port}/solr -{% python - import random; - def randomhash(length): - return hex(random.SystemRandom().getrandbits(length)) -%} -#adhocracy session secret -adhocracy.session.secret = {% if parts.adhocracy.secret == 'autogenerated' %}${randomhash(256)}{% end %}{% if parts.adhocracy.secret != 'autogenerated' %}${parts.adhocracy.secret}{% end %} # beaker beaker.session.key = adhocracy_state @@ -64,7 +57,13 @@ beaker.cache.data_dir = ${parts.buildout.directory}/var/testdata/cache beaker.session.data_dir = ${parts.buildout.directory}/var/testdata/sessions # Turn on all the bells and whistles -adhocracy.crypto.secret = geheim! +{% python + import random; + def randomhash(length): + return hex(random.SystemRandom().getrandbits(length)) +%} + +adhocracy.crypto.secret = {% if parts.adhocracy.secret == 'autogenerated' %}${randomhash(256)}{% end %}{% if parts.adhocracy.secret != 'autogenerated' %}${parts.adhocracy.secret}{% end %} adhocracy.track_outgoing_links = True adhocracy.login_type = openid,username+password,email+password,shibboleth diff --git a/setup.py b/setup.py index 751b05a07..c4d9bb5b3 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,7 @@ "python-openid>=2.2.4", "python-memcached>=1.45", "sunburnt==0.6", - #"Pillow", # use the adhocracy buildout or install system packages - # (python-imaging) for this dependency + "Pillow", "Markdown>=2.3", "lxml>=2.2.6", "Mako>=0.7.3", @@ -95,7 +94,7 @@ 'pytest-pep8', 'cssselect', 'decorator', - 'pep8'] + 'pep8',] }, package_data={'adhocracy': ['i18n/*/LC_MESSAGES/*.mo'], '': ['RELEASE-VERSION'], @@ -122,6 +121,7 @@ ], 'fanstatic.libraries': [ 'stylesheets = adhocracy.static:stylesheets_library', + 'yaml = adhocracy.static:yaml_library', 'autocomplete = adhocracy.static:autocomplete_library', 'placeholder = adhocracy.static:placeholder_library', 'jquerytools = adhocracy.static:jquerytools_library', diff --git a/src/adhocracy/lib/cli.py b/src/adhocracy/lib/cli.py index 496cc9595..f0ee4b125 100644 --- a/src/adhocracy/lib/cli.py +++ b/src/adhocracy/lib/cli.py @@ -17,6 +17,8 @@ from adhocracy.lib import search from adhocracy.lib import queue +from adhocracy.lib.emailcomments import mail_watch + log = getLogger(__name__) @@ -24,7 +26,7 @@ class AdhocracyCommand(Command): parser = Command.standard_parser(verbose=True) parser.add_option('-c', '--config', dest='config', - default='etc/adhocracy.ini', help='Config file to use.') + default='etc/adhocracy.ini', help='Config file to use.') default_verbosity = 1 group_name = 'adhocracy' @@ -195,6 +197,18 @@ def command(self): worker.work() +class EmailCommentWorker(AdhocracyCommand): + '''Run emailcomment background jobs.''' + summary = __doc__.split('\n')[0] + usage = __doc__ + max_args = None + min_args = None + + def command(self): + self._load_config() + mail_watch() + + class Index(AdhocracyCommand): """Re-create Adhocracy's search index.""" summary = __doc__.split('\n')[0] @@ -278,11 +292,11 @@ def start(self, actions, classes, instances): print ('Starting.\n' ' Actions: %s\n' ' Content Types: %s\n' - ' Instances: %s\n' % ( + ' Instances: %s\n') % ( self.printable(actions), self.printable(classes, print_=lambda x: x.__name__.lower()), - self.printable(instances, print_=lambda x: x.key))) + self.printable(instances, print_=lambda x: x.key)) if self.DROP in actions: p_instances = instances if instances else [None] @@ -313,8 +327,8 @@ def usage(self): indexed_classes = sorted(self.indexed_classes.keys()) content_types = '\n '.join(indexed_classes) usage += ( - 'index (INDEX|DROP|DROP_ALL|ALL) [, ...] [-I , ' - '...] -c ' + 'index (INDEX|DROP|DROP_ALL|ALL) [, ...] [-I , ...]' + ' -c ' '\n\n' ' DROP_ALL:\n' ' Remove all documents from solr.\n' diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index 1381bcc0b..401be48a1 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -105,6 +105,7 @@ def parse_payload(message): text = "" text = util.delete_signatures(text) text, sentiment = util.get_sentiment(text) + text = util.delete_debris(text) return (text, sentiment) @@ -116,7 +117,7 @@ def parse_local_part(recipient): sec_token = result.group("sectoken") - secrets = config.get("adhocracy.session.secret") + secrets = config.get("adhocracy.crypto.secret") comp_str = result.group("userid") + result.group("commentid") comp_str = hashlib.sha1(comp_str + secrets).hexdigest() diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index cd86ef81d..a18a51d3b 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -245,13 +245,18 @@ def parse_soup(text): def delete_debris(text): - '''deletes all ending and leading lines''' + ''' + Deletes all ending and leading lines, also shrinks number of lines + to two if more than three are concatenated at once. + ''' replacements = [ur"(\n|\r|\s)*\Z", ur"\A(\n|\r|\s)*"] for item in replacements: text = re.sub(item, r"", text) + text = re.sub(ur"((\n|\r\s*|\r\n\s*){3,})", u"\n\n", text) + return text diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 7dceef3bc..9527f8ea7 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -65,7 +65,7 @@ def mail_sink(pipeline): if str(notification.event.event) == "t_comment_create" or \ str(notification.event.event) == "n_comment_reply": - secrets = config.get("adhocracy.session.secret") + secrets = config.get("adhocracy.crypto.secret") email_from = config.get("adhocracy.email.from") email_domain = email_from.rpartition("@") @@ -102,7 +102,7 @@ def mail_sink(pipeline): _(u"Cheers,\r\n\r\n" u" the %s Team\r\n") % config.get('adhocracy.site.name')) - + notification_body = notification_body + u"\n\n\n" + (_("Please" u" write your answer above the upper line.")) diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py index 06c0f26b5..84a66fb75 100644 --- a/src/adhocracy/tests/lib/test_email_parsing.py +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -139,10 +139,22 @@ def test_content_type_reader(self): self.assertEqual(ctr(u'image/png; name=bar.foobar.png'), (u'png', u'bar.foobar')) + def test_delete_debris(self): + dd = adhocracy.lib.emailcomments.util.delete_debris + self.assertEqual(dd(u'foobar\n\n\n\n'), u'foobar') + self.assertEqual(dd(u'\n\nfoobar\n\n'), u'foobar') + self.assertEqual(dd(u'\n foobar\n '), u'foobar') + self.assertEqual(dd(u' \nfoobar \n'), u'foobar') + self.assertEqual(dd(u' \nfoobar \n'), u'foobar') + self.assertEqual(dd(u'\r\n \n\nfoobar \r\n \n\n'), u'foobar') + self.assertEqual(dd(u'foo\n\n\n\n\nbar'), u'foo\n\nbar') + self.assertEqual(dd(u' \n foo\r\n \n \n \n\r\nbar \n '), u'foo\n\nbar') + + def test_parse_local_part(self): '''test for admin''' from pylons import config - secrets = config.get("adhocracy.session.secret") + secrets = config.get("adhocracy.crypto.secret") comment = testtools.tt_make_comment() From 235f9dfbb352f2c048476b7a40e219a71b3b67e5 Mon Sep 17 00:00:00 2001 From: Dekue Date: Sat, 10 Aug 2013 05:04:34 +0200 Subject: [PATCH 15/21] deleted useless import --- src/adhocracy/lib/emailcomments/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index a18a51d3b..268bb8257 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -2,7 +2,6 @@ import shutil import logging import re -import markdown from pylons.i18n import _ from pylons import config from BeautifulSoup import BeautifulSoup From d703e8f7e1a7aa8c9b9889e1446946cd94a85ac7 Mon Sep 17 00:00:00 2001 From: Dekue Date: Thu, 15 Aug 2013 22:26:59 +0200 Subject: [PATCH 16/21] Added IMAP-reconnetion if connection was lost, changed comment_exists-function --- src/adhocracy/lib/emailcomments/imap.py | 2 ++ src/adhocracy/lib/emailcomments/setupcomment.py | 2 +- src/adhocracy/lib/emailcomments/util.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/imap.py b/src/adhocracy/lib/emailcomments/imap.py index 399cd7edd..fa2668657 100644 --- a/src/adhocracy/lib/emailcomments/imap.py +++ b/src/adhocracy/lib/emailcomments/imap.py @@ -45,6 +45,7 @@ def run(self): continue recon_interval = 30 + log.info("IMAP-connection established") while True: try: @@ -52,6 +53,7 @@ def run(self): self.conn.idle() except self.conn.abort: log.info("IMAP-connection lost, reconnecting...") + break def dosync(self): '''executed if a new mail arrives''' diff --git a/src/adhocracy/lib/emailcomments/setupcomment.py b/src/adhocracy/lib/emailcomments/setupcomment.py index 402a7ed12..1732070bb 100644 --- a/src/adhocracy/lib/emailcomments/setupcomment.py +++ b/src/adhocracy/lib/emailcomments/setupcomment.py @@ -45,7 +45,7 @@ def comment(user_obj, comment_obj, text, sentiment): Setup of template-ontext, request and permissions for instance/user. Finally teardown instance, request-environment and template-context. ''' - if not util.comment_exists(text, user_obj, comment_obj.id): + if util.comment_exists(text, user_obj, comment_obj.id): return instance_filter.setup_thread(comment_obj.topic.instance) setup_req() diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index 268bb8257..7ea49e5f8 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -29,9 +29,9 @@ def comment_exists(text, user, reply_id): cmp_reply_id = user.comments[i].reply_id if reply_id == cmp_reply_id: if cmp_text == text: - return False + return True else: - return True + return False i += 1 return True From b85cfd5054cba0bb93fd24b4640878ee0a702fea Mon Sep 17 00:00:00 2001 From: Dekue Date: Sat, 24 Aug 2013 01:32:14 +0200 Subject: [PATCH 17/21] enhanced functions for MUAs: remove notification-quote + updated tests file --- .../lib/emailcomments/parseincoming.py | 4 +- src/adhocracy/lib/emailcomments/util.py | 6 +++ src/adhocracy/lib/event/notification/sinks.py | 20 +++++----- src/adhocracy/tests/lib/test_email_parsing.py | 37 ++++++++++++++++++- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index 401be48a1..8aae4a33b 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -94,6 +94,7 @@ def parse_payload(message): else: content_list = parse_multipart(message, "na", 0) text = get_usable_content(content_list) + text = util.remove_notification(text) text = util.delete_signatures(text) else: if "text/plain" in message["Content-Type"]: @@ -103,6 +104,7 @@ def parse_payload(message): text = util.html_to_markdown(text) else: text = "" + text = util.remove_notification(text) text = util.delete_signatures(text) text, sentiment = util.get_sentiment(text) text = util.delete_debris(text) @@ -117,7 +119,7 @@ def parse_local_part(recipient): sec_token = result.group("sectoken") - secrets = config.get("adhocracy.crypto.secret") + secrets = config.get("adhocracy.session.secret") comp_str = result.group("userid") + result.group("commentid") comp_str = hashlib.sha1(comp_str + secrets).hexdigest() diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index 7ea49e5f8..b9a31e085 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -87,6 +87,12 @@ def strip_local_part(recipient): return result +def remove_notification(text): + '''removes notification if user didn't''' + text = re.sub(ur">*\s*_{33}(\r|\n|\r\n|.)*_{33}", u"", text) + return text + + def move_overwrite(src, dst): '''overwrite if a file with same name exists''' if os.path.exists(dst): diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 9527f8ea7..60310ca44 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -65,7 +65,7 @@ def mail_sink(pipeline): if str(notification.event.event) == "t_comment_create" or \ str(notification.event.event) == "n_comment_reply": - secrets = config.get("adhocracy.crypto.secret") + secrets = config.get("adhocracy.session.secret") email_from = config.get("adhocracy.email.from") email_domain = email_from.rpartition("@") @@ -85,33 +85,31 @@ def mail_sink(pipeline): reply_to = reply_msg + reply_to headers['In-Reply-To'] = reply_to + headers['Reply-To'] = reply_to - vote_line = (u"vote 0\r\n\r\n" + _(u"Type your answer here.") + - u"\r\n\r\n_________________________\r\n") + boundary = (u"_________________________________\r\n") - notification_body = (_(u"comment by replying to this email." - u"\r\nWrite your answer above the upper line." + notification_body = (_(u"Comment by replying to this email." + u"\r\nWrite your answer above the upper line or under the" + u" bottom line." u"\r\n\r\nYou can use the first line of your reply" u" to vote. Type vote 1 for a positive vote," u" vote 0 for a neutral vote and vote -1 for a" u" negative vote.\r\n\r\n") + notification_body) + # Mention images? (Don't work with markdown safe_mode yet) - notification_body = vote_line + (_(u"Hi %s,") % + notification_body = boundary + (_(u"Hi %s,") % notification.user.name + u"\r\n%s\r\n\r\n" % notification_body + _(u"Cheers,\r\n\r\n" u" the %s Team\r\n") % - config.get('adhocracy.site.name')) - - notification_body = notification_body + u"\n\n\n" + (_("Please" - u" write your answer above the upper line.")) + config.get('adhocracy.site.name')) + u"\r\n" + boundary html = False # html deactivated due to email-comment parsing/voting decorate_body = False else: html = False - decorate_body = True mail.to_user(notification.user, notification.subject, diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py index 84a66fb75..141f26c8d 100644 --- a/src/adhocracy/tests/lib/test_email_parsing.py +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -154,7 +154,7 @@ def test_delete_debris(self): def test_parse_local_part(self): '''test for admin''' from pylons import config - secrets = config.get("adhocracy.crypto.secret") + secrets = config.get("adhocracy.session.secret") comment = testtools.tt_make_comment() @@ -176,3 +176,38 @@ def test_parse_local_part(self): self.assertEqual(plp(u'subs.1-1.feb340279618fb47d6c0feb340279618fb47d6c0@domain.tld'),None) self.assertEqual(plp(test_string), (user, comment)) self.assertEqual(plp(fail_string), None) + + + def test_remove_notification(self): + rm = adhocracy.lib.emailcomments.util.remove_notification + self.assertEqual(rm(u"""first new line +_________________________________ +some text here +some text there +_________________________________ +some new line"""), u"""first new line +some new line""") + self.assertEqual(rm(u"""first new line +>_________________________________ +>some text here +>some text there +>_________________________________ +some new line"""), u"""first new line + +some new line""") + self.assertEqual(rm(u"""first new line +> _________________________________ +> some text here +> some text there +> _________________________________ +some new line"""), u"""first new line + +some new line""") + self.assertEqual(rm(u"""first new line +foobar> _________________________________ +> some text here +> some text there +> _________________________________ +some new line"""), u"""first new line +foobar +some new line""") From f9bb5d083a291c446012c30820bb5e0814826d1c Mon Sep 17 00:00:00 2001 From: Dekue Date: Sun, 25 Aug 2013 16:50:53 +0200 Subject: [PATCH 18/21] FIX: crypto secret reenabled --- src/adhocracy/lib/emailcomments/parseincoming.py | 4 ++-- src/adhocracy/lib/event/notification/sinks.py | 2 +- src/adhocracy/tests/lib/test_email_parsing.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index 8aae4a33b..a30be7674 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -118,8 +118,8 @@ def parse_local_part(recipient): return None sec_token = result.group("sectoken") - - secrets = config.get("adhocracy.session.secret") +c + secrets = config.get("adhocracy.crypto.secret") comp_str = result.group("userid") + result.group("commentid") comp_str = hashlib.sha1(comp_str + secrets).hexdigest() diff --git a/src/adhocracy/lib/event/notification/sinks.py b/src/adhocracy/lib/event/notification/sinks.py index 60310ca44..b543d7451 100644 --- a/src/adhocracy/lib/event/notification/sinks.py +++ b/src/adhocracy/lib/event/notification/sinks.py @@ -65,7 +65,7 @@ def mail_sink(pipeline): if str(notification.event.event) == "t_comment_create" or \ str(notification.event.event) == "n_comment_reply": - secrets = config.get("adhocracy.session.secret") + secrets = config.get("adhocracy.crypto.secret") email_from = config.get("adhocracy.email.from") email_domain = email_from.rpartition("@") diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py index 141f26c8d..cf7365334 100644 --- a/src/adhocracy/tests/lib/test_email_parsing.py +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -154,7 +154,7 @@ def test_delete_debris(self): def test_parse_local_part(self): '''test for admin''' from pylons import config - secrets = config.get("adhocracy.session.secret") + secrets = config.get("adhocracy.crypto.secret") comment = testtools.tt_make_comment() From 1fe5bbe5eef68952a03489dd9e37f67bc43bc367 Mon Sep 17 00:00:00 2001 From: Denis Date: Sun, 25 Aug 2013 16:52:04 +0200 Subject: [PATCH 19/21] Update parseincoming.py --- src/adhocracy/lib/emailcomments/parseincoming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index a30be7674..74efbf83d 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -118,7 +118,7 @@ def parse_local_part(recipient): return None sec_token = result.group("sectoken") -c + secrets = config.get("adhocracy.crypto.secret") comp_str = result.group("userid") + result.group("commentid") From 58f9a70b44242aa0c1c8dca546475e0215a70897 Mon Sep 17 00:00:00 2001 From: Denis Date: Sun, 25 Aug 2013 23:29:32 +0200 Subject: [PATCH 20/21] Voting now works in HTML-messages. --- src/adhocracy/lib/emailcomments/parseincoming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index 74efbf83d..d96cce896 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -106,8 +106,8 @@ def parse_payload(message): text = "" text = util.remove_notification(text) text = util.delete_signatures(text) - text, sentiment = util.get_sentiment(text) text = util.delete_debris(text) + text, sentiment = util.get_sentiment(text) return (text, sentiment) From 1df28b2451b5cdf5a1874b46a7488d2ed15a9363 Mon Sep 17 00:00:00 2001 From: Dekue Date: Mon, 26 Aug 2013 00:58:52 +0200 Subject: [PATCH 21/21] bug-fixes for voting, HTML-headings, images in emails with signatures --- src/adhocracy/lib/emailcomments/localwatch.py | 2 -- src/adhocracy/lib/emailcomments/parseincoming.py | 9 ++++++--- src/adhocracy/lib/emailcomments/util.py | 8 +++----- src/adhocracy/tests/lib/test_email_parsing.py | 6 +++--- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/adhocracy/lib/emailcomments/localwatch.py b/src/adhocracy/lib/emailcomments/localwatch.py index 603ed66e2..6dc0dcfa0 100644 --- a/src/adhocracy/lib/emailcomments/localwatch.py +++ b/src/adhocracy/lib/emailcomments/localwatch.py @@ -60,8 +60,6 @@ def watch_new_mail(path_md, path_mb, ecq): class PTmp(pyinotify.ProcessEvent): - path_decide = None - def process_IN_CREATE(self, event): maildir(path_md, ecq) diff --git a/src/adhocracy/lib/emailcomments/parseincoming.py b/src/adhocracy/lib/emailcomments/parseincoming.py index d96cce896..9317f96ea 100644 --- a/src/adhocracy/lib/emailcomments/parseincoming.py +++ b/src/adhocracy/lib/emailcomments/parseincoming.py @@ -55,6 +55,7 @@ def get_usable_content(content_list): ''' text = "" alt_set = [] + image_list = [] temp_alt_text = "" for rec_dep, alt_version, content_type, payload in content_list: if "na" in alt_version: @@ -65,7 +66,7 @@ def get_usable_content(content_list): elif "text/html" in content_type: text = text + util.html_to_markdown(payload) elif "image" in content_type: - text = text + save_image(payload, content_type) + image_list.append(save_image(payload, content_type)) else: if not rec_dep in alt_set: if "text/plain" in content_type: @@ -74,7 +75,7 @@ def get_usable_content(content_list): alt_set.append(rec_dep) temp_alt_text = util.html_to_markdown(payload) text = text + temp_alt_text - return text + return text, image_list def parse_payload(message): @@ -93,9 +94,11 @@ def parse_payload(message): content_list = parse_multipart(message, "a", 0) else: content_list = parse_multipart(message, "na", 0) - text = get_usable_content(content_list) + text, image_list = get_usable_content(content_list) text = util.remove_notification(text) text = util.delete_signatures(text) + for item in image_list: + text = text + item else: if "text/plain" in message["Content-Type"]: text = message.get_payload(decode=True) diff --git a/src/adhocracy/lib/emailcomments/util.py b/src/adhocracy/lib/emailcomments/util.py index b9a31e085..d208d0cfd 100644 --- a/src/adhocracy/lib/emailcomments/util.py +++ b/src/adhocracy/lib/emailcomments/util.py @@ -107,13 +107,12 @@ def get_sentiment(text): The Vote will be extracted and the line will be deleted if present. Also all leading newlines after vote-line will be deleted. ''' - pattern = re.compile(ur"""\A(?:[vV]ote:?\s*)? - (?P0|1|\+1?|-1?) + pattern = re.compile(ur"""\A(?:[vV][oO][tT][eE]:?\s*)? + (?P0|1|\+1?|-1?)\s* (?:\n|\r|\r\n)+ (?P(.|\n|\r|\r\n)*) """, re.VERBOSE) result = pattern.match(text) - if result: text = result.group("text") sentiment = result.group("sentiment") @@ -224,8 +223,6 @@ def parse_soup(text): replacements = [(ur"(\n|\r|\r\n)[\s]*|\t[\s]*", u""), (ur"\s{2,}", u" "), - (ur">\s*", u">"), - (ur"\s*<", u"<"), (ur"
    ", u" \n"), (ur"([\s]*)|()", u"**"), (ur"([\s]*)|()", u"**"), @@ -234,6 +231,7 @@ def parse_soup(text): (ur"|", u" > "), (ur"|", u" \n\n"), (ur"", h_repl), + (ur"", u" \n"), (u"", a_repl), (ur"", img_repl), (ur"
    • ", u""), diff --git a/src/adhocracy/tests/lib/test_email_parsing.py b/src/adhocracy/tests/lib/test_email_parsing.py index cf7365334..637c4b280 100644 --- a/src/adhocracy/tests/lib/test_email_parsing.py +++ b/src/adhocracy/tests/lib/test_email_parsing.py @@ -51,8 +51,8 @@ def test_html_to_markdown(self): self.assertEqual(rp(u'\n\t
    \t\t\n\n
    '), u' \n \n') self.assertEqual(rp(u'foobar
    \n\nbarfoo'), u'foobar \nbarfoo') self.assertEqual(rp(u' foobar'), u'**foobar**') - self.assertEqual(rp(u'

    foobar

    '), u'## foobar') - self.assertEqual(rp(u'

    foobar

    '), u'## foobar') + self.assertEqual(rp(u'

    foobar

    '), u'## foobar \n') + self.assertEqual(rp(u'

    foobar

    '), u'## foobar \n') self.assertEqual(rp(u'foobar
    \r\n\n\rbarfoo'), u'foobar \nbarfoo') self.assertEqual(rp(u""" @@ -75,7 +75,7 @@ def test_html_to_markdown(self):
  • foo
  • bar
    • -

    bla


    +

    bla

    foo
    bar