save pages version + refacto templating for future use to diff page (…

…wip #18)

gazouilleur/bot.py

@@ -166,7 +166,7 @@ def joined(self, channel):
         # Follow RSS Feeds matching url queries set for this channel with !follow
         self.feeders[lowchan]['news'] = FeederFactory(self, channel, 'news', 299, pagetimeout=35)
         # Monitor webpages set for this channel with !monitor
-        self.feeders[lowchan]['pages'] = FeederFactory(self, channel, 'pages', 299, pagetimeout=35)
+        self.feeders[lowchan]['pages'] = FeederFactory(self, channel, 'pages', 299, timeout=900, pagetimeout=95)
         twuser = get_chan_twitter_user(channel, conf)
         if twuser:
         # Get OAuth2 tokens for twitter search extra limitrate

gazouilleur/lib/feeds.py

@@ -30,6 +30,7 @@
 from gazouilleur.lib.utils import *
 from gazouilleur.lib.microblog import Microblog, check_twitter_results, grab_extra_meta, reformat_extended_tweets
 from gazouilleur.lib.stats import Stats
+from gazouilleur.lib.webmonitor import WebMonitor
 
 class FeederProtocol(object):
 
@@ -146,7 +147,10 @@ def process_elements(self, data, url, name=None):
         if not data:
             returnD(False)
         if self.fact.name == "pages":
-            self.log(name+ ": \n"+"\n".join(data.split('\n')[:3]), hint=True)
+            differ = WebMonitor(name)
+            info = differ.check_diff(url, data)
+            if info:
+                self.fact.ircclient._send_message(info, self.fact.channel)
             returnD(True)
         if not data.entries:
             returnD(False)

gazouilleur/lib/stats.py

@@ -8,17 +8,14 @@
 from gazouilleur import config
 from gazouilleur.lib.mongo import SingleMongo, find_stats, count_followers, find_last_followers, sortasc, sortdesc
 from gazouilleur.lib.log import loggerr
+from gazouilleur.lib.templater import Templater
 from gazouilleur.lib.utils import *
 
-class Stats(object):
+class Stats(Templater):
 
     def __init__(self, user):
         self.user = user
-        try:
-            self.url = '%s/' % config.URL_STATS.rstrip('/')
-        except:
-            self.url = None
-        self.templates = os.path.join("web", "templates")
+        Templater.__init__(self)
 
     @inlineCallbacks
     def print_last(self):
@@ -131,21 +128,6 @@ def dump_data(self):
         self.render_template("static_stats.html", self.user, data)
         returnValue(True)
 
-    def render_template(self, template, name, data):
-        outfile = template.replace('.html', '_%s.html' % name)
-        try:
-            import codecs
-            import pystache
-            from contextlib import nested
-            ofile = os.path.join("web", outfile)
-            with nested(open(os.path.join(self.templates, template), "r"), codecs.open(ofile, "w", encoding="utf-8")) as (temp, generated):
-                generated.write(pystache.Renderer(string_encoding='utf8').render(temp.read(), data))
-            os.chmod(ofile, 0o644)
-            return True
-        except IOError as e:
-            loggerr("Could not write web/%s from %s/%s : %s" % (outfile, self.templates, template, e), action="stats")
-            return False
-
     @inlineCallbacks
     def digest(self, hours, channel):
         now = datetime.today()

gazouilleur/lib/templater.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os, codecs
+from pystache import Renderer
+from contextlib import nested
+from gazouilleur.lib.log import loggerr
+try:
+    from gazouilleur.config import URL_STATS
+except:
+    URL_STATS = None
+
+class Templater(object):
+
+    def __init__(self):
+        self.url = '%s/' % URL_STATS.rstrip('/') if URL_STATS else None
+        self.templates = os.path.join("web", "templates")
+
+    def render_template(self, template, name, data):
+        outfile = template.replace('.html', '_%s.html' % name)
+        try:
+            ofile = os.path.join("web", outfile)
+            with nested(open(os.path.join(self.templates, template), "r"), codecs.open(ofile, "w", encoding="utf-8")) as (temp, generated):
+                generated.write(Renderer(string_encoding='utf8').render(temp.read(), data))
+            os.chmod(ofile, 0o644)
+            return True
+        except IOError as e:
+            loggerr("Could not write web/%s from %s/%s : %s" % (outfile, self.templates, template, e), action="stats")
+            return False
+

gazouilleur/lib/tests.py

@@ -65,7 +65,7 @@
     exit(1)
 
 try:
-    from gazouilleur.lib import ircclient_with_names, irccolors, feeds, filelogger, httpget, log, microblog, mongo, stats, utils
+    from gazouilleur.lib import ircclient_with_names, irccolors, feeds, filelogger, httpget, log, microblog, mongo, stats, utils, templater
 except Exception as e:
     logerr("Oups, looks like something is wrong somewhere in the code, shouldn't be committed...")
     logerr("%s\n%s" % (e, "\n".join(format_exc().splitlines()[-3:-1])))

gazouilleur/lib/webmonitor.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os, time
+from gazouilleur.lib.templater import Templater
+
+class WebMonitor(Templater):
+
+    def __init__(self, name):
+        Templater.__init__(self)
+        self.name = name
+        self.path = os.path.join('web', 'monitor', name)
+        if not os.path.exists(self.path):
+            os.makedirs(self.path)
+
+    def check_diff(self, url, data):
+        # TODO:
+        # apply url_rewrite for '<[^>]*=['"]/([^/]|$)' et '<[^>]*=['"](!:http)'
+        # - check if file -last exists
+        # - if so diff md5 current/last
+        # - check if exist and not diff
+        if False:
+            return None
+        for name in ["last", time.strftime("%y%m%d-%H%M")]:
+            fil = os.path.join(self.path, "%s.html" % name)
+            with open(fil, "w") as f:
+                f.write(data)
+            os.chmod(fil, 0o644)
+        msg = "Looks like the webpage %s at %s just changed!" % (self.name, url)
+        if self.url:
+            self.build_diff_page(url)
+            msg += "\nYou can check the different versions and diffs at %smonitor_%s.html" % (self.url, self.name)
+        return msg
+
+    def build_diff_page(self, url):
+        data = {
+          "name": self.name,
+          "url": url,
+        }
+        data["versions"] = sorted(os.listdir(os.path.join('web', 'monitor', self.name)), reverse=True)
+        self.render_template("monitor.html", self.name, data)

web/templates/digest.html

@@ -2,9 +2,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <html>
   <head>
-      <meta content="text/html; charset=utf-8" http-equiv="content-type"/>
-      <title>Digest gazouilleur {{channel}}</title>
-      <style>
+    <meta content="text/html; charset=utf-8" http-equiv="content-type"/>
+    <title>Digest gazouilleur {{channel}}</title>
+    <style>
 body{
   margin: 10px;
 }
@@ -74,7 +74,7 @@ <h2>Links from tweets</h2>
 <h2>Images from tweets</h2>
 <div id="images"></div>
   </body>
-<script>
+<script type="text/javascript">
 window.twttr = (function(d, s, id) {
   var js, fjs = d.getElementsByTagName(s)[0],
     t = window.twttr || {};

web/templates/monitor.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<?xml version="1.0" encoding="UTF-8"?>
+<html>
+  <head>
+    <meta content="text/html; charset=utf-8" http-equiv="content-type"/>
+    <title>WebMonitor by gazouilleur for {{name}}</title>
+<!-- TODO: stylify everything-->
+    <style>
+    </style>
+  </head>
+  <body>
+    <h1>WebMonitor by gazouilleur for <a target="_blank" href="{{url}}">{{name}} at {{url}}</a></h1>
+    <h2>Available versions</h2>
+    <ul>
+      <li><b><a target="_blank" href="{{url}}">Actual</a></b></li>
+    {{#versions}}
+      <li><b><a target="_blank" href="monitor/{{name}}/{{.}}">Saved {{.}}</a></b></li>
+    {{/versions}}
+    </ul>
+    <h2 id="version">Actual version</h2>
+    <iframe id="page" src="{{url}}"></iframe>
+    <h2>Diff view</h2>
+    <span id="old"></span> Vs. <span id="new"></span>
+    <div id="diff"></div>
+  </body>
+<script type="text/javascript">
+// TODO: load page in iframe on click + set name in version
+// TODO: code differ
+// TODO: juggle between view and diff modes
+</script>
+</html>