Various upstream changes

.gitignore

@@ -1,6 +1,7 @@
 *.pkl
-*~
 *.py[co]
 *.sqlite
 *.swp
+*~
 config.yaml
+env

README.md

@@ -32,29 +32,31 @@ Adding New Reports
 It's easiest to explain this with an example. Suppose the report module
 specified by the config `module` variable has the following code in it:
 
-    from sqlalchemy import *
-    from kohlrabi.db import *
-
-    class DailySignups(Base):
-        __tablename__ = 'daily_signups_report'
-        __metaclass__ = ReportMeta
-
-        id = Column(Integer, primary_key=True)
-        date = Column(Date, nullable=False)
-        referrer = Column(String, nullable=False)
-        clickthroughs = Column(Integer, nullable=False, default=0)
-        signups = Column(Integer, nullable=False, default=0)
-
-        display_name = 'Daily Signups'
-        html_table = [
-            ReportColumn('Referrer', 'referrer'),
-            ReportColumn('Click-Throughs', 'clickthroughs'),
-            ReportColumn('Signups', 'signups'),
-            ]
-
-        @classmethod
-        def report_data(cls, date):
-            return session.query(cls).filter(cls.date == date).order_by(cls.signups.id)
+```python
+from sqlalchemy import *
+from kohlrabi.db import *
+
+class DailySignups(Base):
+    __tablename__ = 'daily_signups_report'
+    __metaclass__ = ReportMeta
+
+    id = Column(Integer, primary_key=True)
+    date = Column(Date, nullable=False)
+    referrer = Column(String, nullable=False)
+    clickthroughs = Column(Integer, nullable=False, default=0)
+    signups = Column(Integer, nullable=False, default=0)
+
+    display_name = 'Daily Signups'
+    html_table = [
+        ReportColumn('Referrer', 'referrer'),
+        ReportColumn('Click-Throughs', 'clickthroughs'),
+        ReportColumn('Signups', 'signups'),
+        ]
+
+    @classmethod
+    def report_data(cls, date):
+        return session.query(cls).filter(cls.date == date).order_by(cls.signups.id)
+```
 
 This is a data source that will track users who sign up on your site, based on
 the HTTP `Referrer` header. The table has three columns: `referrer` will track
@@ -68,15 +70,17 @@ have any indexes. In most cases you should probably at least add an index on the
 `date` column, and probably an index on the full set of columns you plan on
 querying from the `report_data` method:
 
-    CREATE TABLE daily_signups_report (
-        id INTEGER NOT NULL,
-        date DATE NOT NULL,
-        referrer VARCHAR NOT NULL,
-        clickthroughs INTEGER NOT NULL,
-        signups INTEGER NOT NULL,
-        PRIMARY KEY (id)
-    );
-    CREATE INDEX daily_signups_date_idx ON daily_signups_report (date, signups);
+```sql
+CREATE TABLE daily_signups_report (
+    id INTEGER NOT NULL,
+    date DATE NOT NULL,
+    referrer VARCHAR NOT NULL,
+    clickthroughs INTEGER NOT NULL,
+    signups INTEGER NOT NULL,
+    PRIMARY KEY (id)
+);
+CREATE INDEX daily_signups_date_idx ON daily_signups_report (date, signups);
+```
 
 OK, that's all the setup you need to do on the Kohlrabi side of things: create a
 Python SQLAlchemy class, and create a table in your SQLite database. The second
@@ -92,18 +96,20 @@ and the following POST parameters:
 For instance, if we were running Kohlrabi on `http://localhost:8888`, then the
 following Python code would generate a sample report for 2001-01-1:
 
-    import json
-    import urllib
-
-    urllib.urlopen('http://localhost:8888/upload',
-        urllib.urlencode({'date': '2010-01-01',
-                          'data': json.dumps([{'referrer': 'www.yahoo.com',
-                                               'clickthroughs': 100,
-                                               'signups': 7},
-                                              {'referrer': 'www.google.com',
-                                               'clickthroughs': 500,
-                                               'signups': 42}]),
-                          'table': 'DailySignups'}))
+```python
+import json
+import urllib
+
+urllib.urlopen('http://localhost:8888/upload',
+    urllib.urlencode({'date': '2010-01-01',
+                      'data': json.dumps([{'referrer': 'www.yahoo.com',
+                                           'clickthroughs': 100,
+                                           'signups': 7},
+                                          {'referrer': 'www.google.com',
+                                           'clickthroughs': 500,
+                                           'signups': 42}]),
+                      'table': 'DailySignups'}))
+```
 
 Just to reiterate: because the interface to Kohlrabi is a normal HTTP request
 using JSON, you can use any language to send data to Kohlrabi. You can use Java,

kohlrabi/db.py

@@ -1,21 +1,25 @@
 import datetime
+import time
+
 from sqlalchemy import *
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy.pool import NullPool
 from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
 
 report_tables = set()
 
 metadata = MetaData()
-session = None
+Session = None
 
 def bind(engine_path, import_module, create_tables=False):
-    global session
-    create_kw = {}
+    global Session
+    create_kw = {
+        'poolclass': NullPool,
+    }
     if engine_path.startswith('mysql+mysqldb'):
         create_kw['pool_recycle'] = 3600
     engine = create_engine(engine_path, **create_kw)
-    Session = sessionmaker(bind=engine)
-    session = Session()
+    Session = scoped_session(sessionmaker(bind=engine))
 
     if import_module:
         __import__(import_module)
@@ -28,20 +32,47 @@ def bind(engine_path, import_module, create_tables=False):
 
 class _Base(object):
 
+    __abstract__ = True
+
+    _variant_cache = (0, None)
+
     @classmethod
     def current_date(cls):
-        row = session.query(cls).order_by(cls.date.desc()).first()
+        row = Session.query(cls).order_by(cls.date.desc()).first()
         if row:
             return row.date
         else:
             return None
 
+    @classmethod
+    def variant_map(cls):
+        now = time.time()
+        then, vals = getattr(cls, '_variant_cache', (0, []))
+        if now - then <= 60:  # cache for one minute
+            return vals
+
+        variants = []
+        for variant in getattr(cls, 'variants', []):
+            values = []
+            for val in Session.query(getattr(cls, variant)).distinct():
+                val, = val
+                if val:
+                    values.append(val)
+            values.sort()
+            variants.append((variant, values))
+
+        cls._variant_cache = (now, variants)
+        return variants
+
 Base = declarative_base(metadata=metadata, cls=_Base)
 
 class ReportMeta(DeclarativeMeta):
 
     def __init__(cls, name, bases, cls_dict):
         super(ReportMeta, cls).__init__(name, bases, cls_dict)
+        if cls_dict.get('__abstract__', False):
+            return
+
         report_tables.add(cls)
 
         def format_float(v):
@@ -68,8 +99,8 @@ def format_str(s):
 
     def load_report(cls, data, date=None):
         date = date or datetime.date.today()
-        for row in session.query(cls).filter(cls.date == date):
-            session.delete(row)
+        for row in Session.query(cls).filter(cls.date == date):
+            Session.delete(row)
         for datum in data:
             if hasattr(cls, 'column_map'):
                 datum = dict((cls.column_map[k], v) for k, v in datum.iteritems())
@@ -83,11 +114,11 @@ def load_report(cls, data, date=None):
                 if not hasattr(cls, k):
                     del datum[k]
 
-            session.add(cls(**datum))
-        session.commit()
+            Session.add(cls(**datum))
+        Session.commit()
 
     def dates(cls, limit=None):
-        ds = session.query(cls.date).group_by(cls.date).order_by(cls.date.desc())
+        ds = Session.query(cls.date).group_by(cls.date).order_by(cls.date.desc())
         if limit is not None:
             return (row.date for row in ds[:limit])
         else:

kohlrabi/handlers.py

@@ -4,6 +4,7 @@
 import datetime
 import sqlalchemy
 import traceback
+import urlparse
 try:
     from sqlalchemy.exception import OperationalError
 except ImportError:
@@ -35,9 +36,13 @@ def initialize(self):
         self.env = {
             'title': '(kohlrabi)',
             'uri': self.uri,
-            'jquery_url': '//ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js'
+            'jquery_url': '//ajax.googleapis.com/ajax/libs/jquery/2.0.2/jquery.min.js'
         }
 
+    def finish(self, chunk=None):
+        db.Session.remove()
+        super(RequestHandler, self).finish(chunk)
+
     def parse_date(self, date_string):
         if date_string and date_string != 'current':
             return datetime.datetime.strptime(date_string, '%Y-%m-%d').date()
@@ -59,7 +64,6 @@ def render_json(self, obj):
         self.write(json.dumps(obj))
 
     def get_error_html(self, status_code, **kwargs):
-        db.session.rollback()
         self.set_header('Content-Type', 'text/plain')
         return traceback.format_exc()
 
@@ -70,39 +74,31 @@ class Home(RequestHandler):
     DATES_PER_PAGE = 14
 
     def get(self):
-        # this is pretty inefficient
+        # we want to know for each date, which tables are available
         date_map = defaultdict(list)
         for tbl in db.report_tables:
             for d in tbl.dates():
                 date_map[d].append(tbl)
 
+        for k, v in date_map.iteritems():
+            date_map[k] = sorted(v, key=lambda x: x.display_name)
+
+        # these are all the dates available to display reports for
         dates = sorted(date_map.keys(), reverse=True)
         self.env['num_pages'] = int(math.ceil(len(dates) / float(self.DATES_PER_PAGE)))
 
         # show DATES_PER_PAGE things per page
-        page = int(self.request.uri.split('/')[-1] or 1)
+        page = int(self.request.uri.split('/')[-1] or 1)  # FIXME
         self.env['page'] = page
         dates = dates[self.DATES_PER_PAGE * (page - 1) : self.DATES_PER_PAGE * page]
 
         self.env['dates'] = dates
-        report_names = set()
+        reports = set()
         for d in dates:
-            report_names |= set((r.display_name, r.__name__) for r in date_map[d])
+            reports |= set(date_map[d])
 
-        self.env['report_names'] = list(sorted(report_names))
-        self.env['reports'] = []
-
-        # this is *also* really inefficient and ghetto
-        for d in dates:
-            links = []
-            for report_name, report_table in self.env['report_names']:
-                for t in date_map[d]:
-                    if t.display_name == report_name:
-                        links.append((report_name, report_table))
-                        break
-                else:
-                    links.append(None)
-            self.env['reports'].append((d.strftime('%Y-%m-%d'), links))
+        self.env['report_names'] = sorted(report.display_name for report in reports)
+        self.env['date_map'] = date_map
 
         self.env['title'] = 'kohlrabi: home'
         self.render('home.html')
@@ -125,16 +121,22 @@ class Report(RequestHandler):
     path = '/report/.*/.*'
 
     def get(self):
-        path = self.request.uri.lstrip('/')
+        path = self.request.path.lstrip('/')
         components = path.split('/')
         tbl, date = components[-2:]
         if not date or date == 'current':
             date = getattr(db, tbl).current_date()
         else:
             date = self.parse_date(date)
+
         self.env['date'] = date
-        self.env['data'] = getattr(db, tbl).report_data(date)
+        table = getattr(db, tbl)
+        data = table.report_data(date)
+        for k, v in urlparse.parse_qsl(self.request.query):
+            data = data.filter(getattr(table, k) == v)
+        self.env['data'] = data
         self.env['columns'] = getattr(db, tbl).html_table
+        self.env['first_column'] = getattr(db, tbl).html_table[0].display
         self.env['title'] = getattr(db, tbl).display_name + ' ' + date.strftime('%Y-%m-%d')
         self.render('report.html')
 

requirements.txt

@@ -0,0 +1,3 @@
+PyYAML
+sqlalchemy
+tornado

static/css/main.css

@@ -53,3 +53,5 @@ ul.reports > li {
     border-radius: 5px;
     padding: 5px;
 }
+
+.variant_name { margin-left: 1em; }

static/css/report.css

@@ -7,11 +7,12 @@ th { padding-right: 1em; font-weight: bold; min-width: 6em; text-align: left; }
 th:hover { cursor: pointer; }
 td { padding-right: 1em; text-align: left; max-width: 60em; word-wrap: break-word; }
 
-.hoverable:hover { background: #c6daf4; }
+.hoverable:hover { background: #c6daf4; cursor: pointer; }
 
 .align-right { text-align: right; }
 .number { text-align: right; }
 .clicked { background: #d1c5ff !important; }
+.no-overflow { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
 td.tiny { font-size: 80%; }
 
 label { font-weight: bold; }