kanten.py

#!/usr/bin/env python
from __future__ import print_function

DEBUG = True
# debugging only
if DEBUG:
    try:
        import IPython
    except ImportError:
        print("IPython necessary for debugging")
        DEBUG = False

import os
import sys
import argparse

from collections import defaultdict

import urwid
from urwid import Padding, Text, Pile, ProgressBar

try:
    import pygments
    import pygments.lexers
    have_pygments = True
except ImportError:
    have_pygments = False

__version__ = '0.5.2'

PY3 = (sys.version_info[0] >= 3)

class Kanten(object):
    def __init__(self, **kwargs):
        self.__dict__.update(**kwargs)

    @property
    def max_width(self):
        return self.screen.get_cols_rows()[0]

    idx = 0

options_map = {
    'ft':'filetype',
    'nu':'number',
    'is':'incsearch',
    'tw':'textwidth',
}

def main():
    global fname
    global K
    parser = argparse.ArgumentParser(
     description='The enlightened pager: less paging. more content. read widely.')
    parser.add_argument('filenames', metavar='f', nargs='*',
                       help='an integer for the accumulator')
    parser.add_argument( '-w','--width', dest='width', metavar='N', type=int,
                       default=80,
                       help='the number of characters in a column line')
    parser.add_argument( '-c','--columns', dest='columns', metavar='N', type=int,
                       default=0,
                       help='the number of columns to display (overrides -w)')
    parser.add_argument( '-l','--height', '--lines', dest='height', metavar='N', type=int,
                       default='0',
                       help='the number of lines per column (0 for auto)')
    parser.add_argument( '-t','--top', dest='top', metavar='N', type=int,
                       default='4',
                       help='the number of lines to leave blank at the top')
    parser.add_argument( '-b','--bottom', dest='bottom', metavar='N', type=int,
                       default='4',
                       help='the number of lines to leave blank at the bottom')
    parser.add_argument( '-d','--diff', dest='diff', action='store_true',
            help='start in diff mode (same as :set ft=diff)')
    parser.add_argument( '-q','--quick', dest='quick', action='store_true',
            help='quit right away (same as :quit on load)')

    args = parser.parse_args()
    K = Kanten(
        width= args.width,
        height = args.height,
        top = args.top,
        bottom = args.bottom,
        top_margin = args.top,
        screen = urwid.raw_display.Screen(),
        args=args,
    )


    max_width, max_height = K.screen.get_cols_rows()

    max_height = max_height- K.top - K.bottom
    K.height = min(max_height, K.height) if K.height > 0 else max_height
    if args.columns > 0:
        K.width = int(max_width / args.columns)


    if not args.filenames:
        # XXX: in the future this will be an explanation of how to use kanten
        fname = '__missing_file_name__'
    else:
        fname = args.filenames[0]

    kanten_default_options = dict(
        filetype='',
        number=False,
        incsearch=False,
        editor=os.environ.get('EDITOR', 'vim'),
        textwidth=K.width
        )

    K.kanten_options = kanten_default_options.copy()


    # crude "filetype" detection
    if os.path.splitext(fname)[-1] in ('.diff', '.patch') or args.diff:
        K.kanten_options['filetype'] = 'diff'

    # This text instance should become a LazyReader object
    text, fname = read(fname)
    if len(text) == 0:
        sys.exit(0)
    K.fname = fname
    render_text(text, K)

def opt_name(name):
    "Translate short names to their full equivalents"
    if name in options_map:
        return options_map[name]
    else:
        return name


k_debug = ('ctrl k', 'backspace')
k_next = (' ', 'f', 'z', 'l',  'ctrl f', 'ctrl v', 'right', 'down', 'page down', 'J')
k_prev = ('b', 'B', 'w', 'ctrl b', 'left', 'up', 'page up', 'h', 'K')
k_next_one = ('j', 'ctrl y', 'ctrl d')
k_prev_one = ('k', 'ctrl e', 'ctrl u')
k_top = ('g', '<', 'p', 'home')
k_end = ('G', '>', 'end')
k_info = ('ctrl g', '=')
k_search = ('/',)
k_search_bw = ('?',)
k_next_search = ('n',)
k_prev_search = ('N',)
k_toggle_pbar = ('t',)
k_command = (':',)
k_submit = ('enter',)
k_escape = ('esc',)
k_quit = ('q', 'Q')
# not sure if 'h' not being mapped to 'left' is a good idea
k_help = ('h', 'H', 'f1', '?') 
k_version = ('V',)
k_diff = ('d',)     # enable diff highlighting
k_diff_off = ('D',) # disable diff highlighting
k_editor = ('v',) # launch the $EDITOR
m_scroll_up = (4,)   # scroll up
m_scroll_down = (5,) # launch the $EDITOR
m_paste = (2,) # launch the $EDITOR
m_click = (1,) # launch the $EDITOR

c = lambda x: K.cmd_line_text.set_caption(x)
#c = lambda x: cmd_line_prompt.set_text(x)
e = lambda x: K.cmd_line_text.set_edit_text(x)

def help_egg():
    from struct import pack
    magic_number = 0x789ced92bf4ec33010c6773fc575ea52f51d40426a25a8d83abbf125364dce51ecb40a4fcf7776a940626361608892dc9feff3fdce3beec715ed49981da538f029ba65bbdd9addd3f3eb8a0e31d3db9c325959346376a8dfd012673a4bbc7e6d8cc21bf2481b73f42c485d6dd24ae97842050d73e3efffd95ba11c9d5d8c6a5c10522568dd9cd6a98851108d406b318f732635cd9e13135a13d989a92bcefbf5806456a3c47d4b36a57962670ee5946d1087920b5303e30e36c34283062d9e38b220348f2a4d2ec6299932290d4ca12de33656aa49cbdc07e9507615f3a0b2f8243b8e1337c1662ed527d692f58469dd5da9e3acb610c874b2cd99a214c3ae961da3ac73e91e7bb60933d577e530b0296e0a00227d689100e0cf79c029fa82f9862b99970551c718ceb1340c2e3c2440a78b9590bcb6a8bdb7ef5cd8ea169662a050725da28eabb2411237208a759deb126c46ba5c8e7207f4e07d3873c55c57eab01ab06823fafe22d0ff8b5ab97ec3baf92dd7cd0f1f31fa0fb5ec843e
    # yes it is! it's a magic number! 3-6-9, 12-15-18, 21-24-27, 30!
    egg =  list(zip(*([iter(format(magic_number, 'x'))] * 2)))
    egg = pack('B'*(len(egg)), *[int(''.join(x), 16) for x in egg])
    try:
        import zlib
    except ImportError:
        while True:
            yield "no zlib? I'm afraid no one can help you :\\"
    else:
        while True:
            for m in zlib.decompress(egg).decode('utf-8').split('\n'):
                yield m
help = help_egg()

exit_font = urwid.font.HalfBlock7x7Font

def display_version(args=None):
    ver = urwid.BigText(('ver'," kanten v" + __version__), exit_font())
    ver = urwid.Overlay(ver, K.loop.widget, 'center', 'pack', 'middle', 'pack',)
    K.loop.widget= exit
    return True

def display_help(args=None):
    c(next(help))
    #exit = urwid.BigText(('exit'," kanten v" + __version__), exit_font())
    exit = urwid.BigText(('exit'," kanten v" + str(__version__)), exit_font())
    #exit = urwid.Pile([exit, ])
    #exit = urwid.Padding(exit,('relative', 100), width, left=2, right=2 )
    exit = urwid.Overlay(exit, K.loop.widget, 'center', 'pack', ('relative',90), 'pack',
            min_height=15)
                #min_width=20, min_height=5)
    # TODO: maybe do some overlay later - inserting into col content is pretty
    # good for now
    #cols.contents.insert(0, exit)
    K.loop.widget= exit
    return True

def quit(args=None):
    if args and ('!' in args[0] or 'a' in args[0]):
        print("\nold habits die hard! ;)")
    raise urwid.ExitMainLoop()

def edit(args):
    if len(args) > 1:
        e( "NotImplemented: can't open other files yet")
    else:
        return info(args)

def info(args):
    show_or_exit('ctrl g')
    return True

def cmd_not_found(args):
    e('not a kanten command: ' + ' '.join(args))

def set_cmd(args, K):
    """
    Set various aspects about how kanten behaves. A lot of the syntax and
    semantics are borrowed from Vim's `:help set`

    :se[t]              Show all options that differ from their default values
    :se[t] all          Show all options that differ from their default values

    :se[t] {option}     Toggle option: set, switch it on.
                        Number option: show value.
                        String option: show value.

    :se[t] no{option}   Toggle option: Reset, switch it off.

    :se[t] {option}!    Toggle option: Invert value.
    :se[t] inv{option}  Toggle option: Invert value.
    
    :se[t] {opt}={val}  Set string or number option {opt} to {val}.
    :se[t] {opt}:{val}  Set string or number option {opt} to {val}.

    :se[t] all          Reset all options to their default kanten values
    :se[t] all&less     Reset all options to emulate `less`
    :se[t] all&more     Reset all options to emulate `more`

    The {option} arguments to ":set" may be repeated.  For example:
            :set ft=diff nu
    """
    if len(args) == 1:
        ret =''
        for key,val in K.kanten_options.items():
            # skip entries which are default values
            #if val == kanten_default_options[key]:
            #    continue
            if val not in (True, False):
                ret += key + '=' + str(val)
            else:
                ret += key if val else 'no' + key
            ret+="    "
        c(ret)
    for arg in args[1:]:
        invert, negate = False, False
        idx = arg.find('=')
        idx2 = arg.find(':')
        if idx < 0 and idx2 < 0 :
            if arg.startswith('inv'):
                arg = arg[3:]
                invert = True
            if arg.startswith('no'):
                arg = arg[2:]
                negate = True
            arg = opt_name(arg)
            val = K.kanten_options.get(arg, ' unknown option')
            if val not in (True, False):
                msg =  '  ' + arg+ '=' + str(val)
            else:
                # set it
                val = not val if invert else True
                val = False if negate else val
                K.kanten_options[arg] = val
                msg = arg if val else 'no' + arg
                msg = ':set ' + msg
            c(msg)
            continue
        if idx > 0 and idx2 > 0:
            chomp = min(idx, idx2)
        else: # only one of the args is non-negative
            chomp = max(idx, idx2)
        lhs,rhs = arg[:chomp],arg[chomp+1:]
        arg = opt_name(lhs)
        if K.kanten_options[arg] in (True, False):
            c("Cannot assign, use 'set %s', 'set no%s'" % (arg, arg))
        else:
            K.kanten_options[arg] = rhs
            #XXX: turning into spaghetti code here, but what can you do?
            #   - make a callback (reactive) options dictionary?
            if arg == 'filetype' and rhs == 'diff':
                rehighlight(K.txts, '', search=search_diff)
            elif arg == 'filetype':
                #XXX: add pygments-based highlighting here for other files
                rehighlight(K.txts, '', search=search_noop)
    return True # by returning True, the cmd_line_text won't get reset to ''

def search_replace(args):
    pass

def get_search_or_search_next(key):
    def search_or_change_result(x):
        if len(x) == 0:
            return show_or_exit(key[0])
        rehighlight(K.txts, x)
    return search_or_change_result

# All dispatch commands should return True only if the rest of the
# show_or_exit method should be skipped after they are performed.
colon_dispatch_defaults = {
        'help'  : display_help,
        'quit'  : quit,
        'q'     : quit,
        'q!'    : quit,
        'qa'    : quit, # old habits die hard
        'qa!'   : quit, # old habits die hard
        'exit'  : quit,
        'e'     : edit,
        'edit'  : edit,
        'f'     : edit,
        'file'  : edit,
        'set'   : set_cmd,
        'se '   : set_cmd,
        's'     : search_replace,
        }
colon_dispatch = defaultdict(lambda: cmd_not_found, colon_dispatch_defaults)


def colon(cmd):
    #c('would have run' + cmd)
    args = cmd.split()
    if args: # :<enter> will give a blank line as a cmd
        return colon_dispatch[args[0].lower()](args)

do_cmd = colon

def page_back():
    pass

K = None

def show_or_exit(key):
    global last_key
    global show
    global do_cmd
    # clear out old messages
    txt = ''

    # set the progress bar visibility, so info can set it just once
    pbh = K.pbh
    pbh.send(show)
    displayed_columns = K.displayed_columns
    cols = K.cols

    if isinstance(K.loop.widget, urwid.Overlay):
        K.loop.widget = K.loop.widget[0] # pop off the overlay
    if key != '.':
        last_key = key
    else:
        key = last_key
    if key in k_quit:
        raise urwid.ExitMainLoop()
    elif key in k_help:
        display_help()
        return True
    elif key in k_version:
        #display_version()
        display_help()
        return True
    elif key in k_prev_one:
        if K.idx > 0:
            K.idx -= 1
            next_pane = K.reader[K.idx]
            if next_pane:
                cols.contents.pop() # get rid of the last one
                cols.contents.insert(0, (next_pane,('weight', 1, False)))
        else:
            txt += '@ 0 '
        txt += '(prev one)' + debug_line(K)
        #if K.idx < 0:
        #    K.idx = 0
    elif key in k_prev:
        for x in range(displayed_columns):
            if K.idx > 0:
                K.idx -= 1
                next_pane = K.reader[K.idx]
                if next_pane:
                    cols.contents.pop() # get rid of the last one
                    cols.contents.insert(0, (next_pane,('weight', 1, False)))
                cols.focus_position = 0
        #K.idx -= displayed_columns
        txt = '(prev)' + debug_line(K)
        if K.idx < 0:
            K.idx = 0
    elif key in k_top:
        # take it from the top
        #cols.contents = off_screen + cols.contents
        #off_screen = []
        K.idx = 0 
        #cols.contents.pop(0);

        # empty all contents
        while len(cols.contents):
            cols.contents.pop(0)

        for x in range(displayed_columns):
            next_pane = K.reader[K.idx + x]
            #cols.contents.pop(0);
            if next_pane:
                cols.contents.insert(x, (next_pane, ('weight', 1, False)))
            cols.focus_position = x
        #cols.focus_position = 0
        #cols.contents.focus= 0
        txt = '(top)' + debug_line(K)
        #cols.render(True)
    elif key in k_end:
        # this is the end, my friends, the end, the end.
        K.idx = len(K.reader) - displayed_columns
        show_or_exit(k_next[0])
        #off_screen.extend(cols.contents)
        ## backfill here properly - fill the hole screen (add back as many columns as can be displayed)
        #cols.contents = [off_screen.pop() for x in range(displayed_columns) ][::-1]
        ## XXX: finish this implementation
        ## K.reader.exhaust()
        txt = '(END)'
        txt += debug_line(K)
    elif key in k_next_one:
        if K.reader.exhausted and K.idx >= len(K.reader) - K.displayed_columns:
            txt = '(END)'
            K.idx = len(K.reader) - K.displayed_columns
        else:
            K.idx += 1
            try:
                next_pane = K.reader[K.idx + K.displayed_columns]
            except IndexError:
                pass
            else:
                #cols.contents.pop(0)
                cols.contents.pop(0) # get rid of the front
                cols.contents.insert(displayed_columns, (next_pane,('weight', 1, False)))
                #cols.focus_position=displayed_columns
        txt += debug_line(K)
    elif key in k_next:
        txt = 'nope'
        # empty all contents

        K.idx += displayed_columns
        for x in range(displayed_columns):
            if len(cols.contents) < displayed_columns:
                pass
                #while len(cols.contents):
                #cols.contents.pop(0)
                #off_screen.append(cols.contents.pop(0))
            # the tuple here I just got out of the contents, not sure if its
            # the right thing to do, just trying to crawl along for now.
            # Perhaps use some of the  MonitoredList callback stuff here.
            try:
                next_pane = K.reader[K.idx + x]
            except IndexError:
                break
            cols.contents.pop(0)
            #length = len(K.reader) #if K.reader.exhausted else 0
            txt = '(got one)' + debug_line(K)
            cols.contents.insert(x, (next_pane, ('weight', 1, False)))
            #K.idx += 1
            cols.focus_position=x
        #K.idx += x - 1
        #K.idx += x
        if K.reader.exhausted and K.idx >= len(K.reader):
            #txt = '(EXHAUSTED)'
            #K.idx = len(K.reader)
            K.idx = len(K.reader)
            txt = '(le tired)' + debug_line(K)
            pass
    elif key in k_search:
        #cmd_line_text.focus()
        K.all.set_focus('footer')
        txt = '/'
        #do_cmd = lambda x: rehighlight(K.txts, x)
        do_cmd = get_search_or_search_next(k_next_search)
        K.cmd_line_text.set_edit_text('')
    elif key in k_search_bw:
        #cmd_line_text.focus()
        K.all.set_focus('footer')
        txt = '?'
        do_cmd = get_search_or_search_next(k_prev_search)
        K.cmd_line_text.set_edit_text('')
    elif key in k_command:
        #txt = ':'
        c(':')
        K.all.set_focus('footer')
        #cmd_line_text.set_edit_text('')
        do_cmd = colon
        return
    elif key in k_submit:
        if K.all.get_focus() == 'footer':
            input = K.cmd_line_text.get_edit_text()
            K.cmd_line_text.set_edit_text('');
            K.all.set_focus('body')
            if do_cmd(input):
                # colon_dispatch methods return true if the rest of the method
                # should be skipped (because colon_dispatch method also calls
                # it, for example) 
                return
    elif key in k_escape:
        if K.all.get_focus() == 'footer':
            txt = ''
            K.all.set_focus('body')
    elif key in k_next_search:
        # focus pane with a next result only if found
        # pseudocode:
        #   if current pane contains a matched element:
        #       go to the next pane that contains an element (or highlight the
        #       next match?)
        #
        c("next search functionality not implemented yet")
        return True
        pass
    elif key in k_prev_search:
        # focus last result only if found
        c("prev search functionality not implemented yet")
        return True
        pass
    elif key in k_diff:
        rehighlight(K.txts, '', search=search_diff)
    elif key in k_diff_off:
        rehighlight(K.txts, '', search=search_noop)
    elif key in k_info:
        txt = K.fname
        if K.kanten_options['filetype']:
            txt += " (ft=" + K.kanten_options['filetype'] + ")"
        txt += " (%d / %d)" % (K.total_cols-len(cols.contents) +
                displayed_columns , K.total_cols)
        if len(cols.contents) == displayed_columns:
            txt += ' (END)'
        pbh.send(True)
    elif key in k_toggle_pbar:
        show = not show
        pbh.send(show)
    elif key in k_editor:
        editor = K.kanten_options['editor']
        os.spawnvp(os.P_WAIT, editor, [editor, K.fname])
        #K.idx += displayed_columns
    elif isinstance(key, tuple) and key[0] == "mouse press":
        if key[1] in  m_scroll_up:
            show_or_exit(k_next[0])
        elif key[1] in m_scroll_down: 
            show_or_exit(k_prev[0])
        elif key[1] in m_click: 
            column = K.xpos_to_col(key[-2])


            txt = "click in column %d, line %d" % (column, key[-1])
        elif key[1] in m_paste:
            txt = "we would paste X11 clipboard contents here"
        else:
            txt = "unhandled mouse key " + str(key)
    elif key in k_debug:
        if DEBUG:
            IPython.embed()
    elif isinstance(key, tuple):
        txt = "unhandled key " + str(key)
    else:
        txt = "unhandled key " + str(key)
    if DEBUG:
        key = 'space' if key == ' ' else key
        txt += "key = " + str(key)
        txt += ' len(contents) = ' + str(len(cols.contents))
    K.cmd_line_text.set_caption(txt)
    #cmd_line_text.set_edit_text(txt)
    K.pbar.set_completion(K.idx+displayed_columns+1)
    K.cmd_line_text.set_edit_text('') 

show = True
def progress_bar_handler(p):
    """Progress bar coroutine. Send it whether or not you want to show the
    progress bar. 

    XXX: despite good intentions, I think I overengineered this bit. It could
    probably just be a function - I originally was trying to do some timing
    stuff in here, but ended up ripping out before making the commit
    """
    show = (yield)
    while True:
        if not len(p.body):
            p.body.append(K.pbar)
        if not show:
            if len(p.body):
                p.body.pop()
        show = (yield)

lexer = None
def read(fname):
    # XXX: implement buffering here, don't read the whole file / piped message
    global lexer
    if not sys.stdin.isatty():
        text,fname = read_from_pipe()
    else:
        if fname == "__missing_file_name__":
            if DEBUG:
                fname = __file__
            else:
                # XXX: put back the usage in here - or ideally, move it up to
                # the command-line argument processing.
                sys.stderr.write('Missing filename ("kanten --help" for help)\n')
                sys.exit(1)
        with open(fname) as f:
            text = f.read()

        if have_pygments:
            try:
                lexer = pygments.lexers.get_lexer_for_filename(fname)
            except pygments.util.ClassNotFound:
                # TODO: if ipynb, treat it as json
                # lexer = pygments.lexers.web.JsonLexer #XXX placeholder
                lexer = pygments.lexers.TextLexer # null / noop lexer

    return text, fname

def read_from_pipe():
    # read from a pipe
    global lexer
    text = sys.stdin.read()
    import os
    #sys.stdin.close() # this avoids ValueError: I/O operation on closed file
    # reopen stdin now that we've read from the pipe
    sys.__stdin__ = sys.stdin = open('/dev/tty')
    os.dup2(sys.stdin.fileno(), 0)
    fname = 'STDIN'
    # XXX: try to guess about the input using pygments
    if have_pygments:
        # since pygments' detection can be terrible, no point in giving it the
        # whole file.
        try:
            lexer = pygments.lexers.guess_lexer(text[:80])
        except pygments.lexers.ClassNotFound:
            lexer = pygments.lexers.TextLexer
        # the lexer guesser sucks and will say anything it's confused about is
        # Prolog? No.
        if lexer.name == "Prolog":
            lexer = pygments.lexers.TextLexer # null / noop lexer

    return text, fname


def make_text(t, width):
    result = Padding(Text(t, align='left'), ('relative', 100), width, left=2,
            right=2)
    if DEBUG and 0:
        return urwid.LineBox(result)
    return result

#txt = urwid.Text(text)
#text =  text.replace("\n","\n\n")
def search(text, word):
    txts = text.split(word)
    f = lambda x: ('important', word)
    res = list(f((yield t)) for t in txts)
    
    #res = [t for stub in txts for t in (stub, ('important', word))]
    # N. B. this approach adds a superflous trailing match
    return res[:-1]

def search_diff(text, word=None):
    if text and text[0] == '+':
        return [('diff new', text)]
    elif text and text[0] == '-':
        return [('diff old', text)]
    else:
        return text

def search_noop(text, word):
    return text

def rehighlight(txts, s, search=search):
    [t.original_widget.set_text(search(t.original_widget.text, s)) for t in txts]




def trim(t, d, w):
    """Trim the text in `t` to only `d` lines, assuming a width of `w`"""
    if DEBUG and 0:
        pre_rendered_text = t.original_widget.original_widget.text
        lines = t.original_widget.original_widget.render((w-2,)).text
        # now make a new text widget to hold the remaining lines. It will
        # be added to the next pile, which we will also initialize here
        if d >= len(lines):
            # happens because we clip the text, and not the linebox
            next_start = 0
        else:
            next_start = pre_rendered_text.find(lines[d].strip())
        t.original_widget.original_widget.set_text(pre_rendered_text[:next_start])
        return make_text(pre_rendered_text[next_start:], w)

    pre_rendered_text = t.original_widget.text
    lines = t.render((w,)).text

    # now make a new text widget to hold the remaining lines. It will
    # be added to the next pile, which we will also initialize here
    next_start = pre_rendered_text.find(lines[d].strip().decode('utf-8'))
    t.original_widget.set_text(pre_rendered_text[:next_start])
    return make_text(pre_rendered_text[next_start:], w)

def h(e, K):
    return e.rows((K.width,))

def text_generator(text, k):
    pass

def first_paint(text, K):
    pass

def render_text(text, K):
    # XXX: make this code lazy-reader reader-proxy aware
    txts = [make_text(t, K.width) for t in text.split('\n')]
    K.txts = txts
    piles = []
    p = Pile([])
    for t in txts[:]:
        #if 'What emerges' in t.text: pu.db
        p.contents.append((t, p.options()))
        t_size = t.rows((K.width,))
        #if piles and h(piles[-1]) > height: pu.db
        while h(p, K) > K.height:
            # Add a new pile, and send the trimmings in there
            piles.append(p)
            d = h(t, K) - (h(p, K) - K.height)
            
            #if d <= 0: pu.db
            
            # start the next column
            p_new = Pile([])
            t_extra = trim(t, d, K.width)
            # TODO: include diff status in here, and line numbers
            p_new.contents.append((t_extra, p.options()))
            p = p_new
            t = t_extra


        #if piles and h(piles[-1]) > height:
        #    # ACK!
        #    break
        if h(p, K) == K.height:
            piles.append(p)
            # start the next column
            p = Pile([])

    # all done, don't forget the last pile which we haven't added to the list yet
    piles.append(p)


    palette = [
        ('black',  'light gray', 'black'),
        ('heading', 'black', 'light gray'),
        ('important', 'black', 'light cyan'),
        ('line', 'black', 'light gray'),
        ('options', 'dark gray', 'black'),
        ('focus heading', 'white', 'dark red'),
        ('focus line', 'black', 'dark red'),
        ('diff old', 'dark red', 'black'),
        ('diff new', 'dark green', 'black'),
        ('focus options', 'black', 'light gray'),
        ('pg normal',    'white',      'black', 'standout'),
        ('pg complete',  'white',      'dark magenta'),
        ('selected', 'white', 'dark blue')]

    #piles = urwid.ListBox(urwid.SimpleFocusListWalker(piles))
    #cols = piles
    #fill = cols
    dc = int(K.max_width / K.width) # number of displayed columns
    while len(piles)  < int(dc):
        piles.append(Pile([]))
    cols = urwid.Columns(piles[:dc], dividechars=1, min_width=K.width)
    K.cols = cols
    col_widths = cols.column_widths(K.screen.get_cols_rows())
    K.displayed_columns = len( col_widths )

    def tmp_generator():
        for x in piles:
            yield urwid.Columns([x], dividechars=1, min_width=K.width)
    K.reader = LazyReader(tmp_generator())


    # XXX: I need to subclass columns, and make it so the keypress function
    # "rolls" the piles under the hood, and re-renders all the widgets.
    #
    # self.contents.append(self.contents.pop(0))
    #
    #cols.box_columns.extend(cols.widget_list)


    #grid = urwid.GridFlow(txts, cell_width=20, h_sep=4, v_sep=0, align='left')
    fill = urwid.Filler(cols, 'top', top=K.top_margin)
    K.total_cols = len(piles)

    # XXX: this is not the full story, it ignores the borders between columns
    c_columns = [sum(col_widths[:i+1]) for i in  range(K.displayed_columns)]
    border = (K.max_width - c_columns[-1]) /  K.displayed_columns
    def xpos_to_col(pos):
        for i,c in enumerate(c_columns):
            if pos < (c + i * border):
                return i

    K.xpos_to_col = xpos_to_col
    pbar = ProgressBar('pg normal', 'pg complete', K.displayed_columns, K.total_cols)
    K.pbar = pbar


    p = urwid.ListBox(urwid.SimpleListWalker([pbar]))

    all = Pile([ fill, (1, p), ])
    cmd_line_text = urwid.Edit(K.fname)
    K.cmd_line_text = cmd_line_text
    #cmd_line_prompt = urwid.Text('hi there')
    #cmd_line_combined = urwid.Filler([cmd_line_prompt, cmd_line_text])
    #all = urwid.Frame(body=all, footer=cmd_line_combined)
    K.all = urwid.Frame(body=all, footer=cmd_line_text)
    K.loop = urwid.MainLoop(K.all, palette, K.screen, unhandled_input=show_or_exit)
    K.loop.exit = urwid.Text(" Help? ")

    #IPython.embed()
    if K.args.diff:
        set_cmd("set ft=diff".split(), K)
    elif have_pygments:
        set_cmd(("set ft=" + lexer.name.split()[0].lower()).split(), K)

    if K.args.quick:
        K.loop.set_alarm_in(1, lambda x,y:  quit())

    pbh = progress_bar_handler(p)
    K.pbh = pbh
    next(pbh)


    try:
        K.loop.run()
    except KeyboardInterrupt:
        pass

#import IPython
#too_high = 0
#for p in piles:
#    if h(p) > max_height:
#        too_high += 1

#if too_high:
#    IPython.embed(header="There were %d violations of max_height" % too_high)

if DEBUG and 0:
    for p in piles:
        print(h(p, K))
        for c in p.contents:
            print("\t" , h(c[0], K))

#print [type(t.original_widget.text) for t in txts]
#print [(t.original_widget.get_text()[1]) for t in txts[0:100]]
f = lambda t:t.original_widget.get_text()[1]
g = lambda t:len(f(t))
#print [f(t) for t in txts[:] if g(t)>0]

class LazyReader(object):
    def __init__(self, generator):
        self.generator = generator
        self.cached = []
        self.exhausted = False

    def __getitem__(self, i):
        # XXX: idea: maybe index with a string to search for stuff?
        #if type(i) not in ((int, slice) + (() if PY3 else (long,))):
        #    raise TypeError("LazyReaders can only be indexed with integers")

        if isinstance(i, slice):
            last = i.stop
        else:
            last = i

        self.exhaust_until(last)

        return self.cached[i]

    def exhaust_until(self, i):
        try:
            while not self.exhausted and len(self.cached) <= i:
                self.cached.append(next(self.generator))
        except StopIteration:
            self.exhausted = True

    def exhaust(self):
        try:
            while not self.exhausted:
                self.cached.append(next(self.generator))
        except StopIteration:
            self.exhausted = True

    def __len__(self):
        return len(self.cached)
        #self.exhaust()
        #return len(self.cached)

def debug_line(K):
    return 'K.idx=%d, len(K)=%d  len(contents)=%d' % (
            K.idx,
            len(K.reader),
            len(K.cols.contents),
        )
if __name__ == '__main__':
    main()