test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "urls = range(100)\n",
    "\n",
    "\n",
    "def fetch_url(url):\n",
    "    sleep(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def log_progress(sequence, every=10):\n",
    "    for index, item in enumerate(sequence):\n",
    "        if index % every == 0:\n",
    "            print >>sys.stderr, index,\n",
    "        yield item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 0 10 20 30 40 50 60 70 80 90"
     ]
    }
   ],
   "source": [
    "for url in log_progress(urls):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "urls = range(10000)\n",
    "\n",
    "\n",
    "def check_url(url):\n",
    "    sleep(0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for url in log_progress(urls):\n",
    "    check_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "urls = ['http://{0}.ru'.format(_) for _ in range(100)]\n",
    "\n",
    "\n",
    "def fetch_url(url):\n",
    "    print 'Fetching', url\n",
    "    sleep(0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for url in log_progress(urls, every=2):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def log_progress(sequence, every=10):\n",
    "    from ipywidgets import IntProgress\n",
    "    from IPython.display import display\n",
    "\n",
    "    progress = IntProgress(min=0, max=len(sequence), value=0)\n",
    "    display(progress)\n",
    "    \n",
    "    for index, record in enumerate(sequence):\n",
    "        if index % every == 0:\n",
    "            progress.value = index\n",
    "        yield record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for url in log_progress(urls, every=1):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def log_progress(sequence, every=None, size=None):\n",
    "    from ipywidgets import IntProgress, HTML, VBox\n",
    "    from IPython.display import display\n",
    "    \n",
    "    is_iterator = False\n",
    "    if size is None:\n",
    "        try:\n",
    "            size = len(sequence)\n",
    "        except TypeError:\n",
    "            is_iterator = True\n",
    "    if size is not None:\n",
    "        if every is None:\n",
    "            if size <= 200:\n",
    "                every = 1\n",
    "            else:\n",
    "                every = size / 200     # every 0.5%\n",
    "    else:\n",
    "        assert every is not None, 'sequence is iterator, set every'\n",
    "\n",
    "    if is_iterator:\n",
    "        progress = IntProgress(min=0, max=1, value=1)\n",
    "        progress.bar_style = 'info'\n",
    "    else:\n",
    "        progress = IntProgress(min=0, max=size, value=0)\n",
    "    label = HTML()\n",
    "    box = VBox(children=[label, progress])\n",
    "    display(box)\n",
    "    \n",
    "    index = 0\n",
    "    try:\n",
    "        for index, record in enumerate(sequence, 1):\n",
    "            if index == 1 or index % every == 0:\n",
    "                if is_iterator:\n",
    "                    label.value = '{index} / ?'.format(index=index)\n",
    "                else:\n",
    "                    progress.value = index\n",
    "                    label.value = u'{index} / {size}'.format(\n",
    "                        index=index,\n",
    "                        size=size\n",
    "                    )\n",
    "            yield record\n",
    "    except:\n",
    "        progress.bar_style = 'danger'\n",
    "        raise\n",
    "    else:\n",
    "        progress.bar_style = 'success'\n",
    "        progress.value = index\n",
    "        label.value = str(index or '?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from time import sleep\n",
    "\n",
    "\n",
    "urls = range(100)\n",
    "\n",
    "\n",
    "def fetch_url(url):\n",
    "    sleep(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for url in log_progress(urls[:10], every=1):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-dce6a4981a5d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0murl\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlog_progress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mfetch_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-33-4111f39e4599>\u001b[0m in \u001b[0;36mfetch_url\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfetch_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for url in log_progress(urls, every=1):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for url in log_progress(urls[:10]):\n",
    "    fetch_url(url)\n",
    "for url in log_progress(urls[10:20]):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for url in log_progress([]):\n",
    "    fetch_url(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from random import random\n",
    "\n",
    "\n",
    "urls = range(100)\n",
    "\n",
    "\n",
    "def fetch_url(url):\n",
    "    sleep(0.1)\n",
    "    \n",
    "\n",
    "def fetch_urls(urls):\n",
    "    for url in urls:\n",
    "        yield fetch_url(url)\n",
    "    \n",
    "    \n",
    "def is_ok_content(response):\n",
    "    return random() < 0.3\n",
    "\n",
    "    \n",
    "def get_ok_content_responses(responses):\n",
    "    for response in responses:\n",
    "        if is_ok_content(response):\n",
    "            yield response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "responses = fetch_urls(log_progress(urls, every=1))\n",
    "responses = get_ok_content_responses(log_progress(responses, every=1))\n",
    "responses = list(responses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fetch_url(url):\n",
    "    sleep(random() / 3)\n",
    "\n",
    "    \n",
    "def fetch_urls(urls):\n",
    "    for url in urls:\n",
    "        fetch_url(url)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting job # 0 in a separate thread.\n",
      "Starting job # 2 in a separate thread.\n",
      "Starting job # 3 in a separate thread.\n",
      "Starting job # 4 in a separate thread.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<BackgroundJob #4: <function fetch_urls at 0x103dce410>>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.lib import backgroundjobs as bg\n",
    "jobs = bg.BackgroundJobManager()\n",
    "jobs.new(fetch_urls, log_progress(urls, every=1))\n",
    "jobs.new(fetch_urls, log_progress(urls, every=1))\n",
    "jobs.new(fetch_urls, log_progress(urls, every=1))\n",
    "jobs.new(fetch_urls, log_progress(urls, every=1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}