Magics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 23048\n",
      "-rw-rw-r-- 1 joydeep joydeep       19 Nov 22 17:19 README.md\n",
      "-rw-rw-r-- 1 joydeep joydeep     3314 Nov 22 17:27 D3 notebook.py\n",
      "-rw-rw-r-- 1 joydeep joydeep    60166 Nov 22 17:31 rukhmabai-rauts-153rd-birthday-5850695327547392.3-l.png\n",
      "-rw-rw-r-- 1 joydeep joydeep   211223 Nov 22 17:36 120907-Blogging with the IPython Notebook.ipynb\n",
      "-rw-rw-r-- 1 joydeep joydeep    10916 Nov 22 17:36 120907-Blogging with the IPython Notebook.py\n",
      "-rw-rw-r-- 1 joydeep joydeep 23050037 Nov 22 19:33 reddituserpostingbehavior.csv.gz\n",
      "-rw-rw-r-- 1 joydeep joydeep     1335 Nov 22 19:34 Data Science and (Unsupervised) Machine Learning with scikit-learn .py\n",
      "-rw-rw-r-- 1 joydeep joydeep        0 Nov 22 19:51 Untitled.txt\n",
      "-rw-rw-r-- 1 joydeep joydeep      168 Nov 22 19:55 Miscellaneous_commands.py\n",
      "-rw-rw-r-- 1 joydeep joydeep   187334 Nov 22 20:41 329994478_72017_3649_airline_delay_causes.csv\n",
      "-rw-rw-r-- 1 joydeep joydeep    10512 Nov 22 20:43 JsInNb.ipynb\n",
      "-rw-rw-r-- 1 joydeep joydeep     4193 Nov 22 20:43 JsInNb.py\n",
      "-rw-rw-r-- 1 joydeep joydeep     5611 Nov 22 21:49 10-B-pythreejs.py\n",
      "-rw-rw-r-- 1 joydeep joydeep    11433 Nov 23 09:26 More_commands.ipynb\n",
      "-rw-rw-r-- 1 joydeep joydeep      810 Nov 23 09:26 More_commands.py\n",
      "-rw-rw-r-- 1 joydeep joydeep     4986 Nov 23 09:35 Magics.ipynb\n",
      "-rw-rw-r-- 1 joydeep joydeep     1967 Nov 23 09:35 Magics.py\n",
      "will read readme.md\n",
      "# jupyter_notebook\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "ls -ltr\n",
    "echo \"will read readme.md\"\n",
    "cat README.md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('reddituserpostingbehavior.csv.gz', compression='gzip', nrows=10, sep=',', names=['user']+list(range(25))).fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.4 s, sys: 249 ms, total: 13.7 s\n",
      "Wall time: 14.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import gzip\n",
    "\n",
    "def compute_user_subreddit_ids():\n",
    "    user_ids = []\n",
    "    subreddit_ids = []\n",
    "    subreddit_to_id = {}\n",
    "    i = 0\n",
    "    with gzip.open('reddituserpostingbehavior.csv.gz', 'rb') as f:\n",
    "        for line in f:\n",
    "            this_line = line.decode(\"utf-8\")\n",
    "            for sr in this_line.rstrip().split(',')[1:]:\n",
    "                if sr not in subreddit_to_id:\n",
    "                    subreddit_to_id[sr] = len(subreddit_to_id)\n",
    "                user_ids.append(i)\n",
    "                subreddit_ids.append(subreddit_to_id[sr])\n",
    "            i += 1\n",
    "        return user_ids, subreddit_ids\n",
    "    \n",
    "user_ids, subreddit_ids = compute_user_subreddit_ids()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Showing math equations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "\\begin{aligned}\n",
       "\\nabla \\times \\vec{\\mathbf{B}} -\\, \\frac1c\\, \\frac{\\partial\\vec{\\mathbf{E}}}{\\partial t} & = \\frac{4\\pi}{c}\\vec{\\mathbf{j}} \\\\\n",
       "\\nabla \\cdot \\vec{\\mathbf{E}} & = 4 \\pi \\rho \\\\\n",
       "\\nabla \\times \\vec{\\mathbf{E}}\\, +\\, \\frac1c\\, \\frac{\\partial\\vec{\\mathbf{B}}}{\\partial t} & = \\vec{\\mathbf{0}} \\\\\n",
       "\\nabla \\cdot \\vec{\\mathbf{B}} & = 0\n",
       "\\end{aligned}"
      ],
      "text/plain": [
       "<IPython.core.display.Latex object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%latex\n",
    "\\begin{aligned}\n",
    "\\nabla \\times \\vec{\\mathbf{B}} -\\, \\frac1c\\, \\frac{\\partial\\vec{\\mathbf{E}}}{\\partial t} & = \\frac{4\\pi}{c}\\vec{\\mathbf{j}} \\\\\n",
    "\\nabla \\cdot \\vec{\\mathbf{E}} & = 4 \\pi \\rho \\\\\n",
    "\\nabla \\times \\vec{\\mathbf{E}}\\, +\\, \\frac1c\\, \\frac{\\partial\\vec{\\mathbf{B}}}{\\partial t} & = \\vec{\\mathbf{0}} \\\\\n",
    "\\nabla \\cdot \\vec{\\mathbf{B}} & = 0\n",
    "\\end{aligned}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# showing profiling tools\n",
    "\n",
    "The below command installs line profiler\n",
    "\n",
    "    pip install line_profiler\n",
    "    pip install memory_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The line_profiler extension is already loaded. To reload it, use:\n",
      "  %reload_ext line_profiler\n"
     ]
    }
   ],
   "source": [
    "%load_ext line_profiler\n",
    "%load_ext memory_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# some small functions for example purposes\n",
    "import numpy as np\n",
    "\n",
    "def func(n, a):\n",
    "  y = np.arange(n)\n",
    "  y = np.exp(-y*a)\n",
    "  return y\n",
    "\n",
    "def gunc(n, a):\n",
    "  y = np.exp(-n*a)\n",
    "  return y\n",
    "\n",
    "def hunc(n, a):\n",
    "  y1 = func(n, a)\n",
    "  y2 = gunc(n, a)\n",
    "  return y1, y2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "%lprun -f func hunc(50000, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n",
      "/home/joydeep/anaconda3/envs/venv/lib/python3.6/site-packages/memory_profiler.py:163: UserWarning: psutil module not found. memory_profiler will be slow\n",
      "  warnings.warn(\"psutil module not found. memory_profiler will be slow\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 547.14 MiB, increment: 0.13 MiB\n"
     ]
    }
   ],
   "source": [
    "%memit hunc(50000, 0.5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (venv)",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}