01_Scrape_PFR_Combine_and_AV_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pfr\n",
    "import string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape combine data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_url_template = \"http://www.pro-football-reference.com/draft/{}-combine.htm\"\n",
    "combine_row_css_selector = \"#combine > tbody > tr\" \n",
    "combine_col_css_selector = \"#combine > thead > tr > th.poptip\"\n",
    "combine_dfs = []\n",
    "combine_errors = []\n",
    "for i in range (2000, 2019):\n",
    "    url = combine_url_template.format(i)\n",
    "    try:\n",
    "        df = pfr.get_table(url, combine_row_css_selector, \n",
    "                           combine_col_css_selector, 0)\n",
    "        df[\"Year\"] = i\n",
    "        combine_dfs.append(df)\n",
    "    except Exception as e:\n",
    "        combine_errors.append([url, e])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(combine_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(combine_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_df = pd.concat(combine_dfs, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pos</th>\n",
       "      <th>School</th>\n",
       "      <th>College</th>\n",
       "      <th>Ht</th>\n",
       "      <th>Wt</th>\n",
       "      <th>40yd</th>\n",
       "      <th>Vertical</th>\n",
       "      <th>BenchReps</th>\n",
       "      <th>BroadJump</th>\n",
       "      <th>3Cone</th>\n",
       "      <th>Shuttle</th>\n",
       "      <th>Drafted (tm/rnd/yr)</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(John Abraham, AbraJo00)</td>\n",
       "      <td>OLB</td>\n",
       "      <td>South Carolina</td>\n",
       "      <td></td>\n",
       "      <td>6-4</td>\n",
       "      <td>252</td>\n",
       "      <td>4.55</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>New York Jets / 1st / 13th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(Shaun Alexander, AlexSh00)</td>\n",
       "      <td>RB</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-0</td>\n",
       "      <td>218</td>\n",
       "      <td>4.58</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Seattle Seahawks / 1st / 19th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(Darnell Alford, AlfoDa20)</td>\n",
       "      <td>OT</td>\n",
       "      <td>Boston College</td>\n",
       "      <td></td>\n",
       "      <td>6-4</td>\n",
       "      <td>334</td>\n",
       "      <td>5.56</td>\n",
       "      <td>25.0</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>8.48</td>\n",
       "      <td>4.98</td>\n",
       "      <td>Kansas City Chiefs / 6th / 188th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(Kyle Allamon, None)</td>\n",
       "      <td>TE</td>\n",
       "      <td>Texas Tech</td>\n",
       "      <td></td>\n",
       "      <td>6-2</td>\n",
       "      <td>253</td>\n",
       "      <td>4.97</td>\n",
       "      <td>29.0</td>\n",
       "      <td></td>\n",
       "      <td>104</td>\n",
       "      <td>7.29</td>\n",
       "      <td>4.49</td>\n",
       "      <td></td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(Rashard Anderson, AndeRa21)</td>\n",
       "      <td>CB</td>\n",
       "      <td>Jackson State</td>\n",
       "      <td></td>\n",
       "      <td>6-2</td>\n",
       "      <td>206</td>\n",
       "      <td>4.55</td>\n",
       "      <td>34.0</td>\n",
       "      <td></td>\n",
       "      <td>123</td>\n",
       "      <td>7.18</td>\n",
       "      <td>4.15</td>\n",
       "      <td>Carolina Panthers / 1st / 23rd pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Player  Pos          School        College   Ht   Wt  \\\n",
       "0      (John Abraham, AbraJo00)  OLB  South Carolina                 6-4  252   \n",
       "1   (Shaun Alexander, AlexSh00)   RB         Alabama  College Stats  6-0  218   \n",
       "2    (Darnell Alford, AlfoDa20)   OT  Boston College                 6-4  334   \n",
       "3          (Kyle Allamon, None)   TE      Texas Tech                 6-2  253   \n",
       "4  (Rashard Anderson, AndeRa21)   CB   Jackson State                 6-2  206   \n",
       "\n",
       "   40yd Vertical BenchReps BroadJump 3Cone Shuttle  \\\n",
       "0  4.55                                              \n",
       "1  4.58                                              \n",
       "2  5.56     25.0        23        94  8.48    4.98   \n",
       "3  4.97     29.0                 104  7.29    4.49   \n",
       "4  4.55     34.0                 123  7.18    4.15   \n",
       "\n",
       "                            Drafted (tm/rnd/yr)  Year  \n",
       "0        New York Jets / 1st / 13th pick / 2000  2000  \n",
       "1     Seattle Seahawks / 1st / 19th pick / 2000  2000  \n",
       "2  Kansas City Chiefs / 6th / 188th pick / 2000  2000  \n",
       "3                                                2000  \n",
       "4    Carolina Panthers / 1st / 23rd pick / 2000  2000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pos</th>\n",
       "      <th>School</th>\n",
       "      <th>College</th>\n",
       "      <th>Ht</th>\n",
       "      <th>Wt</th>\n",
       "      <th>40yd</th>\n",
       "      <th>Vertical</th>\n",
       "      <th>BenchReps</th>\n",
       "      <th>BroadJump</th>\n",
       "      <th>3Cone</th>\n",
       "      <th>Shuttle</th>\n",
       "      <th>Drafted (tm/rnd/yr)</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6213</th>\n",
       "      <td>(Chris Worley, WorlCh00)</td>\n",
       "      <td>ILB</td>\n",
       "      <td>Ohio St.</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-2</td>\n",
       "      <td>238</td>\n",
       "      <td>4.86</td>\n",
       "      <td>29.5</td>\n",
       "      <td>15</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6214</th>\n",
       "      <td>(Isaiah Wynn, WynnIs00)</td>\n",
       "      <td>G</td>\n",
       "      <td>Georgia</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-3</td>\n",
       "      <td>313</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6215</th>\n",
       "      <td>(Isaac Yiadom, YiadIs00)</td>\n",
       "      <td>CB</td>\n",
       "      <td>Boston College</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-1</td>\n",
       "      <td>190</td>\n",
       "      <td>4.52</td>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>120</td>\n",
       "      <td></td>\n",
       "      <td>4.18</td>\n",
       "      <td></td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6216</th>\n",
       "      <td>(Kenny Young, None)</td>\n",
       "      <td>ILB</td>\n",
       "      <td>UCLA</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-1</td>\n",
       "      <td>236</td>\n",
       "      <td>4.60</td>\n",
       "      <td>36.0</td>\n",
       "      <td>23</td>\n",
       "      <td>117</td>\n",
       "      <td>7.38</td>\n",
       "      <td>4.48</td>\n",
       "      <td></td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6217</th>\n",
       "      <td>(Trevon Young, YounTr00)</td>\n",
       "      <td>EDGE</td>\n",
       "      <td>Louisville</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-4</td>\n",
       "      <td>258</td>\n",
       "      <td>4.78</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25</td>\n",
       "      <td>114</td>\n",
       "      <td>6.99</td>\n",
       "      <td>4.40</td>\n",
       "      <td></td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Player   Pos          School        College   Ht   Wt  \\\n",
       "6213  (Chris Worley, WorlCh00)   ILB        Ohio St.  College Stats  6-2  238   \n",
       "6214   (Isaiah Wynn, WynnIs00)     G         Georgia  College Stats  6-3  313   \n",
       "6215  (Isaac Yiadom, YiadIs00)    CB  Boston College  College Stats  6-1  190   \n",
       "6216       (Kenny Young, None)   ILB            UCLA  College Stats  6-1  236   \n",
       "6217  (Trevon Young, YounTr00)  EDGE      Louisville  College Stats  6-4  258   \n",
       "\n",
       "      40yd Vertical BenchReps BroadJump 3Cone Shuttle Drafted (tm/rnd/yr)  \\\n",
       "6213  4.86     29.5        15                                               \n",
       "6214                                                                        \n",
       "6215  4.52                  8       120          4.18                       \n",
       "6216  4.60     36.0        23       117  7.38    4.48                       \n",
       "6217  4.78     33.0        25       114  6.99    4.40                       \n",
       "\n",
       "      Year  \n",
       "6213  2018  \n",
       "6214  2018  \n",
       "6215  2018  \n",
       "6216  2018  \n",
       "6217  2018  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6218 entries, 0 to 6217\n",
      "Data columns (total 14 columns):\n",
      "Player                 6218 non-null object\n",
      "Pos                    6218 non-null object\n",
      "School                 6218 non-null object\n",
      "College                6218 non-null object\n",
      "Ht                     6218 non-null object\n",
      "Wt                     6218 non-null object\n",
      "40yd                   6218 non-null object\n",
      "Vertical               6218 non-null object\n",
      "BenchReps              6218 non-null object\n",
      "BroadJump              6218 non-null object\n",
      "3Cone                  6218 non-null object\n",
      "Shuttle                6218 non-null object\n",
      "Drafted (tm/rnd/yr)    6218 non-null object\n",
      "Year                   6218 non-null int64\n",
      "dtypes: int64(1), object(13)\n",
      "memory usage: 680.2+ KB\n"
     ]
    }
   ],
   "source": [
    "combine_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_df[[\"Player\", \"Pfr_ID\"]] = combine_df.Player.apply(pd.Series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pos</th>\n",
       "      <th>School</th>\n",
       "      <th>College</th>\n",
       "      <th>Ht</th>\n",
       "      <th>Wt</th>\n",
       "      <th>40yd</th>\n",
       "      <th>Vertical</th>\n",
       "      <th>BenchReps</th>\n",
       "      <th>BroadJump</th>\n",
       "      <th>3Cone</th>\n",
       "      <th>Shuttle</th>\n",
       "      <th>Drafted (tm/rnd/yr)</th>\n",
       "      <th>Year</th>\n",
       "      <th>Pfr_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>John Abraham</td>\n",
       "      <td>OLB</td>\n",
       "      <td>South Carolina</td>\n",
       "      <td></td>\n",
       "      <td>6-4</td>\n",
       "      <td>252</td>\n",
       "      <td>4.55</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>New York Jets / 1st / 13th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AbraJo00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shaun Alexander</td>\n",
       "      <td>RB</td>\n",
       "      <td>Alabama</td>\n",
       "      <td>College Stats</td>\n",
       "      <td>6-0</td>\n",
       "      <td>218</td>\n",
       "      <td>4.58</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Seattle Seahawks / 1st / 19th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AlexSh00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Darnell Alford</td>\n",
       "      <td>OT</td>\n",
       "      <td>Boston College</td>\n",
       "      <td></td>\n",
       "      <td>6-4</td>\n",
       "      <td>334</td>\n",
       "      <td>5.56</td>\n",
       "      <td>25.0</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>8.48</td>\n",
       "      <td>4.98</td>\n",
       "      <td>Kansas City Chiefs / 6th / 188th pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AlfoDa20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kyle Allamon</td>\n",
       "      <td>TE</td>\n",
       "      <td>Texas Tech</td>\n",
       "      <td></td>\n",
       "      <td>6-2</td>\n",
       "      <td>253</td>\n",
       "      <td>4.97</td>\n",
       "      <td>29.0</td>\n",
       "      <td></td>\n",
       "      <td>104</td>\n",
       "      <td>7.29</td>\n",
       "      <td>4.49</td>\n",
       "      <td></td>\n",
       "      <td>2000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Rashard Anderson</td>\n",
       "      <td>CB</td>\n",
       "      <td>Jackson State</td>\n",
       "      <td></td>\n",
       "      <td>6-2</td>\n",
       "      <td>206</td>\n",
       "      <td>4.55</td>\n",
       "      <td>34.0</td>\n",
       "      <td></td>\n",
       "      <td>123</td>\n",
       "      <td>7.18</td>\n",
       "      <td>4.15</td>\n",
       "      <td>Carolina Panthers / 1st / 23rd pick / 2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>AndeRa21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Player  Pos          School        College   Ht   Wt  40yd  \\\n",
       "0      John Abraham  OLB  South Carolina                 6-4  252  4.55   \n",
       "1   Shaun Alexander   RB         Alabama  College Stats  6-0  218  4.58   \n",
       "2    Darnell Alford   OT  Boston College                 6-4  334  5.56   \n",
       "3      Kyle Allamon   TE      Texas Tech                 6-2  253  4.97   \n",
       "4  Rashard Anderson   CB   Jackson State                 6-2  206  4.55   \n",
       "\n",
       "  Vertical BenchReps BroadJump 3Cone Shuttle  \\\n",
       "0                                              \n",
       "1                                              \n",
       "2     25.0        23        94  8.48    4.98   \n",
       "3     29.0                 104  7.29    4.49   \n",
       "4     34.0                 123  7.18    4.15   \n",
       "\n",
       "                            Drafted (tm/rnd/yr)  Year    Pfr_ID  \n",
       "0        New York Jets / 1st / 13th pick / 2000  2000  AbraJo00  \n",
       "1     Seattle Seahawks / 1st / 19th pick / 2000  2000  AlexSh00  \n",
       "2  Kansas City Chiefs / 6th / 188th pick / 2000  2000  AlfoDa20  \n",
       "3                                                2000      None  \n",
       "4    Carolina Panthers / 1st / 23rd pick / 2000  2000  AndeRa21  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "combine_df.to_csv('data/combine_data_RAW_2018-03-13.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape 3yr AV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note year_min=2000 and year_max=2017\n",
    "# will iterate through different values for offset to scrape\n",
    "# all the pages of AV values\n",
    "av_url_template = \"http://www.pro-football-reference.com/play-index/psl_finder.\"\\\n",
    "                  \"cgi?request=1&match=combined&year_min=2000&year_max=2017&seas\"\\\n",
    "                  \"on_start=1&season_end=3&pos%5B%5D=qb&pos%5B%5D=rb&pos%5B%5D=w\"\\\n",
    "                  \"r&pos%5B%5D=te&pos%5B%5D=e&pos%5B%5D=t&pos%5B%5D=g&pos%5B%5D=\"\\\n",
    "                  \"c&pos%5B%5D=ol&pos%5B%5D=dt&pos%5B%5D=de&pos%5B%5D=dl&pos%5B%\"\\\n",
    "                  \"5D=ilb&pos%5B%5D=olb&pos%5B%5D=lb&pos%5B%5D=cb&pos%5B%5D=s&po\"\\\n",
    "                  \"s%5B%5D=db&pos%5B%5D=k&pos%5B%5D=p&draft_year_min=1936&draft_\"\\\n",
    "                  \"year_max=2017&draft_slot_min=1&draft_slot_max=500&draft_pick_\"\\\n",
    "                  \"in_round=pick_overall&conference=any&draft_pos%5B%5D=qb&draft\"\\\n",
    "                  \"_pos%5B%5D=rb&draft_pos%5B%5D=wr&draft_pos%5B%5D=te&draft_pos\"\\\n",
    "                  \"%5B%5D=e&draft_pos%5B%5D=t&draft_pos%5B%5D=g&draft_pos%5B%5D=\"\\\n",
    "                  \"c&draft_pos%5B%5D=ol&draft_pos%5B%5D=dt&draft_pos%5B%5D=de&dr\"\\\n",
    "                  \"aft_pos%5B%5D=dl&draft_pos%5B%5D=ilb&draft_pos%5B%5D=olb&draf\"\\\n",
    "                  \"t_pos%5B%5D=lb&draft_pos%5B%5D=cb&draft_pos%5B%5D=s&draft_pos\"\\\n",
    "                  \"%5B%5D=db&draft_pos%5B%5D=k&draft_pos%5B%5D=p&c5val=1.0&order\"\\\n",
    "                  \"_by=av&offset={}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_row_css_selector = \"#results > tbody > tr\" \n",
    "av_col_css_selector = \"#results > thead > tr:nth-child(2) > th\"\n",
    "av_dfs = []\n",
    "av_errors = []\n",
    "# this range is found by manually checking out what\n",
    "# the offset value is for the last page of results\n",
    "# in \n",
    "for i in range (0, 7801, 100):  \n",
    "    url = av_url_template.format(i)\n",
    "    try:\n",
    "        df = pfr.get_table(url, av_row_css_selector, \n",
    "                           av_col_css_selector, 1)\n",
    "        av_dfs.append(df)\n",
    "    except Exception as e:\n",
    "        av_errors.append([url, e])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(av_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(av_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rk</th>\n",
       "      <th>Player</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Tm</th>\n",
       "      <th>Lg</th>\n",
       "      <th>G</th>\n",
       "      <th>GS</th>\n",
       "      <th>Yrs</th>\n",
       "      <th>PB</th>\n",
       "      <th>AP1</th>\n",
       "      <th>AV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>(Cam Newton, NewtCa00)</td>\n",
       "      <td>2011</td>\n",
       "      <td>2013</td>\n",
       "      <td>1-1</td>\n",
       "      <td>CAR</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>(Russell Wilson, WilsRu00)</td>\n",
       "      <td>2012</td>\n",
       "      <td>2014</td>\n",
       "      <td>3-75</td>\n",
       "      <td>SEA</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>(Patrick Willis, WillPa98)</td>\n",
       "      <td>2007</td>\n",
       "      <td>2009</td>\n",
       "      <td>1-11</td>\n",
       "      <td>SFO</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>(Patrick Peterson, PetePa00)</td>\n",
       "      <td>2011</td>\n",
       "      <td>2013</td>\n",
       "      <td>1-5</td>\n",
       "      <td>CRD</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>(Aaron Donald, DonaAa00)</td>\n",
       "      <td>2014</td>\n",
       "      <td>2016</td>\n",
       "      <td>1-13</td>\n",
       "      <td>RAM</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>44</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Rk                        Player  From    To Draft   Tm   Lg   G  GS Yrs PB  \\\n",
       "0  1        (Cam Newton, NewtCa00)  2011  2013   1-1  CAR  NFL  48  48   3  2   \n",
       "1  2    (Russell Wilson, WilsRu00)  2012  2014  3-75  SEA  NFL  48  48   3  2   \n",
       "2  3    (Patrick Willis, WillPa98)  2007  2009  1-11  SFO  NFL  48  48   3  3   \n",
       "3  4  (Patrick Peterson, PetePa00)  2011  2013   1-5  CRD  NFL  48  48   3  3   \n",
       "4  5      (Aaron Donald, DonaAa00)  2014  2016  1-13  RAM  NFL  48  44   3  3   \n",
       "\n",
       "  AP1  AV  \n",
       "0   0  51  \n",
       "1   0  51  \n",
       "2   2  49  \n",
       "3   2  48  \n",
       "4   2  46  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "av_df = pd.concat(av_dfs, ignore_index=True)\n",
    "av_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rk</th>\n",
       "      <th>Player</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Tm</th>\n",
       "      <th>Lg</th>\n",
       "      <th>G</th>\n",
       "      <th>GS</th>\n",
       "      <th>Yrs</th>\n",
       "      <th>PB</th>\n",
       "      <th>AP1</th>\n",
       "      <th>AV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7865</th>\n",
       "      <td>7866</td>\n",
       "      <td>(Chris Young, YounCh03)</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>NYJ</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7866</th>\n",
       "      <td>7867</td>\n",
       "      <td>(Michael Zimmer, ZimmMi00)</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>SEA</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7867</th>\n",
       "      <td>7868</td>\n",
       "      <td>(Ty Zimmerman, ZimmTy00)</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>NOR</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7868</th>\n",
       "      <td>7869</td>\n",
       "      <td>(Michael Zordich, ZordMi01)</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>NOR</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7869</th>\n",
       "      <td>7870</td>\n",
       "      <td>(Jordan Zumwalt, ZumwJo00)</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td>6-192</td>\n",
       "      <td>PIT</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Rk                       Player  From    To  Draft   Tm   Lg  G GS  \\\n",
       "7865  7866      (Chris Young, YounCh03)  2014  2014         NYJ  NFL  0      \n",
       "7866  7867   (Michael Zimmer, ZimmMi00)  2014  2014         SEA  NFL  0      \n",
       "7867  7868     (Ty Zimmerman, ZimmTy00)  2014  2014         NOR  NFL  0      \n",
       "7868  7869  (Michael Zordich, ZordMi01)  2014  2014         NOR  NFL  0      \n",
       "7869  7870   (Jordan Zumwalt, ZumwJo00)  2014  2014  6-192  PIT  NFL  0      \n",
       "\n",
       "     Yrs PB AP1 AV  \n",
       "7865   1  0   0     \n",
       "7866   1  0   0     \n",
       "7867   1  0   0     \n",
       "7868   1  0   0     \n",
       "7869   1  0   0     "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "av_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7870 entries, 0 to 7869\n",
      "Data columns (total 13 columns):\n",
      "Rk        7870 non-null object\n",
      "Player    7870 non-null object\n",
      "From      7870 non-null object\n",
      "To        7870 non-null object\n",
      "Draft     7870 non-null object\n",
      "Tm        7870 non-null object\n",
      "Lg        7870 non-null object\n",
      "G         7870 non-null object\n",
      "GS        7870 non-null object\n",
      "Yrs       7870 non-null object\n",
      "PB        7870 non-null object\n",
      "AP1       7870 non-null object\n",
      "AV        7870 non-null object\n",
      "dtypes: object(13)\n",
      "memory usage: 799.4+ KB\n"
     ]
    }
   ],
   "source": [
    "av_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_df[[\"Player\", \"Pfr_ID\"]] = av_df.Player.apply(pd.Series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7870 entries, 0 to 7869\n",
      "Data columns (total 14 columns):\n",
      "Rk        7870 non-null object\n",
      "Player    7870 non-null object\n",
      "From      7870 non-null object\n",
      "To        7870 non-null object\n",
      "Draft     7870 non-null object\n",
      "Tm        7870 non-null object\n",
      "Lg        7870 non-null object\n",
      "G         7870 non-null object\n",
      "GS        7870 non-null object\n",
      "Yrs       7870 non-null object\n",
      "PB        7870 non-null object\n",
      "AP1       7870 non-null object\n",
      "AV        7870 non-null object\n",
      "Pfr_ID    7870 non-null object\n",
      "dtypes: object(14)\n",
      "memory usage: 860.9+ KB\n"
     ]
    }
   ],
   "source": [
    "av_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rk</th>\n",
       "      <th>Player</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Tm</th>\n",
       "      <th>Lg</th>\n",
       "      <th>G</th>\n",
       "      <th>GS</th>\n",
       "      <th>Yrs</th>\n",
       "      <th>PB</th>\n",
       "      <th>AP1</th>\n",
       "      <th>AV</th>\n",
       "      <th>Pfr_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Cam Newton</td>\n",
       "      <td>2011</td>\n",
       "      <td>2013</td>\n",
       "      <td>1-1</td>\n",
       "      <td>CAR</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "      <td>NewtCa00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Russell Wilson</td>\n",
       "      <td>2012</td>\n",
       "      <td>2014</td>\n",
       "      <td>3-75</td>\n",
       "      <td>SEA</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>51</td>\n",
       "      <td>WilsRu00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Patrick Willis</td>\n",
       "      <td>2007</td>\n",
       "      <td>2009</td>\n",
       "      <td>1-11</td>\n",
       "      <td>SFO</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>WillPa98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Patrick Peterson</td>\n",
       "      <td>2011</td>\n",
       "      <td>2013</td>\n",
       "      <td>1-5</td>\n",
       "      <td>CRD</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>PetePa00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Aaron Donald</td>\n",
       "      <td>2014</td>\n",
       "      <td>2016</td>\n",
       "      <td>1-13</td>\n",
       "      <td>RAM</td>\n",
       "      <td>NFL</td>\n",
       "      <td>48</td>\n",
       "      <td>44</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>46</td>\n",
       "      <td>DonaAa00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Rk            Player  From    To Draft   Tm   Lg   G  GS Yrs PB AP1  AV  \\\n",
       "0  1        Cam Newton  2011  2013   1-1  CAR  NFL  48  48   3  2   0  51   \n",
       "1  2    Russell Wilson  2012  2014  3-75  SEA  NFL  48  48   3  2   0  51   \n",
       "2  3    Patrick Willis  2007  2009  1-11  SFO  NFL  48  48   3  3   2  49   \n",
       "3  4  Patrick Peterson  2011  2013   1-5  CRD  NFL  48  48   3  3   2  48   \n",
       "4  5      Aaron Donald  2014  2016  1-13  RAM  NFL  48  44   3  3   2  46   \n",
       "\n",
       "     Pfr_ID  \n",
       "0  NewtCa00  \n",
       "1  WilsRu00  \n",
       "2  WillPa98  \n",
       "3  PetePa00  \n",
       "4  DonaAa00  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "av_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rk</th>\n",
       "      <th>Player</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Draft</th>\n",
       "      <th>Tm</th>\n",
       "      <th>Lg</th>\n",
       "      <th>G</th>\n",
       "      <th>GS</th>\n",
       "      <th>Yrs</th>\n",
       "      <th>PB</th>\n",
       "      <th>AP1</th>\n",
       "      <th>AV</th>\n",
       "      <th>Pfr_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7865</th>\n",
       "      <td>7866</td>\n",
       "      <td>Chris Young</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>NYJ</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>YounCh03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7866</th>\n",
       "      <td>7867</td>\n",
       "      <td>Michael Zimmer</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>SEA</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>ZimmMi00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7867</th>\n",
       "      <td>7868</td>\n",
       "      <td>Ty Zimmerman</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>NOR</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>ZimmTy00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7868</th>\n",
       "      <td>7869</td>\n",
       "      <td>Michael Zordich</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td></td>\n",
       "      <td>NOR</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>ZordMi01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7869</th>\n",
       "      <td>7870</td>\n",
       "      <td>Jordan Zumwalt</td>\n",
       "      <td>2014</td>\n",
       "      <td>2014</td>\n",
       "      <td>6-192</td>\n",
       "      <td>PIT</td>\n",
       "      <td>NFL</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>ZumwJo00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Rk           Player  From    To  Draft   Tm   Lg  G GS Yrs PB AP1 AV  \\\n",
       "7865  7866      Chris Young  2014  2014         NYJ  NFL  0      1  0   0      \n",
       "7866  7867   Michael Zimmer  2014  2014         SEA  NFL  0      1  0   0      \n",
       "7867  7868     Ty Zimmerman  2014  2014         NOR  NFL  0      1  0   0      \n",
       "7868  7869  Michael Zordich  2014  2014         NOR  NFL  0      1  0   0      \n",
       "7869  7870   Jordan Zumwalt  2014  2014  6-192  PIT  NFL  0      1  0   0      \n",
       "\n",
       "        Pfr_ID  \n",
       "7865  YounCh03  \n",
       "7866  ZimmMi00  \n",
       "7867  ZimmTy00  \n",
       "7868  ZordMi01  \n",
       "7869  ZumwJo00  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "av_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "av_df.to_csv('data/first_3_season_av_since_2000_RAW_2018-03-13.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape Player Info data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use all letters cept X to pass into the url template and then scrape the data\n",
    "abc = string.ascii_uppercase.replace(\"X\", \"\")\n",
    "player_info_dfs = []\n",
    "player_info_errors = []\n",
    "player_info_url_template = \"http://www.pro-football-reference.com/players/{}/\"\n",
    "for i in abc:\n",
    "    url = player_info_url_template.format(i)\n",
    "    try:\n",
    "        df = pfr.get_pfr_player_ids_and_info(url)\n",
    "        player_info_dfs.append(df)\n",
    "    except Exception as e:\n",
    "        player_info_errors.append([url, e])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(player_info_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(player_info_errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "player_info_df = pd.concat(player_info_dfs, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pfr_ID</th>\n",
       "      <th>Pos</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Link</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Isaako Aaitui</td>\n",
       "      <td>AaitIs00</td>\n",
       "      <td>NT</td>\n",
       "      <td>2013</td>\n",
       "      <td>2014</td>\n",
       "      <td>/players/A/AaitIs00.htm</td>\n",
       "      <td>Isaako Aaitui (NT) 2013-2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Joe Abbey</td>\n",
       "      <td>AbbeJo20</td>\n",
       "      <td>E</td>\n",
       "      <td>1948</td>\n",
       "      <td>1949</td>\n",
       "      <td>/players/A/AbbeJo20.htm</td>\n",
       "      <td>Joe Abbey (E) 1948-1949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fay Abbott</td>\n",
       "      <td>AbboFa20</td>\n",
       "      <td>BB-FB-TB-QB-WB-E</td>\n",
       "      <td>1921</td>\n",
       "      <td>1929</td>\n",
       "      <td>/players/A/AbboFa20.htm</td>\n",
       "      <td>Fay Abbott (BB-FB-TB-QB-WB-E) 1921-1929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vince Abbott</td>\n",
       "      <td>abbotvin01</td>\n",
       "      <td>K</td>\n",
       "      <td>1987</td>\n",
       "      <td>1988</td>\n",
       "      <td>/players/A/abbotvin01.htm</td>\n",
       "      <td>Vince Abbott (K) 1987-1988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jared Abbrederis</td>\n",
       "      <td>AbbrJa00</td>\n",
       "      <td>WR</td>\n",
       "      <td>2014</td>\n",
       "      <td>2017</td>\n",
       "      <td>/players/A/AbbrJa00.htm</td>\n",
       "      <td>Jared Abbrederis (WR) 2014-2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Player      Pfr_ID               Pos  From    To  \\\n",
       "0     Isaako Aaitui    AaitIs00                NT  2013  2014   \n",
       "1         Joe Abbey    AbbeJo20                 E  1948  1949   \n",
       "2        Fay Abbott    AbboFa20  BB-FB-TB-QB-WB-E  1921  1929   \n",
       "3      Vince Abbott  abbotvin01                 K  1987  1988   \n",
       "4  Jared Abbrederis    AbbrJa00                WR  2014  2017   \n",
       "\n",
       "                        Link                                     Text  \n",
       "0    /players/A/AaitIs00.htm             Isaako Aaitui (NT) 2013-2014  \n",
       "1    /players/A/AbbeJo20.htm                  Joe Abbey (E) 1948-1949  \n",
       "2    /players/A/AbboFa20.htm  Fay Abbott (BB-FB-TB-QB-WB-E) 1921-1929  \n",
       "3  /players/A/abbotvin01.htm               Vince Abbott (K) 1987-1988  \n",
       "4    /players/A/AbbrJa00.htm          Jared Abbrederis (WR) 2014-2017  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "player_info_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Player</th>\n",
       "      <th>Pfr_ID</th>\n",
       "      <th>Pos</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>Link</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25372</th>\n",
       "      <td>Markus Zusevics</td>\n",
       "      <td>ZuseMa00</td>\n",
       "      <td>OL</td>\n",
       "      <td>2012</td>\n",
       "      <td>2012</td>\n",
       "      <td>/players/Z/ZuseMa00.htm</td>\n",
       "      <td>Markus Zusevics (OL) 2012-2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25373</th>\n",
       "      <td>Jeremy Zuttah</td>\n",
       "      <td>ZuttJe20</td>\n",
       "      <td>G</td>\n",
       "      <td>2008</td>\n",
       "      <td>2016</td>\n",
       "      <td>/players/Z/ZuttJe20.htm</td>\n",
       "      <td>Jeremy Zuttah (G) 2008-2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25374</th>\n",
       "      <td>Merle Zuver</td>\n",
       "      <td>ZuveMe20</td>\n",
       "      <td>G-C</td>\n",
       "      <td>1930</td>\n",
       "      <td>1930</td>\n",
       "      <td>/players/Z/ZuveMe20.htm</td>\n",
       "      <td>Merle Zuver (G-C) 1930-1930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25375</th>\n",
       "      <td>Tony Zuzzio</td>\n",
       "      <td>ZuzzTo20</td>\n",
       "      <td>G</td>\n",
       "      <td>1942</td>\n",
       "      <td>1942</td>\n",
       "      <td>/players/Z/ZuzzTo20.htm</td>\n",
       "      <td>Tony Zuzzio (G) 1942-1942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25376</th>\n",
       "      <td>Jim Zyntell</td>\n",
       "      <td>ZyntJi20</td>\n",
       "      <td>G</td>\n",
       "      <td>1933</td>\n",
       "      <td>1935</td>\n",
       "      <td>/players/Z/ZyntJi20.htm</td>\n",
       "      <td>Jim Zyntell (G) 1933-1935</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Player    Pfr_ID  Pos  From    To                     Link  \\\n",
       "25372  Markus Zusevics  ZuseMa00   OL  2012  2012  /players/Z/ZuseMa00.htm   \n",
       "25373    Jeremy Zuttah  ZuttJe20    G  2008  2016  /players/Z/ZuttJe20.htm   \n",
       "25374      Merle Zuver  ZuveMe20  G-C  1930  1930  /players/Z/ZuveMe20.htm   \n",
       "25375      Tony Zuzzio  ZuzzTo20    G  1942  1942  /players/Z/ZuzzTo20.htm   \n",
       "25376      Jim Zyntell  ZyntJi20    G  1933  1935  /players/Z/ZyntJi20.htm   \n",
       "\n",
       "                                 Text  \n",
       "25372  Markus Zusevics (OL) 2012-2012  \n",
       "25373     Jeremy Zuttah (G) 2008-2016  \n",
       "25374     Merle Zuver (G-C) 1930-1930  \n",
       "25375       Tony Zuzzio (G) 1942-1942  \n",
       "25376       Jim Zyntell (G) 1933-1935  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "player_info_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 25377 entries, 0 to 25376\n",
      "Data columns (total 7 columns):\n",
      "Player    25377 non-null object\n",
      "Pfr_ID    25377 non-null object\n",
      "Pos       25377 non-null object\n",
      "From      25377 non-null int64\n",
      "To        25377 non-null int64\n",
      "Link      25377 non-null object\n",
      "Text      25377 non-null object\n",
      "dtypes: int64(2), object(5)\n",
      "memory usage: 1.4+ MB\n"
     ]
    }
   ],
   "source": [
    "player_info_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "player_info_df.to_csv('data/pfr_pids_and_info_RAW_2018-03-13.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}