Scraping with Beautiful Soup and Selenium.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ff3a833",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scraping MLB data with BeautifulSoup and Selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "3a40c9c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\dave_\\\\Desktop\\\\git_test\\\\projects\\\\clean'"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0dac9376",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from unicodedata import normalize\n",
    "\n",
    "table_MN = pd.read_html('https://en.wikipedia.org/wiki/Minnesota')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "05082534",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total tables: 29\n"
     ]
    }
   ],
   "source": [
    "print(f'Total tables: {len(table_MN)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a3e6316a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_MN = pd.read_html('https://en.wikipedia.org/wiki/Minnesota', match='Election results from statewide races')\n",
    "len(table_MN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9b003c8e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Office</th>\n",
       "      <th>GOP</th>\n",
       "      <th>DFL</th>\n",
       "      <th>Others</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020</td>\n",
       "      <td>President</td>\n",
       "      <td>45.3%</td>\n",
       "      <td>52.4%</td>\n",
       "      <td>2.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020</td>\n",
       "      <td>Senator</td>\n",
       "      <td>43.5%</td>\n",
       "      <td>48.8%</td>\n",
       "      <td>7.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018</td>\n",
       "      <td>Governor</td>\n",
       "      <td>42.4%</td>\n",
       "      <td>53.9%</td>\n",
       "      <td>3.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018</td>\n",
       "      <td>Senator</td>\n",
       "      <td>36.2%</td>\n",
       "      <td>60.3%</td>\n",
       "      <td>3.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018</td>\n",
       "      <td>Senator</td>\n",
       "      <td>42.4%</td>\n",
       "      <td>53.0%</td>\n",
       "      <td>4.6%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year     Office    GOP    DFL Others\n",
       "0  2020  President  45.3%  52.4%   2.3%\n",
       "1  2020    Senator  43.5%  48.8%   7.7%\n",
       "2  2018   Governor  42.4%  53.9%   3.7%\n",
       "3  2018    Senator  36.2%  60.3%   3.4%\n",
       "4  2018    Senator  42.4%  53.0%   4.6%"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = table_MN[0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "57487e1a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 26 entries, 0 to 25\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   Year    26 non-null     int64 \n",
      " 1   Office  26 non-null     object\n",
      " 2   GOP     26 non-null     object\n",
      " 3   DFL     26 non-null     object\n",
      " 4   Others  26 non-null     object\n",
      "dtypes: int64(1), object(4)\n",
      "memory usage: 1.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "38bd3301",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 26 entries, 0 to 25\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype  \n",
      "---  ------  --------------  -----  \n",
      " 0   Year    26 non-null     int64  \n",
      " 1   Office  26 non-null     object \n",
      " 2   GOP     26 non-null     float64\n",
      " 3   DFL     26 non-null     float64\n",
      " 4   Others  26 non-null     float64\n",
      "dtypes: float64(3), int64(1), object(1)\n",
      "memory usage: 1.1+ KB\n"
     ]
    }
   ],
   "source": [
    "#df['GOP'].replace({'%':''}, regex=True).astype('float')\n",
    "\n",
    "df = df.replace({'%': ''}, regex=True)\n",
    "df[['GOP', 'DFL', 'Others']] = df[['GOP', 'DFL', 'Others']].apply(pd.to_numeric)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "824424c6",
   "metadata": {},
   "source": [
    "# bs4 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1dac4593",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "3169fb77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"https://zlliu.s3.ap-southeast-1.amazonaws.com/index.html\"\n",
    "r = requests.get(url)\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f33c4bfc",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'<div class=\"header\">\\n    <a class=\"logo\">Logo goes here</a>\\n\\n    <div class=\"header-right\">\\n        <a class=\"active\" href=\"\">Home</a>\\n        <a href=\"\">Contact</a>\\n        <a href=\"\">About</a>\\n    </div>\\n</div>\\n  \\n<div class=\"content\">\\n\\n    <div class=\"row\">\\n\\n        <div class=\"left\">\\n            <div>\\n                Shipped from: <span id=\"origin\"> Singapore </span>\\n            </div><br>\\n            <div id=\"description\">\\n                Only the freshest fruits shipped from the state of Singapore, \\n                also historically known by its indigenous name Pulau Ujong.\\n                Description continues ...\\n            </div><br>\\n            <div>Last updated: <span id=\"last-updated-datetime\">20 Nov 21, 1500 hours</span></div>\\n        </div>\\n\\n        <div class=\"right\">\\n            <h2>Fruits for sale</h2>\\n            <table id=\"fruits\">\\n                <tr><th>Fruit</th><th>Price ($)</th></tr>\\n                <tr><td>Apple</td><td>4</td></tr>\\n                <tr><td>Orange</td><td>5</td></tr>\\n                <tr><td>Pear</td><td>6</td></tr>\\n                <tr><td>Durian</td><td>7</td></tr>\\n            </table>\\n        </div>\\n\\n    </div>\\n\\n</div>\\n\\n<div id=\"other-stuff\">\\n    Other stuff goes here\\n</div>\\n\\n<style>\\n\\n    .content {\\n        padding: 3% 10% 3% 10%;\\n    }\\n\\n    #other-stuff {\\n        padding: 3% 10% 3% 10%;\\n    }\\n\\n    .left, .right {\\n        width: 49%;\\n        display: inline-block;\\n    }\\n\\n    .left {\\n        font-size: 24px;\\n\\n    }\\n\\n    .right {\\n        float: right;\\n    }\\n\\n    #origin {\\n        font-weight:600;\\n    }\\n\\n    #last-updated-datetime {\\n        font-weight:600;\\n    }\\n\\n    * {box-sizing: border-box;}\\n    \\n    body { \\n      margin: 0;\\n      font-family: Arial, Helvetica, sans-serif;\\n    }\\n    \\n    .header {\\n      overflow: hidden;\\n      background-color: #f1f1f1;\\n      padding: 20px 10px;\\n    }\\n    \\n    .header a {\\n      float: left;\\n      color: black;\\n      text-align: center;\\n      padding: 12px;\\n      text-decoration: none;\\n      font-size: 18px; \\n      line-height: 25px;\\n      border-radius: 4px;\\n    }\\n    \\n    .header a.logo {\\n      font-size: 25px;\\n      font-weight: bold;\\n    }\\n    \\n    .header a:hover {\\n      background-color: #ddd;\\n      color: black;\\n    }\\n    \\n    .header a.active {\\n      background-color: dodgerblue;\\n      color: white;\\n    }\\n    \\n    .header-right {\\n      float: right;\\n    }\\n    \\n    @media screen and (max-width: 500px) {\\n      .header a {\\n        float: none;\\n        display: block;\\n        text-align: left;\\n      }\\n      \\n      .header-right {\\n        float: none;\\n      }\\n    }\\n\\n    #fruits {\\n        font-family: Arial, Helvetica, sans-serif;\\n        border-collapse: collapse;\\n        width: 100%;\\n    }\\n\\n    #fruits td, #fruits th {\\n        border: 1px solid #ddd;\\n        padding: 8px;\\n    }\\n\\n    #fruits tr:nth-child(even){background-color: #f2f2f2;}\\n\\n    #fruits tr:hover {background-color: #ddd;}\\n\\n    #fruits th {\\n        padding-top: 12px;\\n        padding-bottom: 12px;\\n        text-align: left;\\n        background-color: #f5b800;\\n        color: white;\\n    }\\n\\n</style>'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e17406aa",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<div class=\"header\">\n",
       "<a class=\"logo\">Logo goes here</a>\n",
       "<div class=\"header-right\">\n",
       "<a class=\"active\" href=\"\">Home</a>\n",
       "<a href=\"\">Contact</a>\n",
       "<a href=\"\">About</a>\n",
       "</div>\n",
       "</div>\n",
       "<div class=\"content\">\n",
       "<div class=\"row\">\n",
       "<div class=\"left\">\n",
       "<div>\n",
       "                Shipped from: <span id=\"origin\"> Singapore </span>\n",
       "</div><br/>\n",
       "<div id=\"description\">\n",
       "                Only the freshest fruits shipped from the state of Singapore, \n",
       "                also historically known by its indigenous name Pulau Ujong.\n",
       "                Description continues ...\n",
       "            </div><br/>\n",
       "<div>Last updated: <span id=\"last-updated-datetime\">20 Nov 21, 1500 hours</span></div>\n",
       "</div>\n",
       "<div class=\"right\">\n",
       "<h2>Fruits for sale</h2>\n",
       "<table id=\"fruits\">\n",
       "<tr><th>Fruit</th><th>Price ($)</th></tr>\n",
       "<tr><td>Apple</td><td>4</td></tr>\n",
       "<tr><td>Orange</td><td>5</td></tr>\n",
       "<tr><td>Pear</td><td>6</td></tr>\n",
       "<tr><td>Durian</td><td>7</td></tr>\n",
       "</table>\n",
       "</div>\n",
       "</div>\n",
       "</div>\n",
       "<div id=\"other-stuff\">\n",
       "    Other stuff goes here\n",
       "</div>\n",
       "<style>\n",
       "\n",
       "    .content {\n",
       "        padding: 3% 10% 3% 10%;\n",
       "    }\n",
       "\n",
       "    #other-stuff {\n",
       "        padding: 3% 10% 3% 10%;\n",
       "    }\n",
       "\n",
       "    .left, .right {\n",
       "        width: 49%;\n",
       "        display: inline-block;\n",
       "    }\n",
       "\n",
       "    .left {\n",
       "        font-size: 24px;\n",
       "\n",
       "    }\n",
       "\n",
       "    .right {\n",
       "        float: right;\n",
       "    }\n",
       "\n",
       "    #origin {\n",
       "        font-weight:600;\n",
       "    }\n",
       "\n",
       "    #last-updated-datetime {\n",
       "        font-weight:600;\n",
       "    }\n",
       "\n",
       "    * {box-sizing: border-box;}\n",
       "    \n",
       "    body { \n",
       "      margin: 0;\n",
       "      font-family: Arial, Helvetica, sans-serif;\n",
       "    }\n",
       "    \n",
       "    .header {\n",
       "      overflow: hidden;\n",
       "      background-color: #f1f1f1;\n",
       "      padding: 20px 10px;\n",
       "    }\n",
       "    \n",
       "    .header a {\n",
       "      float: left;\n",
       "      color: black;\n",
       "      text-align: center;\n",
       "      padding: 12px;\n",
       "      text-decoration: none;\n",
       "      font-size: 18px; \n",
       "      line-height: 25px;\n",
       "      border-radius: 4px;\n",
       "    }\n",
       "    \n",
       "    .header a.logo {\n",
       "      font-size: 25px;\n",
       "      font-weight: bold;\n",
       "    }\n",
       "    \n",
       "    .header a:hover {\n",
       "      background-color: #ddd;\n",
       "      color: black;\n",
       "    }\n",
       "    \n",
       "    .header a.active {\n",
       "      background-color: dodgerblue;\n",
       "      color: white;\n",
       "    }\n",
       "    \n",
       "    .header-right {\n",
       "      float: right;\n",
       "    }\n",
       "    \n",
       "    @media screen and (max-width: 500px) {\n",
       "      .header a {\n",
       "        float: none;\n",
       "        display: block;\n",
       "        text-align: left;\n",
       "      }\n",
       "      \n",
       "      .header-right {\n",
       "        float: none;\n",
       "      }\n",
       "    }\n",
       "\n",
       "    #fruits {\n",
       "        font-family: Arial, Helvetica, sans-serif;\n",
       "        border-collapse: collapse;\n",
       "        width: 100%;\n",
       "    }\n",
       "\n",
       "    #fruits td, #fruits th {\n",
       "        border: 1px solid #ddd;\n",
       "        padding: 8px;\n",
       "    }\n",
       "\n",
       "    #fruits tr:nth-child(even){background-color: #f2f2f2;}\n",
       "\n",
       "    #fruits tr:hover {background-color: #ddd;}\n",
       "\n",
       "    #fruits th {\n",
       "        padding-top: 12px;\n",
       "        padding-bottom: 12px;\n",
       "        text-align: left;\n",
       "        background-color: #f5b800;\n",
       "        color: white;\n",
       "    }\n",
       "\n",
       "</style>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup = BeautifulSoup(r.content, \"html.parser\")\n",
    "soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "756e9af2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<span id=\"origin\"> Singapore </span>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "origin = soup.find('span', {'id':'origin'})\n",
    "origin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b85842a0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Singapore \n"
     ]
    }
   ],
   "source": [
    "print(origin.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "dcacd4dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<span id=\"last-updated-datetime\">20 Nov 21, 1500 hours</span>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_updated = soup.find(\"span\", {'id': \"last-updated-datetime\"})\n",
    "last_updated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "3ba0a5dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20 Nov 21, 1500 hours'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_updated.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "26a189ea",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<table id=\"fruits\">\n",
       "<tr><th>Fruit</th><th>Price ($)</th></tr>\n",
       "<tr><td>Apple</td><td>4</td></tr>\n",
       "<tr><td>Orange</td><td>5</td></tr>\n",
       "<tr><td>Pear</td><td>6</td></tr>\n",
       "<tr><td>Durian</td><td>7</td></tr>\n",
       "</table>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = soup.find(\"table\", {\"id\":\"fruits\"})\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "f91befe7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Apple': '4', 'Orange': '5', 'Pear': '6', 'Durian': '7'}\n"
     ]
    }
   ],
   "source": [
    "fruits = {}\n",
    "for tr in table.find_all(\"tr\"):\n",
    "  tds = tr.find_all(\"td\")\n",
    "  \n",
    "  #so we dont get table headers\n",
    "  if len(tds) == 0:\n",
    "    continue\n",
    "    \n",
    "  fruit = tds[0].text\n",
    "  price = tds[1].text\n",
    "  \n",
    "  fruits[fruit]=price\n",
    "  \n",
    "print(fruits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "9a8e2ec5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Singapore\n",
      "20 Nov 21, 1500 hours\n",
      "{'Apple': '4', 'Orange': '5', 'Pear': '6', 'Durian': '7'}\n"
     ]
    }
   ],
   "source": [
    "#all the code in one place\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "url = \"https://zlliu.s3.ap-southeast-1.amazonaws.com/index.html\"\n",
    "\n",
    "r = requests.get(url)\n",
    "\n",
    "soup = BeautifulSoup(r.content, \"html.parser\")\n",
    "\n",
    "origin = soup.find(\"span\", {\"id\":\"origin\"})\n",
    "origin = origin.text.strip()\n",
    "\n",
    "last_updated = soup.find(\"span\", {\"id\":\"last-updated-datetime\"})\n",
    "last_updated = last_updated.text.strip()\n",
    "\n",
    "table = soup.find(\"table\", {\"id\": \"fruits\"})\n",
    "\n",
    "fruits = {}\n",
    "for tr in table.find_all(\"tr\"):\n",
    "    tds = tr.find_all(\"td\")\n",
    "\n",
    "    # so we don't get the table headers\n",
    "    if len(tds) == 0:\n",
    "        continue\n",
    "\n",
    "    fruit = tds[0].text\n",
    "    price = tds[1].text\n",
    "\n",
    "    fruits[fruit] = price\n",
    "\n",
    "print(origin)\n",
    "print(last_updated)\n",
    "print(fruits)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9eb40b3c",
   "metadata": {},
   "source": [
    "# bs4 on mlb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "13eb4e9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<table class=\"wikitable\">\n",
       "<tbody><tr>\n",
       "<th rowspan=\"2\">Game\n",
       "</th>\n",
       "<th rowspan=\"2\">Release date\n",
       "</th>\n",
       "<th colspan=\"2\">Cover Athlete\n",
       "</th>\n",
       "<th rowspan=\"2\">Platforms\n",
       "</th></tr>\n",
       "<tr>\n",
       "<th>Star\n",
       "</th>\n",
       "<th>Team\n",
       "</th></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_%2798\" title=\"MLB '98\">MLB '98</a></i>\n",
       "</td>\n",
       "<td>July 1, 1997\n",
       "</td>\n",
       "<td><a href=\"/wiki/Bernie_Williams\" title=\"Bernie Williams\">Bernie Williams</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/New_York_Yankees\" title=\"New York Yankees\">New York Yankees</a>\n",
       "</td>\n",
       "<td rowspan=\"6\"><a href=\"/wiki/PlayStation_(console)\" title=\"PlayStation (console)\">PlayStation</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_%2799\" title=\"MLB '99\">MLB '99</a></i>\n",
       "</td>\n",
       "<td>March 31, 1998\n",
       "</td>\n",
       "<td><a href=\"/wiki/Cal_Ripken_Jr.\" title=\"Cal Ripken Jr.\">Cal Ripken Jr.</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/Baltimore_Orioles\" title=\"Baltimore Orioles\">Baltimore Orioles</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_2000\" title=\"MLB 2000\">MLB 2000</a></i>\n",
       "</td>\n",
       "<td>February 28, 1999\n",
       "</td>\n",
       "<td><a href=\"/wiki/Mo_Vaughn\" title=\"Mo Vaughn\">Mo Vaughn</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/Los_Angeles_Angels\" title=\"Los Angeles Angels\">Anaheim Angels</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_2001\" title=\"MLB 2001\">MLB 2001</a></i>\n",
       "</td>\n",
       "<td>February 29, 2000\n",
       "</td>\n",
       "<td><a href=\"/wiki/Chipper_Jones\" title=\"Chipper Jones\">Chipper Jones</a>\n",
       "</td>\n",
       "<td rowspan=\"2\"><a href=\"/wiki/Atlanta_Braves\" title=\"Atlanta Braves\">Atlanta Braves</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_2002\" title=\"MLB 2002\">MLB 2002</a></i>\n",
       "</td>\n",
       "<td>May 7, 2001\n",
       "</td>\n",
       "<td><a href=\"/wiki/Andruw_Jones\" title=\"Andruw Jones\">Andruw Jones</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_2003\" title=\"MLB 2003\">MLB 2003</a></i>\n",
       "</td>\n",
       "<td>June 17, 2002\n",
       "</td>\n",
       "<td><a href=\"/wiki/Barry_Bonds\" title=\"Barry Bonds\">Barry Bonds</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/San_Francisco_Giants\" title=\"San Francisco Giants\">San Francisco Giants</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i><a href=\"/wiki/MLB_2004\" title=\"MLB 2004\">MLB 2004</a></i>\n",
       "</td>\n",
       "<td>April 30, 2003\n",
       "</td>\n",
       "<td><a href=\"/wiki/Shawn_Green\" title=\"Shawn Green\">Shawn Green</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/Los_Angeles_Dodgers\" title=\"Los Angeles Dodgers\">Los Angeles Dodgers</a>\n",
       "</td>\n",
       "<td rowspan=\"2\"><a href=\"/wiki/PlayStation_(console)\" title=\"PlayStation (console)\">PlayStation</a>, <a href=\"/wiki/PlayStation_2\" title=\"PlayStation 2\">PlayStation 2</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i>MLB 2005</i>\n",
       "</td>\n",
       "<td>March 4, 2004\n",
       "</td>\n",
       "<td><a href=\"/wiki/Eric_Chavez\" title=\"Eric Chavez\">Eric Chavez</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/Oakland_Athletics\" title=\"Oakland Athletics\">Oakland Athletics</a>\n",
       "</td></tr>\n",
       "<tr>\n",
       "<td><i>MLB 2006</i>\n",
       "</td>\n",
       "<td>March 8, 2005\n",
       "</td>\n",
       "<td><a href=\"/wiki/Vladimir_Guerrero\" title=\"Vladimir Guerrero\">Vladimir Guerrero</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/Los_Angeles_Angels\" title=\"Los Angeles Angels\">Los Angeles Angels</a>\n",
       "</td>\n",
       "<td><a href=\"/wiki/PlayStation_2\" title=\"PlayStation 2\">PlayStation 2</a>, <a href=\"/wiki/PlayStation_Portable\" title=\"PlayStation Portable\">PlayStation Portable</a>\n",
       "</td></tr></tbody></table>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#all the code in one place\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "url = \"https://en.wikipedia.org/wiki/MLB:_The_Show\"\n",
    "\n",
    "r = requests.get(url)\n",
    "\n",
    "soup = BeautifulSoup(r.content, \"html.parser\")\n",
    "\n",
    "soup\n",
    "predecessors = soup.find(\"table\", {\"class\":\"wikitable\"})\n",
    "#predecessors = predecessors.text.strip()\n",
    "predecessors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "345b69e1",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/wiki/Los_Angeles_Angels\" title=\"Los Angeles Angels\">Anaheim Angels</a>\n"
     ]
    }
   ],
   "source": [
    "predecessors.find_all('a')\n",
    "\n",
    "team = soup.find(\"a\", {'href': \"/wiki/Los_Angeles_Angels\"})\n",
    "print(team)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "69e4eb1b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"MLB '98\\n\": 'Bernie Williams\\n',\n",
       " \"MLB '99\\n\": 'Cal Ripken Jr.\\n',\n",
       " 'MLB 2000\\n': 'Mo Vaughn\\n',\n",
       " 'MLB 2001\\n': 'Chipper Jones\\n',\n",
       " 'MLB 2002\\n': 'Andruw Jones\\n',\n",
       " 'MLB 2003\\n': 'Barry Bonds\\n',\n",
       " 'MLB 2004\\n': 'Shawn Green\\n',\n",
       " 'MLB 2005\\n': 'Eric Chavez\\n',\n",
       " 'MLB 2006\\n': 'Vladimir Guerrero\\n'}"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fruits = {}\n",
    "for tr in predecessors.find_all(\"tr\"):\n",
    "    tds = tr.find_all(\"td\")\n",
    "\n",
    "    # so we don't get the table headers\n",
    "    if len(tds) == 0:\n",
    "        continue\n",
    "\n",
    "    fruit = tds[0].text\n",
    "    price = tds[2].text\n",
    "\n",
    "    fruits[fruit] = price\n",
    "\n",
    "fruits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "dc7e8203",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Game Name</th>\n",
       "      <th>Cover Athlete</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MLB '98</td>\n",
       "      <td>Bernie Williams</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MLB '99</td>\n",
       "      <td>Cal Ripken Jr.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MLB 2000</td>\n",
       "      <td>Mo Vaughn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MLB 2001</td>\n",
       "      <td>Chipper Jones</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MLB 2002</td>\n",
       "      <td>Andruw Jones</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MLB 2003</td>\n",
       "      <td>Barry Bonds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MLB 2004</td>\n",
       "      <td>Shawn Green</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MLB 2005</td>\n",
       "      <td>Eric Chavez</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MLB 2006</td>\n",
       "      <td>Vladimir Guerrero</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Game Name      Cover Athlete\n",
       "0   MLB '98    Bernie Williams\n",
       "1   MLB '99     Cal Ripken Jr.\n",
       "2  MLB 2000          Mo Vaughn\n",
       "3  MLB 2001      Chipper Jones\n",
       "4  MLB 2002       Andruw Jones\n",
       "5  MLB 2003        Barry Bonds\n",
       "6  MLB 2004        Shawn Green\n",
       "7  MLB 2005        Eric Chavez\n",
       "8  MLB 2006  Vladimir Guerrero"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = pd.DataFrame.from_dict(fruits, orient='index')\n",
    "f.reset_index(inplace=True)\n",
    "f.columns = ['Game Name', 'Cover Athlete']\n",
    "#f.iloc[0].apply(lambda x:x.strip('\\n'))\n",
    "f['Game Name'] = f['Game Name'].str.replace('\\n','')\n",
    "f['Cover Athlete'] = f['Cover Athlete'].str.replace('\\n','')\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "f41975bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[                MLB: The Show  \\\n",
       " 0  MLB: The Show current logo   \n",
       " 1                    Genre(s)   \n",
       " 2                Developer(s)   \n",
       " 3                Publisher(s)   \n",
       " 4                 Platform(s)   \n",
       " 5               First release   \n",
       " 6              Latest release   \n",
       " \n",
       "                                      MLB: The Show.1  \n",
       " 0                         MLB: The Show current logo  \n",
       " 1                                             Sports  \n",
       " 2                                   San Diego Studio  \n",
       " 3   Sony Interactive EntertainmentMLB Advanced Media  \n",
       " 4  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 5                  MLB 06: The ShowFebruary 28, 2006  \n",
       " 6                       MLB The Show 22April 5, 2022  ,\n",
       "        Game       Release date      Cover Athlete                        \\\n",
       "        Game       Release date               Star                  Team   \n",
       " 0   MLB '98       July 1, 1997    Bernie Williams      New York Yankees   \n",
       " 1   MLB '99     March 31, 1998     Cal Ripken Jr.     Baltimore Orioles   \n",
       " 2  MLB 2000  February 28, 1999          Mo Vaughn        Anaheim Angels   \n",
       " 3  MLB 2001  February 29, 2000      Chipper Jones        Atlanta Braves   \n",
       " 4  MLB 2002        May 7, 2001       Andruw Jones        Atlanta Braves   \n",
       " 5  MLB 2003      June 17, 2002        Barry Bonds  San Francisco Giants   \n",
       " 6  MLB 2004     April 30, 2003        Shawn Green   Los Angeles Dodgers   \n",
       " 7  MLB 2005      March 4, 2004        Eric Chavez     Oakland Athletics   \n",
       " 8  MLB 2006      March 8, 2005  Vladimir Guerrero    Los Angeles Angels   \n",
       " \n",
       "                              Platforms  \n",
       "                              Platforms  \n",
       " 0                          PlayStation  \n",
       " 1                          PlayStation  \n",
       " 2                          PlayStation  \n",
       " 3                          PlayStation  \n",
       " 4                          PlayStation  \n",
       " 5                          PlayStation  \n",
       " 6           PlayStation, PlayStation 2  \n",
       " 7           PlayStation, PlayStation 2  \n",
       " 8  PlayStation 2, PlayStation Portable  ,\n",
       "                 Game                 Release date       Cover Athlete  \\\n",
       "                 Game                 Release date                Star   \n",
       " 0   MLB 06: The Show            February 28, 2006         David Ortiz   \n",
       " 1   MLB 07: The Show  February 27, 2007 (PS2/PSP)        David Wright   \n",
       " 2   MLB 07: The Show           May 15, 2007 (PS3)        David Wright   \n",
       " 3   MLB 08: The Show                March 4, 2008         Ryan Howard   \n",
       " 4   MLB 09: The Show                March 3, 2009      Dustin Pedroia   \n",
       " 5   MLB 10: The Show                March 2, 2010           Joe Mauer   \n",
       " 6   MLB 11: The Show                March 8, 2011           Joe Mauer   \n",
       " 7   MLB 12: The Show                March 6, 2012     Adrián González   \n",
       " 8   MLB 13: The Show                March 5, 2013    Andrew McCutchen   \n",
       " 9   MLB 14: The Show     April 1, 2014 (PS3/Vita)      Miguel Cabrera   \n",
       " 10  MLB 14: The Show            May 6, 2014 (PS4)      Miguel Cabrera   \n",
       " 11  MLB 15: The Show               March 31, 2015         Yasiel Puig   \n",
       " 12   MLB The Show 16               March 29, 2016      Josh Donaldson   \n",
       " 13   MLB The Show 17               March 28, 2017     Ken Griffey Jr.   \n",
       " 14   MLB The Show 18               March 27, 2018         Aaron Judge   \n",
       " 15   MLB The Show 19               March 26, 2019        Bryce Harper   \n",
       " 16   MLB The Show 20               March 17, 2020         Javier Báez   \n",
       " 17   MLB The Show 21               April 20, 2021  Fernando Tatís Jr.   \n",
       " 18   MLB The Show 22                April 5, 2022       Shohei Ohtani   \n",
       " \n",
       "                                                                    Platforms  \n",
       "                      Team                                          Platforms  \n",
       " 0          Boston Red Sox                PlayStation 2, PlayStation Portable  \n",
       " 1           New York Mets  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 2           New York Mets  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 3   Philadelphia Phillies  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 4          Boston Red Sox  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 5         Minnesota Twins  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 6         Minnesota Twins  PlayStation 2, PlayStation 3, PlayStation Port...  \n",
       " 7          Boston Red Sox                    PlayStation 3, PlayStation Vita  \n",
       " 8      Pittsburgh Pirates                    PlayStation 3, PlayStation Vita  \n",
       " 9          Detroit Tigers     PlayStation 3, PlayStation 4, PlayStation Vita  \n",
       " 10         Detroit Tigers     PlayStation 3, PlayStation 4, PlayStation Vita  \n",
       " 11    Los Angeles Dodgers     PlayStation 3, PlayStation 4, PlayStation Vita  \n",
       " 12      Toronto Blue Jays                       PlayStation 3, PlayStation 4  \n",
       " 13       Seattle Mariners                                      PlayStation 4  \n",
       " 14       New York Yankees                                      PlayStation 4  \n",
       " 15  Philadelphia Phillies                                      PlayStation 4  \n",
       " 16           Chicago Cubs                                      PlayStation 4  \n",
       " 17       San Diego Padres  PlayStation 4, PlayStation 5, Xbox One, Xbox S...  \n",
       " 18     Los Angeles Angels  PlayStation 4, PlayStation 5, Xbox One, Xbox S...  ,\n",
       "               Game             Star                Team\n",
       " 0  MLB The Show 21  Jackie Robinson    Brooklyn Dodgers\n",
       " 1  MLB The Show 22    Shohei Ohtani  Los Angeles Angels,\n",
       "                Game            Star               Team\n",
       " 0  MLB 12: The Show   José Bautista  Toronto Blue Jays\n",
       " 1  MLB 13: The Show   José Bautista  Toronto Blue Jays\n",
       " 2  MLB 14: The Show    Brett Lawrie  Toronto Blue Jays\n",
       " 3  MLB 15: The Show  Russell Martin  Toronto Blue Jays\n",
       " 4   MLB The Show 16  Josh Donaldson  Toronto Blue Jays\n",
       " 5   MLB The Show 17   Aaron Sanchez  Toronto Blue Jays\n",
       " 6   MLB The Show 18  Marcus Stroman  Toronto Blue Jays,\n",
       "                Game           Star                Team\n",
       " 0  MLB 06: The Show   Chan Ho Park    San Diego Padres\n",
       " 1  MLB 14: The Show  Shin-Soo Choo       Texas Rangers\n",
       " 2  MLB 15: The Show  Shin-Soo Choo       Texas Rangers\n",
       " 3   MLB The Show 16   Jung-ho Kang  Pittsburgh Pirates\n",
       " 4   MLB The Show 17   Hyun-soo Kim   Baltimore Orioles,\n",
       "                Game          Star               Team\n",
       " 0  MLB 13: The Show  Wei-Yin Chen  Baltimore Orioles\n",
       " 1  MLB 14: The Show  Wei-Yin Chen  Baltimore Orioles\n",
       " 2  MLB 15: The Show  Wei-Yin Chen  Baltimore Orioles\n",
       " 3   MLB The Show 16  Wei-Yin Chen      Miami Marlins\n",
       " 4   MLB The Show 17  Wei-Yin Chen      Miami Marlins,\n",
       "         Commentator   06   07   08   09   10   11   12   13   14   15   16  \\\n",
       " 0   Matt Vasgersian  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 1     Dave Campbell  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 2        Rex Hudler  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 3       Eric Karros  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 4       Steve Lyons  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 5   Harold Reynolds  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 6        Dan Plesac  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 7       Mark DeRosa  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 8      Heidi Watney  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 9       Jon Sciambi  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 10  Chris Singleton  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN  NaN   \n",
       " 11            Total  3.0  3.0  3.0  3.0  3.0  4.0  3.0  3.0  3.0  3.0  3.0   \n",
       " \n",
       "      17   18   19   20   21   22  Total appearances  \n",
       " 0   NaN  NaN  NaN  NaN  NaN  NaN               16.0  \n",
       " 1   NaN  NaN  NaN  NaN  NaN  NaN                7.0  \n",
       " 2   NaN  NaN  NaN  NaN  NaN  NaN                6.0  \n",
       " 3   NaN  NaN  NaN  NaN  NaN  NaN                6.0  \n",
       " 4   NaN  NaN  NaN  NaN  NaN  NaN                4.0  \n",
       " 5   NaN  NaN  NaN  NaN  NaN  NaN                1.0  \n",
       " 6   NaN  NaN  NaN  NaN  NaN  NaN                5.0  \n",
       " 7   NaN  NaN  NaN  NaN  NaN  NaN                4.0  \n",
       " 8   NaN  NaN  NaN  NaN  NaN  NaN                3.0  \n",
       " 9   NaN  NaN  NaN  NaN  NaN  NaN                1.0  \n",
       " 10  NaN  NaN  NaN  NaN  NaN  NaN                1.0  \n",
       " 11  3.0  3.0  4.0  4.0  4.0  2.0                NaN  ,\n",
       "     0                                                  1\n",
       " 0 NaN  This section does not cite any sources. Please...,\n",
       "     Year              Game                                            Sales\n",
       " 0   1997           MLB '98                                              NaN\n",
       " 1   1998           MLB '99                                     2.52 million\n",
       " 2   1999          MLB 2000                                              NaN\n",
       " 3   2000          MLB 2001                                              NaN\n",
       " 4   2001          MLB 2002                                              NaN\n",
       " 5   2002          MLB 2003                                              NaN\n",
       " 6   2003          MLB 2004                                           660000\n",
       " 7   2004          MLB 2005                                           900000\n",
       " 8   2005          MLB 2006                                           400000\n",
       " 9   2006  MLB 06: The Show                     940,000 (PS2), 350,000 (PSP)\n",
       " 10  2007  MLB 07: The Show      930,000 (PS2), 280,000 (PS3), 280,000 (PSP)\n",
       " 11  2008  MLB 08: The Show      420,000 (PS2), 700,000 (PS3), 330,000 (PSP)\n",
       " 12  2009  MLB 09: The Show      330,000 (PS2), 720,000 (PS3), 270,000 (PSP)\n",
       " 13  2010  MLB 10: The Show      410,000 (PS2), 730,000 (PS3), 210,000 (PSP)\n",
       " 14  2011  MLB 11: The Show      130,000 (PS2), 590,000 (PS3), 180,000 (PSP)\n",
       " 15  2012  MLB 12: The Show                     930,000 (PS3), 200,000 (PSV)\n",
       " 16  2013  MLB 13: The Show                     840,000 (PS3), 150,000 (PSV)\n",
       " 17  2014  MLB 14: The Show      430,000 (PS3), 730,000 (PS4), 120,000 (PSV)\n",
       " 18  2015  MLB 15: The Show  400,000 (PS3), 1.01 million (PS4), 40,000 (PSV)\n",
       " 19  2016   MLB The Show 16                     380,000 (PS3), 960,000 (PS4)\n",
       " 20  2017   MLB The Show 17                                     1.16 million\n",
       " 21  2018   MLB The Show 18                                     1.06 million\n",
       " 22  2019   MLB The Show 19                                              NaN\n",
       " 23  2020   MLB The Show 20                                              NaN\n",
       " 24  2021   MLB The Show 21                                     2 million[7],\n",
       "   .mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:\"[ \"}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:\" ]\"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}vteSan Diego Studio  \\\n",
       " 0                                      MLB: The Show                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       " 1                                                NBA                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       " 2                                              Other                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       " \n",
       "   .mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:\"[ \"}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:\" ]\"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}vteSan Diego Studio.1  \n",
       " 0  06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 2...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       " 1        NBA NBA 06 NBA 07 NBA 08 NBA 09: The Inside                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       " 2  The Mark of Kri High Velocity Bowling Pain Mod...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             ]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst = pd.read_html('https://en.wikipedia.org/wiki/MLB:_The_Show')\n",
    "lst"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2331d88c",
   "metadata": {},
   "source": [
    "# selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2fb84d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup\n",
    "url = 'https://pubs.rsc.org/en/content/articlelanding/2020/na/d0na00118j'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "5f8d5a50",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-39-fc2c8cc820c6>:7: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome(path)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Miguel Cabrera Stats | Baseball-Reference.com\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-39-fc2c8cc820c6>:13: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  search = driver.find_element_by_id('standard_fielding')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Year Age Tm Pos Lg G GS CG Inn Ch PO A E DP Fld% Rtot Rdrs Rtot/yr Rdrs/yr RF/9 RF/G lgFld% lgRF9 lgRFG Awards\n",
      "2003 20 FLA LF NL 55 55 51 481.0 107 99 5 3 1 .972 -5 1 -12 2 1.95 1.89 .979 1.91 1.89 MVP-27,RoY-5\n",
      "2003 20 FLA OF NL 55 55 51 481.0 107 99 5 3 1 .972 -5 1 -12 2 1.95 1.89 .984 2.17 2.15 MVP-27,RoY-5\n",
      "2003 20 FLA 3B NL 34 30 30 275.0 71 17 53 1 2 .986 2 -3 7 -12 2.29 2.06 .955 2.74 2.72 MVP-27,RoY-5\n",
      "2004 21 FLA OF NL 158 158 135 1360.0 284 262 13 9 1 .968 -10 -3 -8 -2 1.82 1.74 .982 2.16 2.14 AS,MVP-22\n",
      "2004 21 FLA RF NL 100 100 82 856.0 184 170 7 7 1 .962 -9 -9 -13 -12 1.86 1.77 .980 2.09 2.07 AS,MVP-22\n",
      "2004 21 FLA LF NL 59 58 53 504.0 100 92 6 2 0 .980 0 6 0 14 1.75 1.66 .980 1.85 1.84 AS,MVP-22\n",
      "2004 21 FLA DH NL 1 1 AS,MVP-22\n",
      "2005 22 FLA LF NL 134 128 102 1105.2 205 188 12 5 3 .976 -9 -4 -10 -4 1.63 1.49 .984 1.94 1.92 AS,MVP-5,SS\n",
      "2005 22 FLA OF NL 134 128 102 1105.2 205 188 12 5 3 .976 -9 -4 -10 -4 1.63 1.49 .984 2.19 2.16 AS,MVP-5,SS\n",
      "2005 22 FLA 3B NL 29 28 20 238.0 70 22 46 2 5 .971 -2 0 -8 0 2.57 2.34 .958 2.77 2.74 AS,MVP-5,SS\n",
      "2006 23 FLA 3B NL 157 157 134 1334.0 397 114 266 17 33 .957 -6 -10 -5 -9 2.56 2.42 .954 2.70 2.68 AS,MVP-5,SS\n",
      "2006 23 FLA DH NL 1 1 AS,MVP-5,SS\n",
      "2007 24 FLA 3B NL 153 152 134 1310.2 389 100 266 23 33 .941 -10 -19 -9 -17 2.51 2.39 .954 2.68 2.67 AS,MVP-15\n",
      "2007 24 FLA DH NL 3 3 AS,MVP-15\n",
      "2008 25 DET 1B AL 143 139 122 1204.0 1199 1117 73 9 116 .992 2 -7 2 -7 8.90 8.32 .994 9.26 9.18 MVP-13\n",
      "2008 25 DET 3B AL 14 14 10 116.0 50 15 30 5 4 .900 -2 -3 -25 -30 3.49 3.21 .958 2.74 2.71 MVP-13\n",
      "2008 25 DET DH AL 6 6 MVP-13\n",
      "2009 26 DET 1B AL 153 153 135 1315.0 1327 1215 105 7 128 .995 -1 -1 -1 -1 9.03 8.63 .994 9.13 9.02 MVP-4\n",
      "2009 26 DET DH AL 6 6 MVP-4\n",
      "2010 27 DET 1B AL 148 148 132 1285.1 1327 1218 96 13 133 .990 -3 -5 -2 -4 9.20 8.88 .994 9.32 9.23 AS,MVP-2,SS\n",
      "2010 27 DET DH AL 2 2 AS,MVP-2,SS\n",
      "2011 28 DET 1B AL 152 152 138 1322.0 1379 1263 103 13 117 .991 -8 -2 -7 -2 9.30 8.99 .993 9.30 9.24 AS,MVP-5\n",
      "2011 28 DET DH AL 7 7 AS,MVP-5\n",
      "2012 29 DET 3B AL 154 154 142 1322.0 383 127 243 13 31 .966 -9 -4 -8 -4 2.52 2.40 .952 2.60 2.59 AS,MVP-1,SS\n",
      "2012 29 DET DH AL 7 7 AS,MVP-1,SS\n",
      "2012 29 DET 1B AL 2 0 0 3.0 4 4 0 0 1 1.000 0 0 0 0 12.00 2.00 .995 9.30 9.20 AS,MVP-1,SS\n",
      "2013 30 DET 3B AL 145 145 113 1234.2 283 87 184 12 24 .958 -17 -17 -16 -17 1.98 1.87 .959 2.51 2.50 AS,MVP-1,SS\n",
      "2013 30 DET DH AL 1 1 AS,MVP-1,SS\n",
      "2014 31 DET 1B AL 126 125 100 1083.1 1081 978 98 5 100 .995 7 -1 8 -1 8.94 8.54 .992 8.97 8.94 AS,MVP-9\n",
      "2014 31 DET DH AL 25 25 AS,MVP-9\n",
      "2014 31 DET 3B AL 10 8 7 69.1 20 6 13 1 2 .950 -2 -1 -42 -17 2.47 1.90 .956 2.56 2.55 AS,MVP-9\n",
      "2015 32 DET 1B AL 107 107 92 932.1 983 903 76 4 96 .996 1 1 1 1 9.45 9.15 .994 9.10 9.03 AS,MVP-11,SS\n",
      "2015 32 DET DH AL 11 11 AS,MVP-11,SS\n",
      "2016 33 DET 1B AL 147 147 127 1262.0 1288 1186 95 7 124 .995 -1 -3 -1 -3 9.14 8.71 .994 8.97 8.87 AS,MVP-9,SS\n",
      "2016 33 DET DH AL 8 8 AS,MVP-9,SS\n",
      "2016 33 DET 3B AL 1 1 1 8.0 2 2 0 0 0 1.000 0 0 -45 0 2.25 2.00 .960 2.70 2.60 AS,MVP-9,SS\n",
      "2017 34 DET 1B AL 115 115 86 938.1 825 761 63 1 90 .999 2 -11 3 -14 7.90 7.17 .993 8.64 8.55\n",
      "2017 34 DET DH AL 14 14\n",
      "2018 35 DET 1B AL 32 32 21 261.0 262 248 13 1 22 .996 -2 -5 -8 -23 9.00 8.16 .993 8.51 8.43\n",
      "2018 35 DET DH AL 5 5\n",
      "2019 36 DET DH AL 107 107\n",
      "2019 36 DET 1B AL 26 26 15 219.0 210 204 5 1 13 .995 -3 -1 -14 -5 8.59 8.04 .993 8.33 8.27\n",
      "2020 37 DET DH AL 56 56\n",
      "2021 38 DET DH AL 83 83\n",
      "2021 38 DET 1B AL 44 44 31 367.1 339 312 26 1 34 .997 -1 -4 -3 -13 8.28 7.68 .993 8.20 8.01\n",
      "2022 39 DET DH AL 15 15\n",
      "12 Seasons 1B 1195 1188 999 10192.2 10224 9409 753 62 974 .994 -5 -38 -1 -4 8.97 8.50 .993 9.04 8.95\n",
      "9 Seasons 3B 697 689 591 5907.2 1665 490 1101 74 134 .956 -46 -88 -9 -17 2.42 2.28 .955 2.64 2.62\n",
      "18 Seasons DH 358 358\n",
      "3 Seasons OF 347 341 288 2946.2 596 549 30 17 5 .971 -23 -13 -9 -5 1.77 1.67 .983 2.17 2.15\n",
      "3 Seasons LF 248 241 206 2090.2 412 379 23 10 4 .976 -14 5 -8 2 1.73 1.62 .982 1.91 1.89\n",
      "1 Season RF 100 100 82 856.0 184 170 7 7 1 .962 -9 -18 -13 -25 1.86 1.77 .980 2.09 2.07\n",
      "20 Seasons TOT 2239 2218 1878 19047.0 12485 10448 1884 153 1113 .988 -75 -139 -5 -8 5.83 5.51 .988 5.99 5.93\n",
      "Pos Lg G GS CG Inn Ch PO A E DP Fld% Rtot Rdrs Rtot/yr Rdrs/yr RF/9 RF/G lgFld% lgRF9 lgRFG Awards\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "import time\n",
    "\n",
    "path = 'C:\\Program Files (x86)\\chromedriver.exe'\n",
    "driver = webdriver.Chrome(path)\n",
    "\n",
    "driver.get('https://www.baseball-reference.com/players/c/cabremi01.shtml')\n",
    "print(driver.title)\n",
    "\n",
    "\n",
    "search = driver.find_element_by_id('standard_fielding')\n",
    "print(search.text)\n",
    "\n",
    "\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "ed73b569",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-73-87f7872c7789>:7: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome(path)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Miguel Cabrera Stats | Baseball-Reference.com\n",
      "Year Age Tm Pos Lg G GS CG Inn Ch PO A E DP Fld% Rtot Rdrs Rtot/yr Rdrs/yr RF/9 RF/G lgFld% lgRF9 lgRFG Awards\n",
      "2003 20 FLA LF NL 55 55 51 481.0 107 99 5 3 1 .972 -5 1 -12 2 1.95 1.89 .979 1.91 1.89 MVP-27,RoY-5\n",
      "2003 20 FLA OF NL 55 55 51 481.0 107 99 5 3 1 .972 -5 1 -12 2 1.95 1.89 .984 2.17 2.15 MVP-27,RoY-5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-73-87f7872c7789>:14: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  search = driver.find_element_by_id('standard_fielding')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2003 20 FLA 3B NL 34 30 30 275.0 71 17 53 1 2 .986 2 -3 7 -12 2.29 2.06 .955 2.74 2.72 MVP-27,RoY-5\n",
      "2004 21 FLA OF NL 158 158 135 1360.0 284 262 13 9 1 .968 -10 -3 -8 -2 1.82 1.74 .982 2.16 2.14 AS,MVP-22\n",
      "2004 21 FLA RF NL 100 100 82 856.0 184 170 7 7 1 .962 -9 -9 -13 -12 1.86 1.77 .980 2.09 2.07 AS,MVP-22\n",
      "2004 21 FLA LF NL 59 58 53 504.0 100 92 6 2 0 .980 0 6 0 14 1.75 1.66 .980 1.85 1.84 AS,MVP-22\n",
      "2004 21 FLA DH NL 1 1 AS,MVP-22\n",
      "2005 22 FLA LF NL 134 128 102 1105.2 205 188 12 5 3 .976 -9 -4 -10 -4 1.63 1.49 .984 1.94 1.92 AS,MVP-5,SS\n",
      "2005 22 FLA OF NL 134 128 102 1105.2 205 188 12 5 3 .976 -9 -4 -10 -4 1.63 1.49 .984 2.19 2.16 AS,MVP-5,SS\n",
      "2005 22 FLA 3B NL 29 28 20 238.0 70 22 46 2 5 .971 -2 0 -8 0 2.57 2.34 .958 2.77 2.74 AS,MVP-5,SS\n",
      "2006 23 FLA 3B NL 157 157 134 1334.0 397 114 266 17 33 .957 -6 -10 -5 -9 2.56 2.42 .954 2.70 2.68 AS,MVP-5,SS\n",
      "2006 23 FLA DH NL 1 1 AS,MVP-5,SS\n",
      "2007 24 FLA 3B NL 153 152 134 1310.2 389 100 266 23 33 .941 -10 -19 -9 -17 2.51 2.39 .954 2.68 2.67 AS,MVP-15\n",
      "2007 24 FLA DH NL 3 3 AS,MVP-15\n",
      "2008 25 DET 1B AL 143 139 122 1204.0 1199 1117 73 9 116 .992 2 -7 2 -7 8.90 8.32 .994 9.26 9.18 MVP-13\n",
      "2008 25 DET 3B AL 14 14 10 116.0 50 15 30 5 4 .900 -2 -3 -25 -30 3.49 3.21 .958 2.74 2.71 MVP-13\n",
      "2008 25 DET DH AL 6 6 MVP-13\n",
      "2009 26 DET 1B AL 153 153 135 1315.0 1327 1215 105 7 128 .995 -1 -1 -1 -1 9.03 8.63 .994 9.13 9.02 MVP-4\n",
      "2009 26 DET DH AL 6 6 MVP-4\n",
      "2010 27 DET 1B AL 148 148 132 1285.1 1327 1218 96 13 133 .990 -3 -5 -2 -4 9.20 8.88 .994 9.32 9.23 AS,MVP-2,SS\n",
      "2010 27 DET DH AL 2 2 AS,MVP-2,SS\n",
      "2011 28 DET 1B AL 152 152 138 1322.0 1379 1263 103 13 117 .991 -8 -2 -7 -2 9.30 8.99 .993 9.30 9.24 AS,MVP-5\n",
      "2011 28 DET DH AL 7 7 AS,MVP-5\n",
      "2012 29 DET 3B AL 154 154 142 1322.0 383 127 243 13 31 .966 -9 -4 -8 -4 2.52 2.40 .952 2.60 2.59 AS,MVP-1,SS\n",
      "2012 29 DET DH AL 7 7 AS,MVP-1,SS\n",
      "2012 29 DET 1B AL 2 0 0 3.0 4 4 0 0 1 1.000 0 0 0 0 12.00 2.00 .995 9.30 9.20 AS,MVP-1,SS\n",
      "2013 30 DET 3B AL 145 145 113 1234.2 283 87 184 12 24 .958 -17 -17 -16 -17 1.98 1.87 .959 2.51 2.50 AS,MVP-1,SS\n",
      "2013 30 DET DH AL 1 1 AS,MVP-1,SS\n",
      "2014 31 DET 1B AL 126 125 100 1083.1 1081 978 98 5 100 .995 7 -1 8 -1 8.94 8.54 .992 8.97 8.94 AS,MVP-9\n",
      "2014 31 DET DH AL 25 25 AS,MVP-9\n",
      "2014 31 DET 3B AL 10 8 7 69.1 20 6 13 1 2 .950 -2 -1 -42 -17 2.47 1.90 .956 2.56 2.55 AS,MVP-9\n",
      "2015 32 DET 1B AL 107 107 92 932.1 983 903 76 4 96 .996 1 1 1 1 9.45 9.15 .994 9.10 9.03 AS,MVP-11,SS\n",
      "2015 32 DET DH AL 11 11 AS,MVP-11,SS\n",
      "2016 33 DET 1B AL 147 147 127 1262.0 1288 1186 95 7 124 .995 -1 -3 -1 -3 9.14 8.71 .994 8.97 8.87 AS,MVP-9,SS\n",
      "2016 33 DET DH AL 8 8 AS,MVP-9,SS\n",
      "2016 33 DET 3B AL 1 1 1 8.0 2 2 0 0 0 1.000 0 0 -45 0 2.25 2.00 .960 2.70 2.60 AS,MVP-9,SS\n",
      "2017 34 DET 1B AL 115 115 86 938.1 825 761 63 1 90 .999 2 -11 3 -14 7.90 7.17 .993 8.64 8.55\n",
      "2017 34 DET DH AL 14 14\n",
      "2018 35 DET 1B AL 32 32 21 261.0 262 248 13 1 22 .996 -2 -5 -8 -23 9.00 8.16 .993 8.51 8.43\n",
      "2018 35 DET DH AL 5 5\n",
      "2019 36 DET DH AL 107 107\n",
      "2019 36 DET 1B AL 26 26 15 219.0 210 204 5 1 13 .995 -3 -1 -14 -5 8.59 8.04 .993 8.33 8.27\n",
      "2020 37 DET DH AL 56 56\n",
      "2021 38 DET DH AL 83 83\n",
      "2021 38 DET 1B AL 44 44 31 367.1 339 312 26 1 34 .997 -1 -4 -3 -13 8.28 7.68 .993 8.20 8.01\n",
      "2022 39 DET DH AL 15 15\n",
      "12 Seasons 1B 1195 1188 999 10192.2 10224 9409 753 62 974 .994 -5 -38 -1 -4 8.97 8.50 .993 9.04 8.95\n",
      "9 Seasons 3B 697 689 591 5907.2 1665 490 1101 74 134 .956 -46 -88 -9 -17 2.42 2.28 .955 2.64 2.62\n",
      "18 Seasons DH 358 358\n",
      "3 Seasons OF 347 341 288 2946.2 596 549 30 17 5 .971 -23 -13 -9 -5 1.77 1.67 .983 2.17 2.15\n",
      "3 Seasons LF 248 241 206 2090.2 412 379 23 10 4 .976 -14 5 -8 2 1.73 1.62 .982 1.91 1.89\n",
      "1 Season RF 100 100 82 856.0 184 170 7 7 1 .962 -9 -18 -13 -25 1.86 1.77 .980 2.09 2.07\n",
      "20 Seasons TOT 2239 2218 1878 19047.0 12485 10448 1884 153 1113 .988 -75 -139 -5 -8 5.83 5.51 .988 5.99 5.93\n",
      "Pos Lg G GS CG Inn Ch PO A E DP Fld% Rtot Rdrs Rtot/yr Rdrs/yr RF/9 RF/G lgFld% lgRF9 lgRFG Awards\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "import time\n",
    "\n",
    "path = 'C:\\Program Files (x86)\\chromedriver.exe'\n",
    "driver = webdriver.Chrome(path)\n",
    "\n",
    "\n",
    "driver.get('https://www.baseball-reference.com/players/c/cabremi01.shtml')\n",
    "print(driver.title)\n",
    "\n",
    "\n",
    "search = driver.find_element_by_id('standard_fielding')\n",
    "rows = search.find_elements_by_tag_name('tr')\n",
    "\n",
    "for row in rows:\n",
    "  print(row.text)\n",
    "   \n",
    "\n",
    "driver.quit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}