using_python_datatable.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c0910977-325a-4315-825c-09efb4464e57",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Try datatable package to handle large dataset\n",
    "\n",
    "#### References:\n",
    "- https://datatable.readthedocs.io/en/latest/\n",
    "- https://samukweku.github.io/data-wrangling-blog/python/pydatatable/rdatatable/2020/06/25/Selecting-and-Grouping-Data-with-Python-Datatable.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5f5b676-961f-4edb-8b1f-5415a315a911",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install datatable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "90578fe4-8e00-4a03-adcc-1715a43b49f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0.0\n"
     ]
    }
   ],
   "source": [
    "import datatable as dt\n",
    "from datatable import f, by, sum, min, max # f is frame proxy \n",
    "print(dt.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ba959823-766a-455f-9888-1beaa2d925e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = '/Users/sush/Documents/grad_school/courses/neuromatch/project/all_the_news.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9139b06-ec23-4cdb-9099-cbab328ebc72",
   "metadata": {},
   "source": [
    "### Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "07bae40b-7c9b-4abd-97af-2e8f8ccdc67e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15.6 s, sys: 11.1 s, total: 26.7 s\n",
      "Wall time: 5.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# load csv file into datatable\n",
    "all_news = dt.fread(filepath) \n",
    "\n",
    "### can also convert pandas DataFrame to datatable\n",
    "# dt = dt.Frame(pandas_dataframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5794a5fe-4a20-4416-870f-79c68c2e4fc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>date</th><th>year</th><th>month</th><th>day</th><th>author</th><th>title</th><th>article</th><th>url</th><th>section</th><th>publication</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='time' title='time64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='float' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>2016-12-09<span class=sp>T</span>18:31:00</td><td>2016</td><td>12</td><td>9</td><td>Lee Drutman</td><td>We should take concerns about the health of libera&#133;</td><td>This post is part of Polyarchy, an independent blo&#133;</td><td>https://www.vox.com/polyarchy/2016/12/9/13898340/d&#133;</td><td></td><td>Vox</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>2016-10-07<span class=sp>T</span>21:26:46</td><td>2016</td><td>10</td><td>7</td><td>Scott Davis</td><td>Colts GM Ryan Grigson says Andrew Luck's contract &#133;</td><td> The Indianapolis Colts made Andrew Luck the highe&#133;</td><td>https://www.businessinsider.com/colts-gm-ryan-grig&#133;</td><td></td><td>Business Insider</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>2018-01-26<span class=sp>T</span>00:00:00</td><td>2018</td><td>1</td><td>26</td><td></td><td>Trump denies report he ordered Mueller fired</td><td>DAVOS, Switzerland (Reuters) - U.S. President Dona&#133;</td><td>https://www.reuters.com/article/us-davos-meeting-t&#133;</td><td>Davos</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>2019-06-27<span class=sp>T</span>00:00:00</td><td>2019</td><td>6</td><td>27</td><td></td><td>France's Sarkozy reveals his 'Passions' but insist&#133;</td><td>PARIS (Reuters) - Former French president Nicolas &#133;</td><td>https://www.reuters.com/article/france-politics-sa&#133;</td><td>World News</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>2016-01-27<span class=sp>T</span>00:00:00</td><td>2016</td><td>1</td><td>27</td><td></td><td>Paris Hilton: Woman In Black For Uncle Monty's Fun&#133;</td><td>Paris Hilton arrived at LAX Wednesday dressed to p&#133;</td><td>https://www.tmz.com/2016/01/27/paris-hilton-monty-&#133;</td><td></td><td>TMZ</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>5 rows &times; 10 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<Frame#7fbc502c9f60 5x10>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8fb9260b-8e56-477e-8ff1-a4019b52acd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2688878, 10)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "446e9324-4d9f-4a4c-b6a2-c30640b76fd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>date</th><th>year</th><th>month</th><th>day</th><th>author</th><th>title</th><th>article</th><th>url</th><th>section</th><th>publication</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='time' title='time64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='float' title='float64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td>2016-12-09<span class=sp>T</span>18:31:00</td><td>2016</td><td>12</td><td>9</td><td>Lee Drutman</td><td>We should take concerns about the health of libera&#133;</td><td>This post is part of Polyarchy, an independent blo&#133;</td><td>https://www.vox.com/polyarchy/2016/12/9/13898340/d&#133;</td><td></td><td>Vox</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>2016-10-07<span class=sp>T</span>21:26:46</td><td>2016</td><td>10</td><td>7</td><td>Scott Davis</td><td>Colts GM Ryan Grigson says Andrew Luck's contract &#133;</td><td> The Indianapolis Colts made Andrew Luck the highe&#133;</td><td>https://www.businessinsider.com/colts-gm-ryan-grig&#133;</td><td></td><td>Business Insider</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>2018-01-26<span class=sp>T</span>00:00:00</td><td>2018</td><td>1</td><td>26</td><td></td><td>Trump denies report he ordered Mueller fired</td><td>DAVOS, Switzerland (Reuters) - U.S. President Dona&#133;</td><td>https://www.reuters.com/article/us-davos-meeting-t&#133;</td><td>Davos</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>2019-06-27<span class=sp>T</span>00:00:00</td><td>2019</td><td>6</td><td>27</td><td></td><td>France's Sarkozy reveals his 'Passions' but insist&#133;</td><td>PARIS (Reuters) - Former French president Nicolas &#133;</td><td>https://www.reuters.com/article/france-politics-sa&#133;</td><td>World News</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>2016-01-27<span class=sp>T</span>00:00:00</td><td>2016</td><td>1</td><td>27</td><td></td><td>Paris Hilton: Woman In Black For Uncle Monty's Fun&#133;</td><td>Paris Hilton arrived at LAX Wednesday dressed to p&#133;</td><td>https://www.tmz.com/2016/01/27/paris-hilton-monty-&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>5</td><td>2019-06-17<span class=sp>T</span>00:00:00</td><td>2019</td><td>6</td><td>17</td><td></td><td>ECB's Coeure: If we decide to cut rates, we'd have&#133;</td><td>BERLIN, June 17 (Reuters) - ECB board member Benoi&#133;</td><td>https://www.reuters.com/article/ecb-policy-coeure/&#133;</td><td>Financials</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>6</td><td>2019-06-23<span class=sp>T</span>00:00:00</td><td>2019</td><td>6</td><td>23</td><td></td><td>Venezuela detains six military, police officials: &#133;</td><td>CARACAS (Reuters) - Venezuelan authorities have ar&#133;</td><td>https://www.reuters.com/article/us-venezuela-polit&#133;</td><td>World News</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>7</td><td>2018-05-02<span class=sp>T</span>17:09:00</td><td>2018</td><td>5</td><td>2</td><td>Caroline Williams</td><td>You Can Trick Your Brain Into Being More Focused</td><td>If only every day could be like this. You can’t pu&#133;</td><td>https://www.vice.com/en_us/article/9kgp4v/how-to-i&#133;</td><td>Health</td><td>Vice</td></tr>\n",
       "    <tr><td class='row_index'>8</td><td>2016-05-18<span class=sp>T</span>13:00:06</td><td>2016</td><td>5</td><td>18</td><td>Mark Bergen</td><td>How to watch the Google I/O keynote live</td><td>Google I/O, the company's big developer conference&#133;</td><td>https://www.vox.com/2016/5/18/11697070/how-to-watc&#133;</td><td></td><td>Vox</td></tr>\n",
       "    <tr><td class='row_index'>9</td><td>2017-03-02<span class=sp>T</span>00:00:00</td><td>2017</td><td>3</td><td>2</td><td>Tim Hume</td><td>China is dismissing unfavorable media reports as f&#133;</td><td>China is dismissing unfavorable media reports as f&#133;</td><td>https://news.vice.com/en_us/article/xwvj7j/china-i&#133;</td><td></td><td>Vice News</td></tr>\n",
       "    <tr><td class='row_index'>10</td><td>2019-05-22<span class=sp>T</span>20:10:00</td><td>2019</td><td>5</td><td>22</td><td>Emily Stewart</td><td>“Elizabeth Warren called me!” is turning into a Tw&#133;</td><td>Elizabeth Warren is giving people a new reason to &#133;</td><td>https://www.vox.com/policy-and-politics/2019/5/22/&#133;</td><td></td><td>Vox</td></tr>\n",
       "    <tr><td class='row_index'>11</td><td>2019-06-23<span class=sp>T</span>00:00:00</td><td>2019</td><td>6</td><td>23</td><td>Jessica DiNapoli, Harry Brumpton</td><td>Hudson's Bay's chairman's buyout bid pits retail v&#133;</td><td>(Reuters) - The success of Hudson’s Bay Co Executi&#133;</td><td>https://www.reuters.com/article/hudsons-bay-ma-bak&#133;</td><td>Business News</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>12</td><td>2018-11-05<span class=sp>T</span>00:00:00</td><td>2018</td><td>11</td><td>5</td><td></td><td>Joakim Noah's Victoria Secret Model GF Lais Ribeir&#133;</td><td>Joakim Noah's ﻿model girlfriend Lais Ribeiro was s&#133;</td><td>https://www.tmz.com/2018/11/05/joakim-noah-kissing&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>13</td><td>2019-05-10<span class=sp>T</span>00:00:00</td><td>2019</td><td>5</td><td>10</td><td></td><td>Jermaine Jackson Rips Quincy Jones For Scrubbing M&#133;</td><td>Jermaine Jackson ﻿is slamming Quincy Jones for rem&#133;</td><td>https://www.tmz.com/2019/05/10/jermaine-jackson-qu&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>14</td><td>2019-03-28<span class=sp>T</span>00:00:00</td><td>2019</td><td>3</td><td>28</td><td></td><td>UK PM May presses on with bid to get Brexit deal t&#133;</td><td>LONDON (Reuters) - British Prime Minister Theresa &#133;</td><td>https://www.reuters.com/article/us-britain-eu-talk&#133;</td><td>World News</td><td>Reuters</td></tr>\n",
       "    <tr><td class='row_index'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td><td class='hellipsis'>&#x22EE;</td></tr>\n",
       "    <tr><td class='row_index'>2,583,636</td><td>2020-04-01<span class=sp>T</span>00:00:00</td><td>2020</td><td>4</td><td>1</td><td></td><td>Florida Ammo Selling Out On Heels of Stay-At-Home &#133;</td><td>Florida residents are gunning for all the ammo the&#133;</td><td>https://www.tmz.com/2020/04/01/florida-ammo-sellin&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>2,583,637</td><td>2020-04-01<span class=sp>T</span>00:00:00</td><td>2020</td><td>4</td><td>1</td><td></td><td>Disney Forcing Annual Pass Holders to Continue Pay&#133;</td><td>Disney isn't letting its most loyal customers off &#133;</td><td>https://www.tmz.com/2020/04/01/disney-forcing-annu&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>2,583,638</td><td>2020-04-01<span class=sp>T</span>00:00:00</td><td>2020</td><td>4</td><td>1</td><td></td><td>Nick Cannon Pimps Out His Impala with Custom Nipse&#133;</td><td>Nick Cannon still has Nipsey Hussle in his thought&#133;</td><td>https://www.tmz.com/2020/04/01/nick-cannon-impala-&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>2,583,639</td><td>2020-04-01<span class=sp>T</span>00:00:00</td><td>2020</td><td>4</td><td>1</td><td></td><td>Pete Buttigieg Says Governors Showing More Leaders&#133;</td><td>Pete Buttigieg says it's inspiring to see the lead&#133;</td><td>https://www.tmz.com/2020/04/01/pete-buttigieg-gove&#133;</td><td></td><td>TMZ</td></tr>\n",
       "    <tr><td class='row_index'>2,583,640</td><td>2020-04-01<span class=sp>T</span>00:00:00</td><td>2020</td><td>4</td><td>1</td><td></td><td>Ruth Bader Ginsburg Still Working Out with Trainer&#133;</td><td>Here's some good news we can all use ... Ruth Bade&#133;</td><td>https://www.tmz.com/2020/04/01/ruth-bader-ginsburg&#133;</td><td></td><td>TMZ</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>2,583,641 rows &times; 10 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<Frame#7fccda3d4660 2583641x10>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news[(f.article != ''),:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b25dfff9-b3d9-47b8-bfeb-40315d2ac2a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('date',\n",
       " 'year',\n",
       " 'month',\n",
       " 'day',\n",
       " 'author',\n",
       " 'title',\n",
       " 'article',\n",
       " 'url',\n",
       " 'section',\n",
       " 'publication')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news.names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "040612d1-0523-487b-af77-5308b5214142",
   "metadata": {},
   "source": [
    "## Get some counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28cf2b7c-c83a-4581-b0f7-d27a422cffdb",
   "metadata": {},
   "source": [
    "### Publication counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b6ab394d-0ad8-425a-80e1-c71bd5a4409a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='datatable'>\n",
       "  <table class='frame'>\n",
       "  <thead>\n",
       "    <tr class='colnames'><td class='row_index'></td><th>publication</th><th>pub_count</th></tr>\n",
       "    <tr class='coltypes'><td class='row_index'></td><td class='str' title='str32'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td><td class='int' title='int64'>&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;&#x25AA;</td></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><td class='row_index'>0</td><td></td><td>12577</td></tr>\n",
       "    <tr><td class='row_index'>1</td><td>Axios</td><td>47815</td></tr>\n",
       "    <tr><td class='row_index'>2</td><td>Business Insider</td><td>57953</td></tr>\n",
       "    <tr><td class='row_index'>3</td><td>Buzzfeed News</td><td>32819</td></tr>\n",
       "    <tr><td class='row_index'>4</td><td>CNBC</td><td>238096</td></tr>\n",
       "    <tr><td class='row_index'>5</td><td>CNN</td><td>127602</td></tr>\n",
       "    <tr><td class='row_index'>6</td><td>Economist</td><td>26227</td></tr>\n",
       "    <tr><td class='row_index'>7</td><td>Fox News</td><td>20144</td></tr>\n",
       "    <tr><td class='row_index'>8</td><td>Gizmodo</td><td>27228</td></tr>\n",
       "    <tr><td class='row_index'>9</td><td>Hyperallergic</td><td>13551</td></tr>\n",
       "    <tr><td class='row_index'>10</td><td>Mashable</td><td>94107</td></tr>\n",
       "    <tr><td class='row_index'>11</td><td>New Republic</td><td>11809</td></tr>\n",
       "    <tr><td class='row_index'>12</td><td>New Yorker</td><td>4701</td></tr>\n",
       "    <tr><td class='row_index'>13</td><td>People</td><td>136488</td></tr>\n",
       "    <tr><td class='row_index'>14</td><td>Politico</td><td>46377</td></tr>\n",
       "    <tr><td class='row_index'>15</td><td>Refinery 29</td><td>111433</td></tr>\n",
       "    <tr><td class='row_index'>16</td><td>Reuters</td><td>840094</td></tr>\n",
       "    <tr><td class='row_index'>17</td><td>TMZ</td><td>49595</td></tr>\n",
       "    <tr><td class='row_index'>18</td><td>TechCrunch</td><td>52095</td></tr>\n",
       "    <tr><td class='row_index'>19</td><td>The Hill</td><td>208411</td></tr>\n",
       "    <tr><td class='row_index'>20</td><td>The New York Times</td><td>252259</td></tr>\n",
       "    <tr><td class='row_index'>21</td><td>The Verge</td><td>52424</td></tr>\n",
       "    <tr><td class='row_index'>22</td><td>Vice</td><td>101137</td></tr>\n",
       "    <tr><td class='row_index'>23</td><td>Vice News</td><td>15539</td></tr>\n",
       "    <tr><td class='row_index'>24</td><td>Vox</td><td>47272</td></tr>\n",
       "    <tr><td class='row_index'>25</td><td>Washington Post</td><td>40882</td></tr>\n",
       "    <tr><td class='row_index'>26</td><td>Wired</td><td>20243</td></tr>\n",
       "  </tbody>\n",
       "  </table>\n",
       "  <div class='footer'>\n",
       "    <div class='frame_dimensions'>27 rows &times; 2 columns</div>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<Frame#7fd378a2da80 27x2>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news[:, {\"pub_count\" : dt.count()}, by(\"publication\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "1e28ab21-8c44-42c3-b845-c92870f8446b",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_news_pd = all_news.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "630db2fd-d722-475b-bfe5-299b81d98a88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>article</th>\n",
       "      <th>url</th>\n",
       "      <th>section</th>\n",
       "      <th>publication</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-12-09 18:31:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>12.0</td>\n",
       "      <td>9</td>\n",
       "      <td>Lee Drutman</td>\n",
       "      <td>We should take concerns about the health of li...</td>\n",
       "      <td>This post is part of Polyarchy, an independent...</td>\n",
       "      <td>https://www.vox.com/polyarchy/2016/12/9/138983...</td>\n",
       "      <td></td>\n",
       "      <td>Vox</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-10-07 21:26:46</td>\n",
       "      <td>2016</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7</td>\n",
       "      <td>Scott Davis</td>\n",
       "      <td>Colts GM Ryan Grigson says Andrew Luck's contr...</td>\n",
       "      <td>The Indianapolis Colts made Andrew Luck the h...</td>\n",
       "      <td>https://www.businessinsider.com/colts-gm-ryan-...</td>\n",
       "      <td></td>\n",
       "      <td>Business Insider</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018-01-26 00:00:00</td>\n",
       "      <td>2018</td>\n",
       "      <td>1.0</td>\n",
       "      <td>26</td>\n",
       "      <td></td>\n",
       "      <td>Trump denies report he ordered Mueller fired</td>\n",
       "      <td>DAVOS, Switzerland (Reuters) - U.S. President ...</td>\n",
       "      <td>https://www.reuters.com/article/us-davos-meeti...</td>\n",
       "      <td>Davos</td>\n",
       "      <td>Reuters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-06-27 00:00:00</td>\n",
       "      <td>2019</td>\n",
       "      <td>6.0</td>\n",
       "      <td>27</td>\n",
       "      <td></td>\n",
       "      <td>France's Sarkozy reveals his 'Passions' but in...</td>\n",
       "      <td>PARIS (Reuters) - Former French president Nico...</td>\n",
       "      <td>https://www.reuters.com/article/france-politic...</td>\n",
       "      <td>World News</td>\n",
       "      <td>Reuters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-01-27 00:00:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27</td>\n",
       "      <td></td>\n",
       "      <td>Paris Hilton: Woman In Black For Uncle Monty's...</td>\n",
       "      <td>Paris Hilton arrived at LAX Wednesday dressed ...</td>\n",
       "      <td>https://www.tmz.com/2016/01/27/paris-hilton-mo...</td>\n",
       "      <td></td>\n",
       "      <td>TMZ</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 date  year  month  day       author  \\\n",
       "0 2016-12-09 18:31:00  2016   12.0    9  Lee Drutman   \n",
       "1 2016-10-07 21:26:46  2016   10.0    7  Scott Davis   \n",
       "2 2018-01-26 00:00:00  2018    1.0   26                \n",
       "3 2019-06-27 00:00:00  2019    6.0   27                \n",
       "4 2016-01-27 00:00:00  2016    1.0   27                \n",
       "\n",
       "                                               title  \\\n",
       "0  We should take concerns about the health of li...   \n",
       "1  Colts GM Ryan Grigson says Andrew Luck's contr...   \n",
       "2       Trump denies report he ordered Mueller fired   \n",
       "3  France's Sarkozy reveals his 'Passions' but in...   \n",
       "4  Paris Hilton: Woman In Black For Uncle Monty's...   \n",
       "\n",
       "                                             article  \\\n",
       "0  This post is part of Polyarchy, an independent...   \n",
       "1   The Indianapolis Colts made Andrew Luck the h...   \n",
       "2  DAVOS, Switzerland (Reuters) - U.S. President ...   \n",
       "3  PARIS (Reuters) - Former French president Nico...   \n",
       "4  Paris Hilton arrived at LAX Wednesday dressed ...   \n",
       "\n",
       "                                                 url     section  \\\n",
       "0  https://www.vox.com/polyarchy/2016/12/9/138983...               \n",
       "1  https://www.businessinsider.com/colts-gm-ryan-...               \n",
       "2  https://www.reuters.com/article/us-davos-meeti...       Davos   \n",
       "3  https://www.reuters.com/article/france-politic...  World News   \n",
       "4  https://www.tmz.com/2016/01/27/paris-hilton-mo...               \n",
       "\n",
       "        publication  \n",
       "0               Vox  \n",
       "1  Business Insider  \n",
       "2           Reuters  \n",
       "3           Reuters  \n",
       "4               TMZ  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news_pd.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}