step1_multithread.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "matching between transcripts and audios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as py\n",
    "from multiprocessing import Process, JoinableQueue, Manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcripts = pd.read_csv(\"transcript_metadata.csv\", encoding = \"ISO-8859-1\")\n",
    "audios = pd.read_csv(\"audio_metadata.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcripts[\"TextDateTime\"] = transcripts[\"TextDateTime\"].astype(str)\n",
    "transcripts[\"Year\"] = transcripts[\"Year\"].astype(str)\n",
    "# transcripts[\"Quarter\"] = pd.to_datetime(transcripts[\"TextDateTime\"]).dt.quarter\n",
    "# transcripts[\"Quarter\"] = transcripts[\"Quarter\"].fillna(0)\n",
    "# transcripts[\"Quarter\"] = transcripts[\"Quarter\"].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fixing the date format in audios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates_list = []\n",
    "audios[\"Date\"] = audios[\"Date\"].astype(str)\n",
    "for date in audios[\"Date\"]:\n",
    "    date_list = date.split(\"/\")\n",
    "    if len(date_list[0]) == 1:\n",
    "        date_list[0] = \"0\" + date_list[0]\n",
    "    if len(date_list[1]) == 1:\n",
    "        date_list[1] = \"0\" + date_list[1]\n",
    "    dates_list.append(\"/\".join(date_list))\n",
    "audios[\"Date\"] = pd.DataFrame(dates_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "add a Date field to transcripts in MM/DD/YY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_list = []\n",
    "for date in transcripts.TextDateTime:\n",
    "    try:\n",
    "        temp = date[5:7]+ \"/\" + date[8:10] + \"/\" + date[0:4]\n",
    "    except BaseException:\n",
    "        date_list.append(0)\n",
    "    else:\n",
    "        date_list.append(temp)\n",
    "transcripts[\"Date\"] = pd.DataFrame(date_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Exchange</th>\n",
       "      <th>Symbol</th>\n",
       "      <th>Description</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>Year</th>\n",
       "      <th>Time</th>\n",
       "      <th>APM</th>\n",
       "      <th>TimeZone</th>\n",
       "      <th>TextDateTime</th>\n",
       "      <th>Person</th>\n",
       "      <th>Title/Institution</th>\n",
       "      <th>Role</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>First Solar, Inc.</td>\n",
       "      <td>NASDAQ</td>\n",
       "      <td>FSLR</td>\n",
       "      <td>First Solar, Inc. (FSLR) CEO Mark Widmar on 20...</td>\n",
       "      <td>December</td>\n",
       "      <td>11</td>\n",
       "      <td>2018</td>\n",
       "      <td>4:30</td>\n",
       "      <td>PM</td>\n",
       "      <td>ET</td>\n",
       "      <td>2018-12-11 16:30 ET</td>\n",
       "      <td>Steve Haymore</td>\n",
       "      <td>Investor Relations</td>\n",
       "      <td>E</td>\n",
       "      <td>12/11/2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>First Solar, Inc.</td>\n",
       "      <td>NASDAQ</td>\n",
       "      <td>FSLR</td>\n",
       "      <td>First Solar, Inc. (FSLR) CEO Mark Widmar on 20...</td>\n",
       "      <td>December</td>\n",
       "      <td>11</td>\n",
       "      <td>2018</td>\n",
       "      <td>4:30</td>\n",
       "      <td>PM</td>\n",
       "      <td>ET</td>\n",
       "      <td>2018-12-11 16:30 ET</td>\n",
       "      <td>Mark Widmar</td>\n",
       "      <td>Chief Executive Officer</td>\n",
       "      <td>E</td>\n",
       "      <td>12/11/2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>First Solar, Inc.</td>\n",
       "      <td>NASDAQ</td>\n",
       "      <td>FSLR</td>\n",
       "      <td>First Solar, Inc. (FSLR) CEO Mark Widmar on 20...</td>\n",
       "      <td>December</td>\n",
       "      <td>11</td>\n",
       "      <td>2018</td>\n",
       "      <td>4:30</td>\n",
       "      <td>PM</td>\n",
       "      <td>ET</td>\n",
       "      <td>2018-12-11 16:30 ET</td>\n",
       "      <td>Alex Bradley</td>\n",
       "      <td>Chief Financial Officer</td>\n",
       "      <td>E</td>\n",
       "      <td>12/11/2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>First Solar, Inc.</td>\n",
       "      <td>NASDAQ</td>\n",
       "      <td>FSLR</td>\n",
       "      <td>First Solar, Inc. (FSLR) CEO Mark Widmar on 20...</td>\n",
       "      <td>December</td>\n",
       "      <td>11</td>\n",
       "      <td>2018</td>\n",
       "      <td>4:30</td>\n",
       "      <td>PM</td>\n",
       "      <td>ET</td>\n",
       "      <td>2018-12-11 16:30 ET</td>\n",
       "      <td>Philip Shen</td>\n",
       "      <td>ROTH Capital Partners</td>\n",
       "      <td>A</td>\n",
       "      <td>12/11/2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>First Solar, Inc.</td>\n",
       "      <td>NASDAQ</td>\n",
       "      <td>FSLR</td>\n",
       "      <td>First Solar, Inc. (FSLR) CEO Mark Widmar on 20...</td>\n",
       "      <td>December</td>\n",
       "      <td>11</td>\n",
       "      <td>2018</td>\n",
       "      <td>4:30</td>\n",
       "      <td>PM</td>\n",
       "      <td>ET</td>\n",
       "      <td>2018-12-11 16:30 ET</td>\n",
       "      <td>Brian Lee</td>\n",
       "      <td>Goldman Sachs</td>\n",
       "      <td>A</td>\n",
       "      <td>12/11/2018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name Exchange Symbol  \\\n",
       "0  First Solar, Inc.   NASDAQ   FSLR   \n",
       "1  First Solar, Inc.   NASDAQ   FSLR   \n",
       "2  First Solar, Inc.   NASDAQ   FSLR   \n",
       "3  First Solar, Inc.   NASDAQ   FSLR   \n",
       "4  First Solar, Inc.   NASDAQ   FSLR   \n",
       "\n",
       "                                         Description     Month  Day  Year  \\\n",
       "0  First Solar, Inc. (FSLR) CEO Mark Widmar on 20...  December   11  2018   \n",
       "1  First Solar, Inc. (FSLR) CEO Mark Widmar on 20...  December   11  2018   \n",
       "2  First Solar, Inc. (FSLR) CEO Mark Widmar on 20...  December   11  2018   \n",
       "3  First Solar, Inc. (FSLR) CEO Mark Widmar on 20...  December   11  2018   \n",
       "4  First Solar, Inc. (FSLR) CEO Mark Widmar on 20...  December   11  2018   \n",
       "\n",
       "   Time APM TimeZone         TextDateTime         Person  \\\n",
       "0  4:30  PM       ET  2018-12-11 16:30 ET  Steve Haymore   \n",
       "1  4:30  PM       ET  2018-12-11 16:30 ET    Mark Widmar   \n",
       "2  4:30  PM       ET  2018-12-11 16:30 ET   Alex Bradley   \n",
       "3  4:30  PM       ET  2018-12-11 16:30 ET    Philip Shen   \n",
       "4  4:30  PM       ET  2018-12-11 16:30 ET      Brian Lee   \n",
       "\n",
       "         Title/Institution Role        Date  \n",
       "0       Investor Relations    E  12/11/2018  \n",
       "1  Chief Executive Officer    E  12/11/2018  \n",
       "2  Chief Financial Officer    E  12/11/2018  \n",
       "3    ROTH Capital Partners    A  12/11/2018  \n",
       "4            Goldman Sachs    A  12/11/2018  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transcripts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ticker</th>\n",
       "      <th>Quarter</th>\n",
       "      <th>Year</th>\n",
       "      <th>Name</th>\n",
       "      <th>Act</th>\n",
       "      <th>Est</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Timezone</th>\n",
       "      <th>Length</th>\n",
       "      <th>Filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OHRP</td>\n",
       "      <td>Q4</td>\n",
       "      <td>2017</td>\n",
       "      <td>Ohr Pharmaceuticals, Inc.</td>\n",
       "      <td>Act -0.08</td>\n",
       "      <td>Est -0.08</td>\n",
       "      <td>12/15/2017</td>\n",
       "      <td>8:30 am</td>\n",
       "      <td>EST</td>\n",
       "      <td>9:55</td>\n",
       "      <td>Q4_2017_Ohr_Pharmaceuticals,_Inc._Earnings_Con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BOOT</td>\n",
       "      <td>Q3</td>\n",
       "      <td>2018</td>\n",
       "      <td>Boot Barn Holdings, Inc.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>01/31/2018</td>\n",
       "      <td>4:30 pm</td>\n",
       "      <td>EST</td>\n",
       "      <td>0:10</td>\n",
       "      <td>Q3_2018_Boot_Barn_Holdings,_Inc._Earnings_Conf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PAYX</td>\n",
       "      <td>Q2</td>\n",
       "      <td>2018</td>\n",
       "      <td>Paychex, Inc.</td>\n",
       "      <td>Act 0.59</td>\n",
       "      <td>Est 0.594</td>\n",
       "      <td>12/21/2017</td>\n",
       "      <td>9:30 am</td>\n",
       "      <td>EST</td>\n",
       "      <td>1:20:13</td>\n",
       "      <td>Q2_2018_Paychex,_Inc._Earnings_Conference_call...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AU</td>\n",
       "      <td>Q4</td>\n",
       "      <td>2017</td>\n",
       "      <td>Anglo Gold Ashanti Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02/21/2018</td>\n",
       "      <td>8:00 am</td>\n",
       "      <td>EST</td>\n",
       "      <td>0:10</td>\n",
       "      <td>Q4_2017_Anglo_Gold_Ashanti_Limited_Earnings_Co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CYOU</td>\n",
       "      <td>Q4</td>\n",
       "      <td>2017</td>\n",
       "      <td>Changyou.Com Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02/21/2018</td>\n",
       "      <td>7:00 am</td>\n",
       "      <td>EST</td>\n",
       "      <td>0:10</td>\n",
       "      <td>Q4_2017_Changyou.Com_Limited_Earnings_Conferen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Ticker Quarter  Year                        Name        Act        Est  \\\n",
       "0   OHRP      Q4  2017   Ohr Pharmaceuticals, Inc.  Act -0.08  Est -0.08   \n",
       "1   BOOT      Q3  2018    Boot Barn Holdings, Inc.        NaN        NaN   \n",
       "2   PAYX      Q2  2018               Paychex, Inc.   Act 0.59  Est 0.594   \n",
       "3     AU      Q4  2017  Anglo Gold Ashanti Limited        NaN        NaN   \n",
       "4   CYOU      Q4  2017        Changyou.Com Limited        NaN        NaN   \n",
       "\n",
       "         Date     Time Timezone   Length  \\\n",
       "0  12/15/2017  8:30 am      EST     9:55   \n",
       "1  01/31/2018  4:30 pm      EST     0:10   \n",
       "2  12/21/2017  9:30 am      EST  1:20:13   \n",
       "3  02/21/2018  8:00 am      EST     0:10   \n",
       "4  02/21/2018  7:00 am      EST     0:10   \n",
       "\n",
       "                                            Filename  \n",
       "0  Q4_2017_Ohr_Pharmaceuticals,_Inc._Earnings_Con...  \n",
       "1  Q3_2018_Boot_Barn_Holdings,_Inc._Earnings_Conf...  \n",
       "2  Q2_2018_Paychex,_Inc._Earnings_Conference_call...  \n",
       "3  Q4_2017_Anglo_Gold_Ashanti_Limited_Earnings_Co...  \n",
       "4  Q4_2017_Changyou.Com_Limited_Earnings_Conferen...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audios.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "10",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2656\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2657\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2658\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 10",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-22b95a31999a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtranscripts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2925\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2926\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2927\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2928\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2929\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2657\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2658\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2659\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2660\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2661\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 10"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-d0b508db5d3e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtranscript\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m11\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m12\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m13\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m14\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtranscripts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitertuples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Pandas'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtranscript\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/queues.py\u001b[0m in \u001b[0;36mput\u001b[0;34m(self, obj, block, timeout)\u001b[0m\n\u001b[1;32m    300\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Queue {0!r} is closed\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mFull\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "manager1 = Manager()\n",
    "a = [None] * len(transcripts)\n",
    "filename_list = manager1.list(a)\n",
    "\n",
    "manager2 = Manager()\n",
    "b = [0] * len(transcripts)\n",
    "found_number = manager2.list(a)\n",
    "\n",
    "q = JoinableQueue()\n",
    "\n",
    "for transcript in transcripts.itertuples(index=True, name='Pandas'):\n",
    "    q.put(transcript)\n",
    "\n",
    "def match(q):\n",
    "    while not q.empty():\n",
    "        transcript = q.get()\n",
    "        flag = False\n",
    "        for audio in audios.loc[(audios[\"Ticker\"] == transcript[3]) & (audios[\"Date\"] == transcript[15])].itertuples(index=True, name='Pandas'):\n",
    "            flag = True\n",
    "            filename_list[transcript[0]] = audio[11]\n",
    "            found_number[transcript[0]] += 1\n",
    "        if not flag:\n",
    "            filename_list[transcript[0]] = \"NaN\"\n",
    "        if transcript[0] % 10000 == 0:\n",
    "            print(transcript[0])\n",
    "        q.task_done()\n",
    "\n",
    "num_threads = 8\n",
    "for i in range(num_threads):\n",
    "    print('Starting thread ', i)\n",
    "    worker = Process(target=match, args=(q,))\n",
    "    worker.start()\n",
    "q.join()\n",
    "print('All tasks completed.')\n",
    "\n",
    "transcripts[\"the corresponding audio file name\"] = pd.DataFrame(filename_list)\n",
    "transcripts[\"# of match\"] = pd.DataFrame(found_number)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcripts.to_csv(\"final_multithread.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "audio conversion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "install ffmpeg, pydub before using"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "m4a -> mp3 increase file size a HELL LOT, be aware when using"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydub import AudioSegment\n",
    "import os\n",
    "from multiprocessing import Process, JoinableQueue, cpu_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "config the input output directories manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.getcwd()\n",
    "audio_input_folder = \"sample\"\n",
    "audio_output_folder = \"audio_output\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "single file conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sound = AudioSegment.from_file(path + \"/\" + audio_input_folder + \"/\" + \"1Q_2013_Pike_Electric_Corporation_Earnings_Conference_Call.m4a\")\n",
    "sound.export(path + \"/\" + audio_output_folder + \"/\" + \"test.mp3\", format=\"mp3\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "batch conversion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BE AWARE OF THE LAGGGGGGGG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filelist = os.listdir(path + \"/\" + audio_input_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "file checking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".DS_Store is wrong\n"
     ]
    }
   ],
   "source": [
    "for file in filelist:\n",
    "    if len(file) <= 5:\n",
    "        filelist.remove(file)\n",
    "        print(file + \" is wrong\")\n",
    "    elif file[-4:] != \".m4a\":\n",
    "        filelist.remove(file)\n",
    "        print(file + \" is wrong\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting thread  0\n",
      "Starting thread  1\n",
      "Thread 0 now processing 3D_Systems_Corp_Analyst_and_Investor_Meeting.m4a\n",
      "Starting thread  2\n",
      "Thread 1 now processing 2012_Oshkosh_Corporation_Analyst_Day.m4a\n",
      "Starting thread  3\n",
      "Thread 2 now processing 2012_A&B_Real_Estate_Sell-Side_Analyst_Day.m4a\n",
      "Starting thread  4\n",
      "Thread 3 now processing 2012_Pinnacle_West_Analyst_Day.m4a\n",
      "Starting thread  5\n",
      "Thread 4 now processing 3_M_Company_At_Sanford_C._Bernstein_&_Co._Strategic_Decisions_Conference.m4a\n",
      "Starting thread  6\n",
      "Thread 5 now processing 1Q12_Earnings_Conference_Call_(Replay).m4a\n",
      "Starting thread  7\n",
      "Thread 7 now processing 4Q_and_Full_Year_2010_Goodrich_Corporation_Earnings_Conference_Call.m4a\n",
      "Thread 6 now processing 1Q_2013_Pike_Electric_Corporation_Earnings_Conference_Call.m4a\n",
      "Thread 6 now processing 2012_ASCO_Investor_Event.m4a\n",
      "Thread 5 now processing 4Q11_Earnings_Conference_Call.m4a\n",
      "Thread 4 now processing 4Q_FY11_Cardinal_Health_Inc_Earnings_Conference_Call.m4a\n",
      "Thread 7 now processing 2011_Assurant_Investor_Day_.m4a\n",
      "Thread 6 now processing 2010_ONEOK_and_ONEOK_Partners_Investor_Conference.m4a\n",
      "Thread 5 now processing 3M_Company_at_Deutsche_Bank_Global_Industrials_and_Basic_Materials_Conference.m4a\n",
      "Thread 4 now processing 2011_Annual_Meeting_of_Stockholders.m4a\n",
      "Thread 5 now processing 2Q13_Lexmark_International,_Inc._Earnings_Conference_Call_(Replay).m4a\n",
      "Thread 3 now processing 2Q12_Earnings_Conference_Call__-_(Replay).m4a\n",
      "Thread 2 now processing 3M_Co_2012_Outlook_Meeting.m4a\n",
      "Thread 5 now processing 2012_Guidance_and_Pipeline_Update.m4a\n",
      "Thread 3 now processing 2012_Annual_Meeting_of_Shareholders.m4a\n",
      "Thread 2 now processing 3_M_Company_At_Jp_Morgan_Diversified_Industries_Conference.m4a\n",
      "Thread 0 now processing 1Q13_Lexmark_International,_Inc._Earnings_Conference_Call_(Replay).m4a\n",
      "Thread 3 now processing 2Q_FY2013_National_Fuel_Gas_Company_Earnings_Conference_Call.m4a\n",
      "Thread 5 now processing 3Q_FY_2013_Earnings_Call.m4a\n",
      "Thread 2 now processing 3M_Company_at_Bank_of_America_Merrill_Lynch_Global_Industrial_&_EU_Autos_Conference.m4a\n",
      "Thread 0 now processing 2012_StanCorp_Financial_Group,_Inc._Investor_Meeting.m4a\n",
      "Thread 6 now processing 4Q12_Lexmark_International_Inc_Earnings_Call_(Replay).m4a\n",
      "Thread 3 now processing 2011_Emerson_Investor_Conference.m4a\n",
      "Thread 5 now processing 2012_Matson_Sell-Side_Analyst_Day.m4a\n",
      "Thread 2 now processing 3M_Company_Investor_Day_Meeting.m4a\n",
      "Thread 6 now processing 2011_Duke_Realty_Investor_and_Analyst_Conference.m4a\n",
      "Thread 5 now processing 3_M_Company_At_William_Blair_&_Company,_Llc_Growth_Stock_Conference.m4a\n",
      "Thread 1 now processing 2012_Annual_Meeting_of_Stockholders.m4a\n",
      "Thread 3 now processing 24th_Annual_Piper_Jaffray_Healthcare_Conference.m4a\n",
      "Thread 1 now processing 4Q_2011_Earnings_Conference_Call.m4a\n",
      "Thread 4 now processing 4Q12_Pension_Mark-to-Market_Presentation.m4a\n",
      "Thread 3 now processing 2011_Annual_Meeting_of_Shareholders.m4a\n",
      "Thread 4 now processing 2012_Dell_Financial_Analyst_Meeting.m4a\n",
      "Thread 0 now processing 3D_Systems_Investor_Event_at_NYSE.m4a\n",
      "Thread 5 now processing 2012_HP_Annual_Meeting_of_Stockholders.m4a\n",
      "Thread 1 now processing 2011_ASH_Investor_Webcast_Presentation.m4a\n",
      "Thread 3 now processing 3D_Systems_Corporation_to_Acquire_Z_Corporation_and_Vidar_Systems_-_Conference_Call.m4a\n",
      "Thread 6 now processing 2012_Baxter_Investor_Conference_Webcast.m4a\n",
      "Thread 3 now processing 2011_Northrop_Grumman_Corp_Shareholders_Meeting.m4a\n",
      "Thread 1 now processing 2011_KBR_Inc_Analyst_and_Investor_Day.m4a\n",
      "Thread 3 now processing 3M_Co_at_Goldman_Sachs_Global_Industrials_Conference.m4a\n",
      "Thread 5 now processing 3M_Company_Analyst_Meeting.m4a\n",
      "Thread 0 now processing 2012_StanCorp_Financial_Group,_Inc._Shareholder_Meeting.m4a\n",
      "Thread 3 now processing 3Q12_Earnings_Conference_Call_(Replay).m4a\n",
      "Thread 0 now processing 2011_News_Corp_Annual_StockHolder_Meeting.m4a\n",
      "Thread 3 now processing 2011_Q4_Coinstar,_Inc._Earnings_Conference_Call.m4a\n",
      "Thread 3 now processing 4Q_2011_LSI_Corporation_Earnings_Conference_Call.m4a\n",
      "All tasks completed.\n"
     ]
    }
   ],
   "source": [
    "q = JoinableQueue()\n",
    "for file in filelist:\n",
    "    q.put(file)\n",
    "\n",
    "def convert(q, i):\n",
    "    while not q.empty():\n",
    "        file = q.get()\n",
    "        print(\"Thread \" + str(i) + \" now processing \" + file)\n",
    "        sound = AudioSegment.from_file(path + \"/\" + audio_input_folder + \"/\" + file)\n",
    "        mp3name = file[:-4] + \".wav\"\n",
    "        sound.export(path + \"/\" + audio_output_folder + \"/\" + mp3name, format=\"wav\")\n",
    "        q.task_done()\n",
    "\n",
    "num_threads = cpu_count()\n",
    "for i in range(num_threads):\n",
    "    print('Starting thread ', i)\n",
    "    worker = Process(target=convert, args=(q, i))\n",
    "    worker.start()\n",
    "q.join()\n",
    "print('All tasks completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}