feature_generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook was run on an AWS SageMaker instance.  Training data is cleaned using functions from feature_generation_functions.py and the pickeled and sent to an AWS S3 bucket.  Limitations from EC2 instance required that we clean and pickle data in smaller frames of chunk size 100,000 rows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/ec2-user/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "import pickle_functions as pf #has io, boto3, boto3.session, _pickle, pandas\n",
    "import feature_generation_functions as fgf\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"s3://advancedml-koch-mathur-hinkson/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (100000, 45),\n",
       " (4874, 45)]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#citation:  https://stackoverflow.com/questions/33367142/split-dataframe-into-relatively-even-chunks-according-to-length/33368088\n",
    "\n",
    "n = 100000  #chunk row size\n",
    "train_sub_dfs = [train[i:i+n] for i in range(0,train.shape[0],n)]\n",
    "\n",
    "[i.shape for i in train_sub_dfs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split comments...Elapsed Time:  0.002 minutes\n",
      "Cleaned with stopwords...Elapsed Time:  0.004 minutes\n",
      "Cleaned without stopwords...Elapsed Time:  0.006 minutes\n",
      "Stemmed (Porter)...Elapsed Time:  0.074 minutes\n",
      "Stemmed (Lancaster)...Elapsed Time:  0.063 minutes\n",
      "Created bigrams...Elapsed Time:  0.002 minutes\n",
      "Calculated uppercase pct...Elapsed Time:  0.002 minutes\n",
      "Count punctuation...Elapsed Time:  0.001 minutes\n",
      "Count words...Elapsed Time:  0.001 minutes\n",
      "Count stopwords pct...Elapsed Time:  0.003 minutes\n",
      "Count uppercase words...Elapsed Time:  0.002 minutes\n",
      "\n",
      "DONE GENERATING FEATURES\n",
      "Pickled and sent to bucket!\n"
     ]
    }
   ],
   "source": [
    "sub_train_df19 = train_sub_dfs[18]\n",
    "sub_train_df19_preprocessed = fgf.generate_features(sub_train_df19)\n",
    "pf.write_pickle_to_s3bucket(filename='sub_train_df19_preprocessed', \n",
    "                            df=sub_train_df19_preprocessed, \n",
    "                            bucket_name='advancedml-koch-mathur-hinkson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4874, 60)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_train_df19_preprocessed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}