make_dataset_pake_season.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2016-05-11 20:13:27.020383\n",
      "Minimal 12 menitan\n",
      "2016-05-11 20:16:12.834993\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "#from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "LABEL = 'returnQuantity'\n",
    "\n",
    "def preprocess(df):\n",
    "\n",
    "    print 'Minimal 12 menitan'\n",
    "    # Deletions\n",
    "    # none\n",
    "    \n",
    "    # Temporary Label\n",
    "    df['binaryLabel'] = df['returnQuantity'].apply(lambda x: 1 if x > 1 else x)\n",
    "\n",
    "    # Months\n",
    "    df['mmdd'] = df.orderDate.str[-5:]\n",
    "    df['months'] = df.orderDate.str[-5:-3].astype(np.int8)\n",
    "    \n",
    "    # Seasons\n",
    "    def getSeasons(months):\n",
    "        if months == 12 or months <= 2:\n",
    "            quarter = 1\n",
    "        elif months <= 5:\n",
    "            quarter = 2                 \n",
    "        elif months <= 8:\n",
    "            quarter = 3  \n",
    "        else:\n",
    "            quarter = 4\n",
    "        return quarter\n",
    "    df['seasons'] = df['months'].apply(getSeasons).astype(np.int32)\n",
    "\n",
    "    # Total price for an order\n",
    "    order_total_dict = df[['orderID', 'price']].groupby('orderID').sum()['price'].to_dict()\n",
    "    df['order_total'] = df.orderID.apply(order_total_dict.get).astype(np.float32)\n",
    "    del order_total_dict\n",
    "\n",
    "    # Average budget of the customer\n",
    "    customer_budget_dict = df[['customerID', 'order_total']].groupby('customerID').mean()['order_total'].to_dict()\n",
    "    df['customer_budget'] = df.customerID.apply(customer_budget_dict.get).astype(np.float32)\n",
    "    del customer_budget_dict\n",
    "\n",
    "    # Customer expense ratio\t\n",
    "    total_spent_dict = df[['customerID', 'order_total']].groupby('customerID').sum()['order_total'].to_dict()\n",
    "    df['total_spent'] = df.customerID.apply(total_spent_dict.get).astype(np.float32)\n",
    "    del total_spent_dict\n",
    "    df['expense_ratio'] = (df['customer_budget'] / df['total_spent']).astype(np.float16)\n",
    "    del df['total_spent']\n",
    "\n",
    "    # Mau ga mau mesti impute rrp dulu biar ga NaN :(\n",
    "    df.rrp.fillna(df.rrp.median(), inplace=True)\n",
    "    \n",
    "    # Impute productGroup biar ga NaN\n",
    "    df.productGroup.fillna(111, inplace=True)\n",
    "\n",
    "    # Metode farah\n",
    "    temp_quantity = df.quantity.copy()\n",
    "    temp_quantity[temp_quantity==0] = 1\n",
    "    df['unit_price'] = (df.price/temp_quantity).astype(np.float32)\n",
    "\n",
    "    usual_unit_price_dict = df[['articleID', 'unit_price']].groupby('articleID').median().unit_price.to_dict()\n",
    "    df['usual_unit_price']=df.articleID.apply(usual_unit_price_dict.get).astype(np.float32)\n",
    "    del usual_unit_price_dict\n",
    "\n",
    "    df['price_diff'] = (df['unit_price']-df.usual_unit_price).astype(np.float32)\n",
    "\n",
    "    # Price after rebate = order_total - voucherAmount\n",
    "    df['after_voucher'] = df.order_total - df.voucherAmount\n",
    "\n",
    "    # Orders (as in rank)\n",
    "    df['order_order']  = df[['customerID', 'orderID']].groupby(['customerID']).cumcount() + 1\n",
    "    df['choice_order'] = df[['orderID', 'articleID']].groupby(['orderID']).cumcount() + 1\n",
    "\n",
    "    # Return Probabilities\n",
    "\n",
    "    def append_return_prob(df, column):\n",
    "        df2 = df[[column,'returnQuantity','quantity']]\n",
    "        df_return_probability = df2.groupby(column).sum()\n",
    "        df_return_probability[ column + '_prob' ]  = df_return_probability.returnQuantity / df_return_probability.quantity\n",
    "        return_prob_dict = df_return_probability[ column + '_prob' ].to_dict()\n",
    "        del df_return_probability\n",
    "        df[ column + '_prob' ] = df[column].apply(return_prob_dict.get).replace(np.NaN, 0.5).replace(np.inf, 0.5)\n",
    "        del return_prob_dict\n",
    "\n",
    "    def append_return_cumprob(df, column):\n",
    "        df2 = df[[column,'returnQuantity','quantity']]\n",
    "        df_return_probability = df2.groupby(column)['returnQuantity','quantity'].cumsum()\n",
    "        df_return_probability[ column + '_prob' ]  = df_return_probability.returnQuantity / df_return_probability.quantity\n",
    "        df[ column + '_cumprob' ] = df_return_probability[ column + '_prob' ].replace(np.NaN, 0.5).replace(np.inf, 0.5).apply(lambda x: 1 if x > 1 else x)\n",
    "\n",
    "    append_return_cumprob(df, 'articleID')\n",
    "    append_return_cumprob(df, 'colorCode')\n",
    "    append_return_cumprob(df, 'customerID')\n",
    "    append_return_cumprob(df, 'sizeCode')\n",
    "\n",
    "    float_64_columns = df.loc[:, df.dtypes == np.float64].columns\n",
    "    for col in float_64_columns:\n",
    "        df[col] = df[col].astype(np.float16)\n",
    "\n",
    "    int_64_columns = df.loc[:, df.dtypes == np.int64].columns\n",
    "    for col in int_64_columns:\n",
    "        df[col] = df[col].astype(np.int32)\n",
    "\n",
    "    # Article & Color\n",
    "    column='ac'\n",
    "    columns=['articleID','colorCode']\n",
    "    df2 = df[['articleID','colorCode','returnQuantity','quantity']]\n",
    "    df_return_probability = df2.groupby(columns).cumsum()\n",
    "    df_return_probability[ 'ac_cumprob' ]  = df_return_probability.returnQuantity / df_return_probability.quantity\n",
    "    df[ 'ac_cumprob' ] = df_return_probability[ 'ac_cumprob' ].replace(np.NaN, 0.5).replace(np.inf, 0.5).apply(lambda x: 1 if x > 1 else x)\n",
    "\n",
    "    # Article & Size\n",
    "    columns=['articleID','sizeCode']\n",
    "    df2 = df[['articleID','sizeCode','returnQuantity','quantity']]\n",
    "    df_return_probability = df2.groupby(columns).cumsum()\n",
    "    df_return_probability[ 'as_cumprob' ]  = df_return_probability.returnQuantity / df_return_probability.quantity\n",
    "    df[ 'as_cumprob' ] = df_return_probability[ 'as_cumprob' ].replace(np.NaN, 0.5).replace(np.inf, 0.5).apply(lambda x: 1 if x > 1 else x)\n",
    "\n",
    "    # Product Group & Season\n",
    "    columns=['productGroup','seasons']\n",
    "    df2 = df[['productGroup','seasons','returnQuantity','quantity']]\n",
    "    df_return_probability = df2.groupby(columns).cumsum()\n",
    "    df_return_probability[ 'proSea_cumprob' ]  = df_return_probability.returnQuantity / df_return_probability.quantity\n",
    "    df[ 'proSea_cumprob' ] = df_return_probability[ 'proSea_cumprob' ].replace(np.NaN, 0.5).replace(np.inf, 0.5).apply(lambda x: 1 if x > 1 else x)\n",
    "    \n",
    "    #df.productGroup = df.productGroup.astype(np.int8)\t\n",
    "\n",
    "    # Iseng ubah float ke integer.\n",
    "    # df.price = df.price.astype(np.int32)\n",
    "    # df.budget = df.budget.astype(np.int32)\n",
    "\n",
    "    # Is it Wednesday?\n",
    "    #print(\"[SLOW] Get weekday\")\n",
    "    #df['weekday'] = df.orderDate.apply(pd.to_datetime).apply(lambda x: x.weekday())\n",
    "    #df['wednesday'] = 0\n",
    "    #df['wednesday'][df.weekday==2] = 1\n",
    "\n",
    "    # Konversi data bertipe kategori/object ke numerik. Komen baris ini hingga blok for kalau tidak ingin konversi data bertipe kategori\n",
    "    # print(\"Konversi kategori/object ke numerik:\")\n",
    "    # Cari kolom yang tipenya object, bukan integer maupun float\n",
    "    # object_columns = df.loc[:, df.dtypes == object].columns\n",
    "    # for col in object_columns:\n",
    "    #     print(col)\n",
    "    #     le = LabelEncoder()\n",
    "    #     # Konversi deh\n",
    "    #     df[col] = le.fit_transform(df[col])\n",
    "\n",
    "    # Unused functions\n",
    "    # df['cSizeCode'] = df.sizeCode.apply(changeSizeCode)\n",
    "    # df['season'] = df.months.apply(getSeason)\n",
    "    # df['priceDiff'] = df['rrp'] - (df['price'] / df['quantity'])\n",
    "    # df['gpriceDiff'] = df['priceDiff'].apply(grouppriceDiff)\n",
    "\n",
    "    return df\n",
    "\n",
    "def main():\n",
    "    from datetime import datetime\n",
    "    print datetime.now()\n",
    "    train_df = pd.read_csv('orders_train.txt', sep=';')\n",
    "    train_df = preprocess(train_df)\n",
    "    train_df.to_csv('train_gue2.csv', index=False)\n",
    "    train_df.to_pickle('train_gue2.pkl')\n",
    "    print datetime.now()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}