Feature_Engineering_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "%matplotlib inline\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# bids = pd.read_csv('bids.csv')\n",
    "bids = pickle.load(open('bids.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7656329, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bid_id</th>\n",
       "      <th>bidder_id</th>\n",
       "      <th>auction</th>\n",
       "      <th>merchandise</th>\n",
       "      <th>device</th>\n",
       "      <th>time</th>\n",
       "      <th>country</th>\n",
       "      <th>ip</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>8dac2b259fd1c6d1120e519fb1ac14fbqvax8</td>\n",
       "      <td>ewmzr</td>\n",
       "      <td>jewelry</td>\n",
       "      <td>phone0</td>\n",
       "      <td>9759243157894736</td>\n",
       "      <td>us</td>\n",
       "      <td>69.166.231.58</td>\n",
       "      <td>vasstdc27m7nks3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>668d393e858e8126275433046bbd35c6tywop</td>\n",
       "      <td>aeqok</td>\n",
       "      <td>furniture</td>\n",
       "      <td>phone1</td>\n",
       "      <td>9759243157894736</td>\n",
       "      <td>in</td>\n",
       "      <td>50.201.125.84</td>\n",
       "      <td>jmqlhflrzwuay9c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>aa5f360084278b35d746fa6af3a7a1a5ra3xe</td>\n",
       "      <td>wa00e</td>\n",
       "      <td>home goods</td>\n",
       "      <td>phone2</td>\n",
       "      <td>9759243157894736</td>\n",
       "      <td>py</td>\n",
       "      <td>112.54.208.157</td>\n",
       "      <td>vasstdc27m7nks3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3939ac3ef7d472a59a9c5f893dd3e39fh9ofi</td>\n",
       "      <td>jefix</td>\n",
       "      <td>jewelry</td>\n",
       "      <td>phone4</td>\n",
       "      <td>9759243157894736</td>\n",
       "      <td>in</td>\n",
       "      <td>18.99.175.133</td>\n",
       "      <td>vasstdc27m7nks3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>8393c48eaf4b8fa96886edc7cf27b372dsibi</td>\n",
       "      <td>jefix</td>\n",
       "      <td>jewelry</td>\n",
       "      <td>phone5</td>\n",
       "      <td>9759243157894736</td>\n",
       "      <td>in</td>\n",
       "      <td>145.138.5.37</td>\n",
       "      <td>vasstdc27m7nks3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bid_id                              bidder_id auction merchandise  device  \\\n",
       "0       0  8dac2b259fd1c6d1120e519fb1ac14fbqvax8   ewmzr     jewelry  phone0   \n",
       "1       1  668d393e858e8126275433046bbd35c6tywop   aeqok   furniture  phone1   \n",
       "2       2  aa5f360084278b35d746fa6af3a7a1a5ra3xe   wa00e  home goods  phone2   \n",
       "3       3  3939ac3ef7d472a59a9c5f893dd3e39fh9ofi   jefix     jewelry  phone4   \n",
       "4       4  8393c48eaf4b8fa96886edc7cf27b372dsibi   jefix     jewelry  phone5   \n",
       "\n",
       "               time country              ip              url  \n",
       "0  9759243157894736      us   69.166.231.58  vasstdc27m7nks3  \n",
       "1  9759243157894736      in   50.201.125.84  jmqlhflrzwuay9c  \n",
       "2  9759243157894736      py  112.54.208.157  vasstdc27m7nks3  \n",
       "3  9759243157894736      in   18.99.175.133  vasstdc27m7nks3  \n",
       "4  9759243157894736      in    145.138.5.37  vasstdc27m7nks3  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print bids.shape\n",
    "display(bids.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidders = bids.groupby('bidder_id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 针对国家、商品单一特征多类别转换为多个独立特征进行统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cates = (bids['merchandise'].unique()).tolist()\n",
    "countries = (bids['country'].unique()).tolist()\n",
    "\n",
    "def dummy_coun_cate(group):\n",
    "    coun_cate = dict.fromkeys(cates, 0)\n",
    "    coun_cate.update(dict.fromkeys(countries, 0))\n",
    "    for cat, value in group['merchandise'].value_counts().iteritems():\n",
    "        coun_cate[cat] = value\n",
    "\n",
    "    for c in group['country'].unique():\n",
    "        coun_cate[c] = 1\n",
    "\n",
    "    coun_cate = pd.Series(coun_cate)\n",
    "    return coun_cate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bidder_coun_cate = bidders.apply(dummy_coun_cate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ad</th>\n",
       "      <th>ae</th>\n",
       "      <th>af</th>\n",
       "      <th>ag</th>\n",
       "      <th>al</th>\n",
       "      <th>am</th>\n",
       "      <th>an</th>\n",
       "      <th>ao</th>\n",
       "      <th>ar</th>\n",
       "      <th>at</th>\n",
       "      <th>...</th>\n",
       "      <th>vc</th>\n",
       "      <th>ve</th>\n",
       "      <th>vi</th>\n",
       "      <th>vn</th>\n",
       "      <th>ws</th>\n",
       "      <th>ye</th>\n",
       "      <th>za</th>\n",
       "      <th>zm</th>\n",
       "      <th>zw</th>\n",
       "      <th>zz</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "      <td>6609.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.002724</td>\n",
       "      <td>0.205629</td>\n",
       "      <td>0.054774</td>\n",
       "      <td>0.001059</td>\n",
       "      <td>0.048570</td>\n",
       "      <td>0.023907</td>\n",
       "      <td>0.000303</td>\n",
       "      <td>0.036314</td>\n",
       "      <td>0.120442</td>\n",
       "      <td>0.052655</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000605</td>\n",
       "      <td>0.033591</td>\n",
       "      <td>0.000303</td>\n",
       "      <td>0.130882</td>\n",
       "      <td>0.001967</td>\n",
       "      <td>0.040551</td>\n",
       "      <td>0.274474</td>\n",
       "      <td>0.067181</td>\n",
       "      <td>0.069753</td>\n",
       "      <td>0.000757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.052121</td>\n",
       "      <td>0.404191</td>\n",
       "      <td>0.227555</td>\n",
       "      <td>0.032530</td>\n",
       "      <td>0.214984</td>\n",
       "      <td>0.152770</td>\n",
       "      <td>0.017395</td>\n",
       "      <td>0.187085</td>\n",
       "      <td>0.325502</td>\n",
       "      <td>0.223362</td>\n",
       "      <td>...</td>\n",
       "      <td>0.024596</td>\n",
       "      <td>0.180186</td>\n",
       "      <td>0.017395</td>\n",
       "      <td>0.337297</td>\n",
       "      <td>0.044311</td>\n",
       "      <td>0.197262</td>\n",
       "      <td>0.446283</td>\n",
       "      <td>0.250354</td>\n",
       "      <td>0.254750</td>\n",
       "      <td>0.027497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 209 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                ad           ae           af           ag           al  \\\n",
       "count  6609.000000  6609.000000  6609.000000  6609.000000  6609.000000   \n",
       "mean      0.002724     0.205629     0.054774     0.001059     0.048570   \n",
       "std       0.052121     0.404191     0.227555     0.032530     0.214984   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "\n",
       "                am           an           ao           ar           at  \\\n",
       "count  6609.000000  6609.000000  6609.000000  6609.000000  6609.000000   \n",
       "mean      0.023907     0.000303     0.036314     0.120442     0.052655   \n",
       "std       0.152770     0.017395     0.187085     0.325502     0.223362   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "\n",
       "          ...                vc           ve           vi           vn  \\\n",
       "count     ...       6609.000000  6609.000000  6609.000000  6609.000000   \n",
       "mean      ...          0.000605     0.033591     0.000303     0.130882   \n",
       "std       ...          0.024596     0.180186     0.017395     0.337297   \n",
       "min       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "25%       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "50%       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "75%       ...          0.000000     0.000000     0.000000     0.000000   \n",
       "max       ...          1.000000     1.000000     1.000000     1.000000   \n",
       "\n",
       "                ws           ye           za           zm           zw  \\\n",
       "count  6609.000000  6609.000000  6609.000000  6609.000000  6609.000000   \n",
       "mean      0.001967     0.040551     0.274474     0.067181     0.069753   \n",
       "std       0.044311     0.197262     0.446283     0.250354     0.254750   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "75%       0.000000     0.000000     1.000000     0.000000     0.000000   \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "\n",
       "                zz  \n",
       "count  6609.000000  \n",
       "mean      0.000757  \n",
       "std       0.027497  \n",
       "min       0.000000  \n",
       "25%       0.000000  \n",
       "50%       0.000000  \n",
       "75%       0.000000  \n",
       "max       1.000000  \n",
       "\n",
       "[8 rows x 209 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(bidder_coun_cate.describe())\n",
    "bidder_coun_cate.to_csv('coun_cate.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "同样的，对于每个用户需要统计他对于自己每次竞拍行为的时间间隔情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bidder_interval(group):\n",
    "    time_diff = np.ediff1d(group['time'])\n",
    "    bidder_interval = {}\n",
    "    if len(time_diff) == 0:\n",
    "        diff_mean = 0\n",
    "        diff_std = 0\n",
    "        diff_median = 0\n",
    "        diff_zeros = 0\n",
    "    else:\n",
    "        diff_mean = np.mean(time_diff)\n",
    "        diff_std = np.std(time_diff)\n",
    "        diff_median = np.median(time_diff)\n",
    "        diff_zeros = time_diff.shape[0] - np.count_nonzero(time_diff)\n",
    "    bidder_interval['tmean'] = diff_mean\n",
    "    bidder_interval['tstd'] = diff_std\n",
    "    bidder_interval['tmedian'] = diff_median\n",
    "    bidder_interval['tzeros'] = diff_zeros\n",
    "    bidder_interval = pd.Series(bidder_interval)\n",
    "    return bidder_interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidder_inv = bidders.apply(bidder_interval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tmean</th>\n",
       "      <th>tmedian</th>\n",
       "      <th>tstd</th>\n",
       "      <th>tzeros</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6.609000e+03</td>\n",
       "      <td>6.609000e+03</td>\n",
       "      <td>6.609000e+03</td>\n",
       "      <td>6609.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.933038e+12</td>\n",
       "      <td>1.860285e+12</td>\n",
       "      <td>3.440901e+12</td>\n",
       "      <td>122.986231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.552343e+12</td>\n",
       "      <td>7.993497e+12</td>\n",
       "      <td>6.512992e+12</td>\n",
       "      <td>3190.805229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.192853e+10</td>\n",
       "      <td>2.578947e+09</td>\n",
       "      <td>1.749995e+09</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.641139e+11</td>\n",
       "      <td>5.726316e+10</td>\n",
       "      <td>5.510107e+11</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.847456e+12</td>\n",
       "      <td>6.339474e+11</td>\n",
       "      <td>2.911282e+12</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.610295e+13</td>\n",
       "      <td>7.610295e+13</td>\n",
       "      <td>3.800092e+13</td>\n",
       "      <td>231570.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tmean       tmedian          tstd         tzeros\n",
       "count  6.609000e+03  6.609000e+03  6.609000e+03    6609.000000\n",
       "mean   2.933038e+12  1.860285e+12  3.440901e+12     122.986231\n",
       "std    8.552343e+12  7.993497e+12  6.512992e+12    3190.805229\n",
       "min    0.000000e+00  0.000000e+00  0.000000e+00       0.000000\n",
       "25%    1.192853e+10  2.578947e+09  1.749995e+09       0.000000\n",
       "50%    2.641139e+11  5.726316e+10  5.510107e+11       0.000000\n",
       "75%    1.847456e+12  6.339474e+11  2.911282e+12       0.000000\n",
       "max    7.610295e+13  7.610295e+13  3.800092e+13  231570.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(bidder_inv.describe())\n",
    "bidder_inv.to_csv('bidder_inv.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 按照用户-拍卖场分组进一步分析\n",
    "之前的统计是按照用户进行分组，针对各个用户从整体上针对竞标行为统计其各项特征，下面根据拍卖场来对用户进一步细分，看一看每个用户在不同拍卖场的行为模式,类似上述按照用户分组来统计各个用户的各项特征，针对用户-拍卖场结对分组进行统计以下特征\n",
    "\n",
    "* 基本计数统计，针对各个用户在各个拍卖场统计设备、国家、ip、url、商品类别、竞标次数等特征的数目作为新的特征\n",
    "* 时间间隔统计：统计各个用户在各个拍卖场每次竞拍的时间间隔的 均值、方差、中位数和0值\n",
    "* 针对商品类别、国家进一步转化为多类别进行统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auc_features_count(group):\n",
    "    time_diff = np.ediff1d(group['time'])\n",
    "    \n",
    "    if len(time_diff) == 0:\n",
    "        diff_mean = 0\n",
    "        diff_std = 0\n",
    "        diff_median = 0\n",
    "        diff_zeros = 0\n",
    "    else:\n",
    "        diff_mean = np.mean(time_diff)\n",
    "        diff_std = np.std(time_diff)\n",
    "        diff_median = np.median(time_diff)\n",
    "        diff_zeros = time_diff.shape[0] - np.count_nonzero(time_diff)\n",
    "\n",
    "    row = dict.fromkeys(cates, 0)\n",
    "    row.update(dict.fromkeys(countries, 0))\n",
    "\n",
    "    row['devices_c'] = group['device'].unique().shape[0]\n",
    "    row['countries_c'] = group['country'].unique().shape[0]\n",
    "    row['ip_c'] = group['ip'].unique().shape[0]\n",
    "    row['url_c'] = group['url'].unique().shape[0]\n",
    "#     row['merch_c'] = group['merchandise'].unique().shape[0]\n",
    "    row['bids_c'] = group.shape[0]\n",
    "    row['tmean'] = diff_mean\n",
    "    row['tstd'] = diff_std\n",
    "    row['tmedian'] = diff_median\n",
    "    row['tzeros'] = diff_zeros\n",
    "\n",
    "    for cat, value in group['merchandise'].value_counts().iteritems():\n",
    "        row[cat] = value\n",
    "\n",
    "    for c in group['country'].unique():\n",
    "        row[c] = 1\n",
    "\n",
    "    row = pd.Series(row)\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bidder_auc = bids.groupby(['bidder_id', 'auction']).apply(auc_features_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidder_auc.to_csv('bids_auc.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(382336, 218)\n"
     ]
    }
   ],
   "source": [
    "print bidder_auc.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}