Python75-xgboostx-lr的使用.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_curve,auc,roc_auc_score\n",
    "from sklearn.externals import joblib\n",
    "import numpy as np\n",
    "from scipy.sparse import hstack\n",
    "from sklearn.preprocessing.data import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost auc score:1.00000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:1228: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = -1.\n",
      "  \" = {}.\".format(self.n_jobs))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "基于特征组合的LR AUC：1.00000\n"
     ]
    }
   ],
   "source": [
    "def xgboost_lr_train(libsvmfile):\n",
    "    \n",
    "    # 加载样本数据\n",
    "    X_all,y_all=load_svmlight_file(libsvmfile)\n",
    "    \n",
    "    # 训练集和测试集分割\n",
    "    X_train,X_test,y_train,y_test=train_test_split(X_all,y_all,test_size=0.3,random_state=42)\n",
    "    \n",
    "    # 定义xgb模型\n",
    "    xgb_model=xgb.XGBClassifier(nthread=4,learning_rate=0.08,\n",
    "                                n_estimators=150,max_depth=5,gamma=0,subsample=0.9,colsample_bytree=0.5)\n",
    "    \n",
    "    # 训练xgb_model\n",
    "    xgb_model.fit(X_train,y_train)\n",
    "    \n",
    "    # 预测以及auc评测\n",
    "    y_pred=xgb_model.predict_proba(X_test)[:,1]\n",
    "    xgb_test_auc=roc_auc_score(y_test,y_pred)\n",
    "    print(\"xgboost auc score:%.5f\" % xgb_test_auc)\n",
    "    \n",
    "    # xgboost 编码原有特征\n",
    "    X_train_leaves=xgb_model.apply(X_train)\n",
    "    X_test_leaves=xgb_model.apply(X_test)\n",
    "    # 合并编码特征\n",
    "    All_leaves=np.concatenate((X_train_leaves,X_test_leaves),axis=0)\n",
    "    All_leaves=All_leaves.astype(np.int32)\n",
    "    \n",
    "    # 对所有特征进行one-hot编码\n",
    "    xgb_encoder=OneHotEncoder()\n",
    "    X_trans=xgb_encoder.fit_transform(All_leaves)\n",
    "   \n",
    "    # 定义LR模型\n",
    "    (train_rows, cols) = X_train_leaves.shape\n",
    "    lr=LogisticRegression(n_jobs=-1)\n",
    "    # 特征组合\n",
    "    X_train_ext=hstack([X_trans[:train_rows,:],X_train])\n",
    "    X_test_ext=hstack([X_trans[train_rows:,:],X_test])\n",
    "    \n",
    "    # 对特征组合进行训练\n",
    "    lr.fit(X_train_ext,y_train)\n",
    "    \n",
    "    # 预测以及AUC评测\n",
    "    y_pred_xgblr=lr.predict_proba(X_test_ext)[:,1]\n",
    "    xgb_lr_auc2=roc_auc_score(y_test,y_pred_xgblr)\n",
    "    print(\"基于特征组合的LR AUC：%.5f\" % xgb_lr_auc2)\n",
    "libsvmfile='files/data/python75/sample_libsvm_data.txt'\n",
    "xgboost_lr_train(libsvmfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[【实践】CTR中xgboost/gbdt +lr](https://blog.csdn.net/dengxing1234/article/details/73739836)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost test auc: 1.00000\n",
      "Transform features genenrated by xgboost...\n",
      "Write xgboost learned features to file ...\n",
      "基于原有特征的LR AUC: 1.00000\n",
      "基于Xgboost特征编码后的LR AUC: 0.97619\n",
      "基于组合特征的LR AUC: 0.64550\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:1228: UserWarning: 'n_jobs' > 1 does not have any effect when 'solver' is set to 'liblinear'. Got 'n_jobs' = -1.\n",
      "  \" = {}.\".format(self.n_jobs))\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "from sklearn.externals import joblib\n",
    "import numpy as np\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "\n",
    "\n",
    "def xgb_feature_encode(libsvmFileNameInitial):\n",
    "\n",
    "    # load样本数据\n",
    "    X_all, y_all = load_svmlight_file(libsvmFileNameInitial)\n",
    "\n",
    "    # 训练/测试数据分割\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size = 0.3, random_state = 42)\n",
    "\n",
    "    # 定义模型\n",
    "    xgboost = xgb.XGBClassifier(learning_rate=0.05,\n",
    "                            n_estimators=50, max_depth=3, gamma=0, subsample=0.7, colsample_bytree=0.7)\n",
    "    # 训练学习\n",
    "    xgboost.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "    # 预测及AUC评测\n",
    "    y_pred_test = xgboost.predict_proba(X_test)[:, 1]\n",
    "    xgb_test_auc = roc_auc_score(y_test, y_pred_test)\n",
    "    print('xgboost test auc: %.5f' % xgb_test_auc)\n",
    "\n",
    "    # xgboost编码原有特征\n",
    "    X_train_leaves = xgboost.apply(X_train)\n",
    "    X_test_leaves = xgboost.apply(X_test)\n",
    "    # 训练样本个数\n",
    "    train_rows = X_train_leaves.shape[0]\n",
    "    # 合并编码后的训练数据和测试数据\n",
    "    X_leaves = np.concatenate((X_train_leaves, X_test_leaves), axis=0)\n",
    "    X_leaves = X_leaves.astype(np.int32)\n",
    "\n",
    "    (rows, cols) = X_leaves.shape\n",
    "\n",
    "    # 记录每棵树的编码区间\n",
    "    cum_count = np.zeros((1, cols), dtype=np.int32)\n",
    "\n",
    "    for j in range(cols):\n",
    "        if j == 0:\n",
    "            cum_count[0][j] = len(np.unique(X_leaves[:, j]))\n",
    "        else:\n",
    "            cum_count[0][j] = len(np.unique(X_leaves[:, j])) + cum_count[0][j-1]\n",
    "\n",
    "    print('Transform features genenrated by xgboost...')\n",
    "    # 对所有特征进行ont-hot编码\n",
    "    for j in range(cols):\n",
    "        keyMapDict = {}\n",
    "        if j == 0:\n",
    "            initial_index = 1\n",
    "        else:\n",
    "            initial_index = cum_count[0][j-1]+1\n",
    "        for i in range(rows):\n",
    "            if X_leaves[i, j] not in keyMapDict:\n",
    "                keyMapDict[X_leaves[i, j]] = initial_index\n",
    "                X_leaves[i, j] = initial_index\n",
    "                initial_index = initial_index + 1\n",
    "            else:\n",
    "                X_leaves[i, j] = keyMapDict[X_leaves[i, j]]\n",
    "\n",
    "    # 基于编码后的特征，将特征处理为libsvm格式且写入文件\n",
    "    print('Write xgboost learned features to file ...')\n",
    "    xgbFeatureLibsvm = open('files/data/python75/xgb_feature_libsvm', 'w')\n",
    "    for i in range(rows):\n",
    "        if i < train_rows:\n",
    "            xgbFeatureLibsvm.write(str(y_train[i]))\n",
    "        else:\n",
    "            xgbFeatureLibsvm.write(str(y_test[i-train_rows]))\n",
    "        for j in range(cols):\n",
    "            xgbFeatureLibsvm.write(' '+str(X_leaves[i, j])+':1.0')\n",
    "        xgbFeatureLibsvm.write('\\n')\n",
    "    xgbFeatureLibsvm.close()\n",
    "\n",
    "\n",
    "def xgboost_lr_train(xgbfeaturefile, origin_libsvm_file):\n",
    "\n",
    "    # load xgboost特征编码后的样本数据\n",
    "    X_xg_all, y_xg_all = load_svmlight_file(xgbfeaturefile)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_xg_all, y_xg_all, test_size = 0.3, random_state = 42)\n",
    "\n",
    "    # load 原始样本数据\n",
    "    X_all, y_all = load_svmlight_file(origin_libsvm_file)\n",
    "    X_train_origin, X_test_origin, y_train_origin, y_test_origin = train_test_split(X_all, y_all, test_size = 0.3, random_state = 42)\n",
    "\n",
    "\n",
    "    # lr对原始特征样本模型训练\n",
    "    lr = LogisticRegression(n_jobs=-1, C=0.1, penalty='l1')\n",
    "    lr.fit(X_train_origin, y_train_origin)\n",
    "    joblib.dump(lr, 'files/data/python75/lr_orgin.m')\n",
    "    # 预测及AUC评测\n",
    "    y_pred_test = lr.predict_proba(X_test_origin)[:, 1]\n",
    "    lr_test_auc = roc_auc_score(y_test_origin, y_pred_test)\n",
    "    print('基于原有特征的LR AUC: %.5f' % lr_test_auc)\n",
    "\n",
    "    # lr对load xgboost特征编码后的样本模型训练\n",
    "    lr = LogisticRegression(n_jobs=-1, C=0.1, penalty='l1')\n",
    "    lr.fit(X_train, y_train)\n",
    "    joblib.dump(lr, 'files/data/python75/lr_xgb.m')\n",
    "    # 预测及AUC评测\n",
    "    y_pred_test = lr.predict_proba(X_test)[:, 1]\n",
    "    lr_test_auc = roc_auc_score(y_test, y_pred_test)\n",
    "    print('基于Xgboost特征编码后的LR AUC: %.5f' % lr_test_auc)\n",
    "\n",
    "    # 基于原始特征组合xgboost编码后的特征\n",
    "    X_train_ext = hstack([X_train_origin, X_train])\n",
    "    del(X_train)\n",
    "    del(X_train_origin)\n",
    "    X_test_ext = hstack([X_test_origin, X_test])\n",
    "    del(X_test)\n",
    "    del(X_test_origin)\n",
    "\n",
    "    # lr对组合后的新特征的样本进行模型训练\n",
    "    lr = LogisticRegression(n_jobs=-1, C=0.1, penalty='l1')\n",
    "    lr.fit(X_train_ext, y_train)\n",
    "    joblib.dump(lr, 'files/data/python75/lr_ext.m')\n",
    "    # 预测及AUC评测\n",
    "    y_pred_test = lr.predict_proba(X_test_ext)[:, 1]\n",
    "    lr_test_auc = roc_auc_score(y_test, y_pred_test)\n",
    "    print('基于组合特征的LR AUC: %.5f' % lr_test_auc)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    xgb_feature_encode(\"files/data/python75/sample_libsvm_data.txt\")\n",
    "    xgboost_lr_train(\"files/data/python75/xgb_feature_libsvm\",\n",
    "                     \"files/data/python75/sample_libsvm_data.txt\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}