allnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from  sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = np.load('../feature/nn_train.npy')\n",
    "test = np.load('../feature/nn_test.npy')\n",
    "train_weight = np.load('../feature/nn_weight_train.npy')\n",
    "test_weight = np.load('../feature/nn_weight_test.npy')\n",
    "train_person = np.load('../feature/nn_person_train.npy')\n",
    "test_person = np.load('../feature/nn_person_test.npy')\n",
    "train_mat1 = np.load('../feature/nn_mat1_train.npy')\n",
    "test_mat1 = np.load('../feature/nn_mat1_test.npy')\n",
    "train_mat2 = np.load('../feature/nn_mat2_train.npy')\n",
    "test_mat2 = np.load('../feature/nn_mat2_test.npy')\n",
    "# train_mat3 = np.load('../feature/nn_mat5_train.npy')\n",
    "# test_mat3 = np.load('../feature/nn_mat5_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = train[0:10000]\n",
    "# test = test[0:10000]\n",
    "# train_weight = train_weight[0:10000]\n",
    "# test_weight = test_weight[0:10000]\n",
    "# train_person = train_person[0:10000]\n",
    "# test_person = test_person[0:10000]\n",
    "# train_mat1 = train_mat1[0:10000]\n",
    "# test_mat1 = test_mat1[0:10000]\n",
    "# train_mat2 = train_mat2[0:10000]\n",
    "# test_mat2 = test_mat2[0:10000]\n",
    "# train_mat3 = train_mat3[0:10000]\n",
    "# test_mat3 = test_mat3[0:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2010000, 28)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_mat2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mat = np.concatenate([train_mat1],1)\n",
    "test_mat = np.concatenate([test_mat1],1)\n",
    "del train_mat1\n",
    "del test_mat1\n",
    "train_person = np.concatenate([train_person,train_mat2],1)\n",
    "test_person = np.concatenate([test_person,test_mat2],1)\n",
    "del train_mat2\n",
    "del test_mat2\n",
    "# del train_mat3\n",
    "# del test_mat3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaler = preprocessing.StandardScaler().fit(train_person)\n",
    "# train_person = scaler.transform(train_person) \n",
    "# test_person = scaler.transform(test_person)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_weight = train_weight.reshape(len(train),45)\n",
    "# test_weight = test_weight.reshape(len(test),45)\n",
    "# scaler = preprocessing.StandardScaler().fit(train_weight)\n",
    "# train_weight = scaler.transform(train_weight) \n",
    "# test_weight = scaler.transform(test_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_weight = train_weight.reshape(len(train),15,3)\n",
    "test_weight = test_weight.reshape(len(test),15,3)\n",
    "\n",
    "train_person = train_person.reshape(len(train),1,-1)\n",
    "test_person = test_person.reshape(len(test),1,-1)\n",
    "\n",
    "\n",
    "train_mat = train_mat.reshape(len(train),1,-1)\n",
    "test_mat = test_mat.reshape(len(test),1,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_train_data = pd.read_csv('../data/age_train.csv',names=['uid','age_group'])\n",
    "age_train_data['age_group'] = age_train_data['age_group'].apply(lambda x:x-1)\n",
    "y_train = age_train_data['age_group'].values\n",
    "\n",
    "# y_train = y_train[0:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=2018)\n",
    "from  sklearn import preprocessing\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "for i in range(train.shape[2]):\n",
    "    scaler = preprocessing.StandardScaler().fit(train[:,:,i])\n",
    "    train[:,:,i] = scaler.transform(train[:,:,i]) \n",
    "    test[:,:,i] = scaler.transform(test[:,:,i])\n",
    "#     train[:,:,i] = min_max_scaler.fit_transform(train[:,:,i])\n",
    "#     test[:,:,i] = min_max_scaler.transform(test[:,:,i])\n",
    "for i in range(train_weight.shape[2]):\n",
    "    scaler = preprocessing.StandardScaler().fit(train_weight[:,:,i])\n",
    "    train_weight[:,:,i] = scaler.transform(train_weight[:,:,i]) \n",
    "    test_weight[:,:,i] = scaler.transform(test_weight[:,:,i])\n",
    "#     train_weight[:,:,i] = min_max_scaler.fit_transform(train_weight[:,:,i])\n",
    "#     test_weight[:,:,i] = min_max_scaler.transform(test_weight[:,:,i])\n",
    "for i in range(train_person.shape[2]):\n",
    "    scaler = preprocessing.StandardScaler().fit(train_person[:,:,i])\n",
    "    train_person[:,:,i] = scaler.transform(train_person[:,:,i]) \n",
    "    test_person[:,:,i] = scaler.transform(test_person[:,:,i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader,Dataset\n",
    "class Getdata(Dataset):\n",
    "    def __init__(self,feature,weight,person,mat1,labels,transform=preprocessing.normalize):\n",
    "        self.feature = feature\n",
    "        self.weight = weight\n",
    "        self.person = person\n",
    "        self.mat1 = mat1\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "    def __len__(self):\n",
    "        return len(self.feature)\n",
    "    def __getitem__(self, item):\n",
    "        sample={}\n",
    "        if self.labels is not None:\n",
    "            sample['label'] = self.labels[item]\n",
    "            sample['feature'] = self.transform(self.feature[item],norm='l2')\n",
    "            sample['weight'] = self.transform(self.weight[item],norm='l2')\n",
    "            sample['person'] = self.transform(self.person[item],norm='l2')\n",
    "            sample['mat1'] = self.mat1[item]\n",
    "        return sample\n",
    "\n",
    "class Gettestdata(Dataset):\n",
    "    def __init__(self,feature,weight,person,mat1,transform=preprocessing.normalize):\n",
    "        self.feature = feature\n",
    "        self.weight = weight\n",
    "        self.person = person\n",
    "        self.mat1 = mat1\n",
    "        self.transform = transform\n",
    "    def __len__(self):\n",
    "        return len(self.feature)\n",
    "    def __getitem__(self, item):\n",
    "        sample={}\n",
    "        if self.feature is not None:\n",
    "            sample['feature'] = self.transform(self.feature[item],norm='l2')\n",
    "            sample['weight'] = self.transform(self.weight[item],norm='l2')\n",
    "            sample['person'] = self.transform(self.person[item],norm='l2')\n",
    "            sample['mat1'] = self.mat1[item]\n",
    "        return sample\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "class Mymodel(nn.Module):\n",
    "    def __init__(self,hidden_dim1=666,hidden_dim2=100,hidden_dim3=6):\n",
    "        super(Mymodel,self).__init__()\n",
    "        self.mat1cnn = nn.Sequential(*[\n",
    "            nn.Conv2d(1,16,(1,3),stride=1),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(),\n",
    "#             nn.MaxPool2d(1,1),\n",
    "            nn.Conv2d(16,32,(1,3),stride=3),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "#             nn.MaxPool2d(1,1),\n",
    "            nn.Conv2d(32,64,(1,5),stride=5),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "#             nn.MaxPool2d(1,1),\n",
    "        ])\n",
    "        self.modelperson = nn.Sequential(*[\n",
    "            nn.Linear(1856, hidden_dim1),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_dim1, hidden_dim2),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_dim2, hidden_dim3),\n",
    "            nn.ReLU(),\n",
    "        ])\n",
    "        self.personcnn = nn.Sequential(*[\n",
    "            nn.Conv2d(1,16,(1,3),stride=1),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(),\n",
    "#             nn.MaxPool2d(1,1),\n",
    "            nn.Conv2d(16,32,(1,3),stride=3),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "#             nn.MaxPool2d(1,1),      \n",
    "        ])\n",
    "        self.featurecnn = nn.Sequential(*[\n",
    "            nn.Conv2d(1,16,2,2),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(1,1),\n",
    "            nn.Conv2d(16,32,3,3),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(1,1),\n",
    "        ])\n",
    "    def forward(self,fea_weight,person,mat1):\n",
    "        fea_weight = fea_weight.float()\n",
    "        person = person.float()\n",
    "        person = person.unsqueeze(1)\n",
    "        mat1 = mat1.float()\n",
    "        mat1 = self.mat1cnn(mat1)\n",
    "        mat1 = mat1.view(mat1.shape[0],-1)\n",
    "        fea_weight = fea_weight.unsqueeze(1)\n",
    "        fea_weight = self.featurecnn(fea_weight)\n",
    "        fea_weight = fea_weight.view(fea_weight.shape[0],-1)\n",
    "        person = self.personcnn(person)\n",
    "        person = person.view(person.shape[0],-1)\n",
    "        allfeature = torch.cat((mat1,fea_weight,person),1)\n",
    "        end = self.modelperson(allfeature)\n",
    "        return end\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cpu\")\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25125/25125 [16:47<00:00, 24.93it/s]\n",
      "  0%|          | 1/25125 [00:00<54:49,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6050149253731343\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 187/25125 [00:07<17:10, 24.20it/s]ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-11-2fbcf1972322>\", line 26, in <module>\n",
      "    for _,sample in tqdm(enumerate(train_loaders),total=len(train_loaders)):\n",
      "  File \"/home/lzy/.local/lib/python3.6/site-packages/tqdm/_tqdm.py\", line 933, in __iter__\n",
      "    for obj in iterable:\n",
      "  File \"/home/lzy/.local/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 615, in __next__\n",
      "    batch = self.collate_fn([self.dataset[i] for i in indices])\n",
      "  File \"/home/lzy/.local/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 615, in <listcomp>\n",
      "    batch = self.collate_fn([self.dataset[i] for i in indices])\n",
      "  File \"<ipython-input-8-4341f8ee5248>\", line 17, in __getitem__\n",
      "    sample['weight'] = self.transform(self.weight[item],norm='l2')\n",
      "  File \"/home/lzy/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py\", line 1412, in normalize\n",
      "    estimator='the normalize function', dtype=FLOAT_DTYPES)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\", line 1823, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3/dist-packages/IPython/core/ultratb.py\", line 1132, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/usr/lib/python3/dist-packages/IPython/core/ultratb.py\", line 313, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/usr/lib/python3/dist-packages/IPython/core/ultratb.py\", line 370, in _fixed_getinnerframes\n",
      "    aux = traceback.extract_tb(etb)\n",
      "  File \"/usr/lib/python3.6/traceback.py\", line 72, in extract_tb\n",
      "    return StackSummary.extract(walk_tb(tb), limit=limit)\n",
      "  File \"/usr/lib/python3.6/traceback.py\", line 360, in extract\n",
      "    linecache.checkcache(filename)\n",
      "  File \"/usr/lib/python3/dist-packages/IPython/core/compilerop.py\", line 141, in check_linecache_ipython\n",
      "    linecache._checkcache_ori(*args)\n",
      "  File \"/usr/lib/python3.6/linecache.py\", line 53, in checkcache\n",
      "    def checkcache(filename=None):\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from torch.autograd import Variable\n",
    "from sklearn.metrics import accuracy_score\n",
    "np.random.seed(666)\n",
    "torch.manual_seed(666)\n",
    "torch.cuda.manual_seed_all(666)\n",
    "oof = np.zeros(len(train))\n",
    "predictions = np.zeros(len(test))\n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(train, y_train)):\n",
    "    print(fold_)\n",
    "    if fold_==1:\n",
    "        break\n",
    "    model = Mymodel()\n",
    "    model = model.to(device)\n",
    "    train_loaders = DataLoader(Getdata(train[trn_idx],train_weight[trn_idx],\n",
    "                                       train_person[trn_idx],train_mat[trn_idx],y_train[trn_idx]),batch_size=64,shuffle=True)\n",
    "    val_loaders = DataLoader(Getdata(train[val_idx],train_weight[val_idx],\n",
    "                                      train_person[val_idx],train_mat[val_idx],y_train[val_idx]),batch_size=64,shuffle=True)\n",
    "    test_loaders = DataLoader(Gettestdata(test,test_weight,test_person,test_mat),batch_size=64)\n",
    "    n_ep = 4\n",
    "    wd = 1e-5\n",
    "    lr = 1e-4\n",
    "    criterion =nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), weight_decay=wd, lr=lr)\n",
    "    for i_ep in range(n_ep):\n",
    "        for _,sample in tqdm(enumerate(train_loaders),total=len(train_loaders)):\n",
    "            feature = sample['feature'].to(device)\n",
    "#             feature = feature.unsqueeze(1)\n",
    "            weight = sample['weight'].to(device)\n",
    "            fea_weight = torch.cat((feature,weight),2)\n",
    "            person = sample['person'].to(device)\n",
    "#             person = person.unsqueeze(1)\n",
    "            mat1 = sample['mat1'].to(device)\n",
    "            mat1 = mat1.unsqueeze(1)\n",
    "            label = sample['label'].to(device)\n",
    "            optimizer.zero_grad()\n",
    "            result = model.forward(fea_weight,person,mat1)\n",
    "            loss = criterion(result,label)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        with torch.no_grad():\n",
    "            valmodel = model.eval()\n",
    "            losses = []\n",
    "            labels = []\n",
    "            preds = []\n",
    "            for _,sample_val in enumerate(val_loaders):\n",
    "                feature_val = sample_val['feature'].to(device)\n",
    "#                 feature_val = feature_val.unsqueeze(1)\n",
    "                weight_val = sample_val['weight'].to(device)\n",
    "                fea_weight_val = torch.cat((feature_val,weight_val),2)\n",
    "                person_val = sample_val['person'].to(device)\n",
    "#                 person_val = person_val.unsqueeze(1)\n",
    "                mat1_val = sample_val['mat1'].to(device)\n",
    "                mat1_val = mat1_val.unsqueeze(1)\n",
    "                label_val = sample_val['label'].to(device)\n",
    "                result_val = valmodel.forward(fea_weight_val,person_val,mat1_val)\n",
    "#                 print(np.argmax(result_test.numpy(),axis=1))\n",
    "                preds += [np.argmax(result_val.detach().cpu().numpy(),axis=1)]\n",
    "                labels += [label_val.detach().cpu().numpy()]\n",
    "            preds = np.concatenate(preds, 0)\n",
    "            labels = np.concatenate(labels, 0)\n",
    "            print(accuracy_score(labels,preds))\n",
    "    with torch.no_grad():\n",
    "        testmodel = model.eval()\n",
    "        preds_test = []\n",
    "        for _,sample_test in tqdm(enumerate(test_loaders),total=len(test_loaders)):\n",
    "            feature_test = sample_test['feature'].to(device)\n",
    "#             feature_test = feature_test.unsqueeze(1)\n",
    "            weight_test = sample_test['weight'].to(device)\n",
    "            fea_weight_test = torch.cat((feature_test,weight_test),2)\n",
    "            person_test = sample_test['person'].to(device)\n",
    "#             person_test = person_test.unsqueeze(1)\n",
    "            mat1_test = sample_test['mat1'].to(device)\n",
    "            mat1_test = mat1_test.unsqueeze(1)\n",
    "            result_test = testmodel.forward(fea_weight_test,person_test,mat1_test)\n",
    "            preds_test += [np.argmax(result_test.detach().cpu().numpy(),axis=1)]\n",
    "        predictions = np.concatenate(preds_test, 0)\n",
    "        print(predictions)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_mat\n",
    "del test_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(502500,)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_test_data = pd.read_csv('../data/age_test.csv',names=['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_test_data['label'] = predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_test_data['label'] = age_test_data['label'].apply(lambda x:round(x+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_test_data[['id','label']].to_csv('../submit/submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}