p3d_train_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "import os\n",
    "import random\n",
    "import PIL.Image as Image\n",
    "import numpy as np\n",
    "import cv2\n",
    "import time\n",
    "import math\n",
    "import sys\n",
    "from torch.autograd import Variable\n",
    "from functools import partial\n",
    "from __future__ import print_function\n",
    "from torch.utils.data import DataLoader\n",
    "from logging import Logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv_S(in_planes,out_planes,stride=1,padding=1):\n",
    "    # as is descriped, conv S is 1x3x3\n",
    "    return nn.Conv3d(in_planes,out_planes,kernel_size=(1,3,3),stride=1,\n",
    "                     padding=padding,bias=False)\n",
    "\n",
    "def conv_T(in_planes,out_planes,stride=1,padding=1):\n",
    "    # conv T is 3x1x1\n",
    "    return nn.Conv3d(in_planes,out_planes,kernel_size=(3,1,1),stride=1,\n",
    "                     padding=padding,bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def downsample_basic_block(x, planes, stride):\n",
    "    out = F.avg_pool3d(x, kernel_size=1, stride=stride)\n",
    "    zero_pads = torch.Tensor(out.size(0), planes - out.size(1),\n",
    "                             out.size(2), out.size(3),\n",
    "                             out.size(4)).zero_()\n",
    "    if isinstance(out.data, torch.cuda.FloatTensor):\n",
    "        zero_pads = zero_pads.cuda()\n",
    "\n",
    "    out = Variable(torch.cat([out.data, zero_pads], dim=1))\n",
    "\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Bottleneck(nn.Module):\n",
    "    expansion=4\n",
    "    def __init__(self,inplanes,planes,stride=1,downsample=None,n_s=0,depth_3d=47,ST_struc=('A','B','C')):\n",
    "        super(Bottleneck,self).__init__()\n",
    "        self.downsample=downsample\n",
    "        self.depth_3d=depth_3d\n",
    "        self.ST_struc=ST_struc\n",
    "        self.len_ST=len(self.ST_struc)\n",
    "        stride_p=stride\n",
    "        if not self.downsample == None:\n",
    "            stride_p=(1,2,2)\n",
    "        if n_s<self.depth_3d:\n",
    "            if n_s==0:\n",
    "                stride_p=1\n",
    "            self.conv1=nn.Conv3d(inplanes,planes,kernel_size=1,bias=False,stride=stride_p)\n",
    "            self.bn1=nn.BatchNorm3d(planes)\n",
    "        else:\n",
    "            if n_s==self.depth_3d:\n",
    "                stride_p=2\n",
    "            else:\n",
    "                stride_p=1\n",
    "            self.conv1=nn.Conv2d(inplanes,planes,kernel_size=1,bias=False,stride=stride_p)\n",
    "            self.bn1=nn.BatchNorm2d(planes)\n",
    "        self.id=n_s\n",
    "        self.ST=list(self.ST_struc)[self.id % self.len_ST]\n",
    "        if self.id<self.depth_3d:\n",
    "            self.conv2=conv_S(planes,planes,stride=1,padding=(0,1,1))\n",
    "            self.bn2=nn.BatchNorm3d(planes)\n",
    "            self.conv3=conv_T(planes,planes,stride=1,padding=(1,0,0))\n",
    "            self.bn3=nn.BatchNorm3d(planes)\n",
    "        else:\n",
    "            self.conv_normal=nn.Conv2d(planes,planes,kernel_size=3,stride=1,padding=1,bias=False)\n",
    "            self.bn_normal=nn.BatchNorm2d(planes)\n",
    "        \n",
    "        if n_s<self.depth_3d:\n",
    "            self.conv4=nn.Conv3d(planes,planes*4,kernel_size=1,bias=False)\n",
    "            self.bn4=nn.BatchNorm3d(planes *4 )\n",
    "        else:\n",
    "            self.conv4=nn.Conv2d(planes,planes*4,kernel_size=1,bias=False)\n",
    "            self.bn4=nn.BatchNorm2d(planes *4 )\n",
    "        self.relu=nn.ReLU(inplace=True)\n",
    "        self.stride=stride\n",
    "    \n",
    "    def ST_A(self,x):\n",
    "        x=self.conv2(x)\n",
    "        x=self.bn2(x)\n",
    "        x=self.relu(x)\n",
    "        x=self.conv3(x)\n",
    "        x=self.bn3(x)\n",
    "        x=self.relu(x)\n",
    "        return x\n",
    "    def ST_B(self,x):\n",
    "        tmp_x=self.conv2(x)\n",
    "        tmp_x=self.bn2(tmp_x)\n",
    "        tmp_x=self.relu(tmp_x)\n",
    "        x=self.conv3(x)\n",
    "        x=self.bn3(x)\n",
    "        x=self.relu(x)\n",
    "        return x+tmp_x\n",
    "    def ST_C(self,x):\n",
    "        x=self.conv2(x)\n",
    "        x=self.bn2(x)\n",
    "        x=self.relu(x)\n",
    "        tmp_x=self.conv3(x)\n",
    "        tmp_x=self.bn3(tmp_x)\n",
    "        tmp_x=self.relu(tmp_x)\n",
    "        return x+tmp_x\n",
    "    def forward(self,x):\n",
    "        residual=x\n",
    "        out=self.conv1(x)\n",
    "        out=self.bn1(out)\n",
    "        out=self.relu(out)\n",
    "        \n",
    "        if self.id<self.depth_3d:\n",
    "            if self.ST=='A':\n",
    "                out=self.ST_A(out)\n",
    "            elif self.ST=='B':\n",
    "                out=self.ST_B(out)\n",
    "            elif self.ST=='C':\n",
    "                out=self.ST_C(out)\n",
    "        else:\n",
    "            out=self.conv_normal(out)\n",
    "            out=self.bn_normal(out)\n",
    "            out=self.relu(out)\n",
    "        out=self.conv4(out)\n",
    "        out=self.bn4(out)\n",
    "        if self.downsample is not None:\n",
    "            residual=self.downsample(x)\n",
    "        out+=residual\n",
    "        out=self.relu(out)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class P3D(nn.Module):\n",
    "\n",
    "    def __init__(self, block, layers, modality='RGB',\n",
    "        shortcut_type='B', num_classes=3,dropout=0.5,ST_struc=('A','B','C')):\n",
    "        print('num_classes=',num_classes)\n",
    "        print('dropout=',dropout)\n",
    "        self.inplanes = 64\n",
    "        super(P3D, self).__init__()\n",
    "        # self.conv1 = nn.Conv3d(3, 64, kernel_size=7, stride=(1, 2, 2),\n",
    "        #                        padding=(3, 3, 3), bias=False)\n",
    "        self.input_channel = 3 if modality=='RGB' else 2  # 2 is for flow \n",
    "        self.ST_struc=ST_struc\n",
    "\n",
    "        self.conv1_custom = nn.Conv3d(self.input_channel, 64, kernel_size=(1,7,7), stride=(1,2,2),\n",
    "                                padding=(0,3,3), bias=False)\n",
    "\n",
    "        self.depth_3d=sum(layers[:3])# C3D layers are only (res2,res3,res4),  res5 is C2D\n",
    "\n",
    "        self.bn1 = nn.BatchNorm3d(64) # bn1 is followed by conv1\n",
    "        self.cnt=0\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.maxpool = nn.MaxPool3d(kernel_size=(2, 3, 3), stride=2, padding=(0,1,1))       # pooling layer for conv1.\n",
    "        self.maxpool_2 = nn.MaxPool3d(kernel_size=(2,1,1),padding=0,stride=(2,1,1))   # pooling layer for res2, 3, 4.\n",
    "\n",
    "        self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type)\n",
    "        self.layer2 = self._make_layer(block, 128, layers[1], shortcut_type, stride=2)\n",
    "        self.layer3 = self._make_layer(block, 256, layers[2], shortcut_type, stride=2)\n",
    "        self.layer4 = self._make_layer(block, 512, layers[3], shortcut_type, stride=2)\n",
    "\n",
    "        self.avgpool = nn.AvgPool2d(kernel_size=(5, 5), stride=1)                              # pooling layer for res5.\n",
    "        self.dropout=nn.Dropout(p=dropout)\n",
    "        self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv3d):\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "            elif isinstance(m, nn.BatchNorm3d):\n",
    "                m.weight.data.fill_(1)\n",
    "                m.bias.data.zero_()\n",
    "\n",
    "        # some private attribute\n",
    "#         self.input_size=(self.input_channel,16,160,160)       # input of the network\n",
    "#         self.input_mean = [0.485, 0.456, 0.406] if modality=='RGB' else [0.5]\n",
    "#         self.input_std = [0.229, 0.224, 0.225] if modality=='RGB' else [np.mean([0.229, 0.224, 0.225])]\n",
    "\n",
    "\n",
    "    @property\n",
    "    def scale_size(self):\n",
    "        return self.input_size[2] * 256 // 160   # asume that raw images are resized (340,256).\n",
    "\n",
    "    @property\n",
    "    def temporal_length(self):\n",
    "        return self.input_size[1]\n",
    "\n",
    "    @property\n",
    "    def crop_size(self):\n",
    "        return self.input_size[2]\n",
    "\n",
    "    def _make_layer(self, block, planes, blocks, shortcut_type, stride=1):\n",
    "        downsample = None\n",
    "        stride_p=stride #especially for downsample branch.\n",
    "\n",
    "        if self.cnt<self.depth_3d:\n",
    "            if self.cnt==0:\n",
    "                stride_p=1\n",
    "            else:\n",
    "                stride_p=(1,2,2)\n",
    "            if stride != 1 or self.inplanes != planes * block.expansion:\n",
    "                if shortcut_type == 'A':\n",
    "                    downsample = partial(downsample_basic_block,\n",
    "                                         planes=planes * block.expansion,\n",
    "                                         stride=stride)\n",
    "                else:\n",
    "                    \n",
    "                    downsample = nn.Sequential(\n",
    "                        nn.Conv3d(self.inplanes, planes * block.expansion,\n",
    "                                  kernel_size=1, stride=stride_p, bias=False),\n",
    "                        nn.BatchNorm3d(planes * block.expansion)\n",
    "                    )\n",
    "\n",
    "        else:\n",
    "            if stride != 1 or self.inplanes != planes * block.expansion:\n",
    "                if shortcut_type == 'A':\n",
    "                    downsample = partial(downsample_basic_block,\n",
    "                                         planes=planes * block.expansion,\n",
    "                                         stride=stride)\n",
    "                else:\n",
    "                    \n",
    "                    downsample = nn.Sequential(\n",
    "                        nn.Conv2d(self.inplanes, planes * block.expansion,\n",
    "                                  kernel_size=1, stride=2, bias=False),\n",
    "                        nn.BatchNorm2d(planes * block.expansion)\n",
    "                    )\n",
    "        layers = []\n",
    "        layers.append(block(self.inplanes, planes, stride, downsample,n_s=self.cnt,depth_3d=self.depth_3d,ST_struc=self.ST_struc))\n",
    "        self.cnt+=1\n",
    "        \n",
    "        self.inplanes = planes * block.expansion\n",
    "        for i in range(1, blocks):\n",
    "            layers.append(block(self.inplanes, planes,n_s=self.cnt,depth_3d=self.depth_3d,ST_struc=self.ST_struc))\n",
    "            self.cnt+=1\n",
    "\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        x = self.conv1_custom(x) #-1\n",
    "\n",
    "        x = self.bn1(x) #-1\n",
    "\n",
    "        x = self.relu(x) #-1\n",
    "\n",
    "        x = self.maxpool(x) #-1\n",
    "\n",
    "        \n",
    "        x = self.maxpool_2(self.layer1(x))  #   -1  -            Part Res2\n",
    "\n",
    "        \n",
    "        x = self.maxpool_2(self.layer2(x))  #   -1               Part Res3\n",
    "\n",
    "        \n",
    "        x = self.maxpool_2(self.layer3(x))  #      -1              Part Res4\n",
    "\n",
    "\n",
    "        sizes=x.size()\n",
    "        \n",
    "       \n",
    "        x = x.view(-1,sizes[1],sizes[3],sizes[4])  #  Part Res5\n",
    "        \n",
    "        x = self.layer4(x)\n",
    "\n",
    "        x = self.avgpool(x)\n",
    "\n",
    "        x = x.view(-1,self.fc.in_features)\n",
    "\n",
    "        x = self.fc(self.dropout(x))\n",
    "\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_optim_policies(model=None,modality='RGB',enable_pbn=True):\n",
    "    first_conv_weight=[]\n",
    "    first_conv_bias=[]\n",
    "    normal_weight=[]\n",
    "    normal_bias=[]\n",
    "    bn=[]\n",
    "    if model==None:\n",
    "        log.l.info('no model')\n",
    "        exit()\n",
    "    conv_cnt=0\n",
    "    bn_cnt=0\n",
    "    for m in model.modules():\n",
    "        if isinstance(m,torch.nn.Conv3d) or isinstance(m,torch.nn.Conv2d):\n",
    "            ps=list(m.parameters())\n",
    "            conv_cnt+=1\n",
    "            if conv_cnt==1:\n",
    "                first_conv_weight.append(ps[0])\n",
    "                if len(ps)==2:\n",
    "                    first_conv_bias.append(ps[1])\n",
    "            else:\n",
    "                normal_weight.append(ps[0])\n",
    "                if len(ps)==2:\n",
    "                    normal_bias.append(ps[1])\n",
    "        elif isinstance(m,torch.nn.Linear):\n",
    "            ps=list(m.parameters())\n",
    "            normal_weight.append(ps[0])\n",
    "            if len(ps)==2:\n",
    "                normal_bias.append(ps[1])\n",
    "        elif isinstance(m,torch.nn.BatchNorm3d):\n",
    "            bn_cnt+=1\n",
    "            if not enable_pbn or bn_cnt==1:\n",
    "                bn.extend(list(m.parameters()))\n",
    "        elif isinstance(m,torch.nn.BatchNorm2d):\n",
    "            bn.extend(list(m.parameters()))\n",
    "        elif len(m._modules)==0:\n",
    "            if len(list(m.parameters()))>0:\n",
    "                raise ValueError(\"{}. Need to give it a learning policy\".format(type(m)))\n",
    "    slow_rate=0.7\n",
    "    n_fore=int(len(normal_weight)*slow_rate)\n",
    "    slow_feat=normal_weight[:n_fore]\n",
    "    slow_bias=normal_bias[:n_fore]\n",
    "    normal_feat=normal_weight[n_fore:]\n",
    "    normal_bias=normal_bias[n_fore:]\n",
    "    return [\n",
    "        {'params': first_conv_weight, 'lr_mult': 5 if modality == 'Flow' else 1, 'decay_mult': 1,\n",
    "         'name': \"first_conv_weight\"},\n",
    "        {'params': first_conv_bias, 'lr_mult': 10 if modality == 'Flow' else 2, 'decay_mult': 0,\n",
    "         'name': \"first_conv_bias\"},\n",
    "        {'params': slow_feat, 'lr_mult': 1, 'decay_mult': 1,\n",
    "         'name': \"slow_feat\"},\n",
    "        {'params': slow_bias, 'lr_mult': 2, 'decay_mult': 0,\n",
    "         'name': \"slow_bias\"},\n",
    "        {'params': normal_feat, 'lr_mult': 1 , 'decay_mult': 1,\n",
    "         'name': \"normal_feat\"},\n",
    "        {'params': normal_bias, 'lr_mult': 2, 'decay_mult':0,\n",
    "         'name': \"normal_bias\"},\n",
    "        {'params': bn, 'lr_mult': 1, 'decay_mult': 0,\n",
    "         'name': \"BN scale/shift\"},\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def P3D199(pretrained=False,modality='RGB',**kwargs):\n",
    "    \"\"\"construct a P3D199 model based on a ResNet-152-3D model.\n",
    "    \"\"\"\n",
    "    model = P3D(Bottleneck, [3, 8, 36, 3], modality=modality,**kwargs)\n",
    "    \n",
    "    if pretrained==True:\n",
    "        if modality=='RGB':\n",
    "            pretrained_file='p3d_rgb_199.checkpoint.pth.tar'\n",
    "        elif modality=='Flow':\n",
    "            pretrained_file='p3d_flow_199.checkpoint.pth.tar'\n",
    "        weights=torch.load(pretrained_file)['state_dict']\n",
    "        model.load_state_dict(weights)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = P3D199(pretrained=False,num_classes=3,dropout=0.5)\n",
    "model=model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#utils function for preparing our datasets.\n",
    "def get_frames_data(filename,num_frames_per_clip=16):\n",
    "    ret_arr=[]\n",
    "    s_index=0\n",
    "    for parent, dirnames,filenames in os.walk(filename):\n",
    "        if(len(filenames)<num_frames_per_clip):\n",
    "            print(\"Get invaild data!\")\n",
    "            return [],s_index\n",
    "        filenames=sorted(filenames)\n",
    "        s_index=random.randint(0,len(filenames)-num_frames_per_clip)\n",
    "        for i in range(s_index,s_index+num_frames_per_clip):\n",
    "            image_name=str(filename)+'/'+str(filenames[i])\n",
    "            img=Image.open(image_name)\n",
    "            img_data=np.array(img)\n",
    "            ret_arr.append(img_data)\n",
    "    return ret_arr,s_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.00000015, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.0005)\n",
    "\n",
    "logger = Logger('./logs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_ordered_data(filename,index,num_frames_per_clip):\n",
    "    crop_size=160\n",
    "    lines=open(filename,'r')\n",
    "    lines=list(lines)\n",
    "    line=lines[index].strip('\\n').split()\n",
    "    dirname=line[0]\n",
    "    label=line[1]\n",
    "    \n",
    "    tmp_data,_=get_frames_data(dirname,num_frames_per_clip)\n",
    "    img_datas=[]\n",
    "    if(len(tmp_data)!=0):\n",
    "        for j in xrange(len(tmp_data)):\n",
    "            img=Image.fromarray(tmp_data[j].astype(np.uint8))   \n",
    "            if(img.width>img.height):\n",
    "                scale=float(crop_size)/float(img.height)\n",
    "                img=np.array(cv2.resize(np.array(img),(int(img.width*scale+1),crop_size))).astype(np.float32)      \n",
    "            else:                    \n",
    "                scale=float(crop_size)/float(img.width)\n",
    "                img=np.array(cv2.resize(np.array(img),(int(crop_size),img.height*scale+1))).astype(np.float32)\n",
    "            crop_x=int((img.shape[0]-crop_size)/2)\n",
    "            crop_y=int((img.shape[1]-crop_size)/2)\n",
    "            img=img[crop_x:crop_x+crop_size,crop_y:crop_y+crop_size,:] #-np_mean[j] \n",
    "            img_datas.append(img)\n",
    "            \n",
    "    return np.array(img_datas).astype(np.float32),np.array(label).astype(np.int64),dirname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This class must be needed for preparing our own dataset using the interface of Pytorch(inherit from torch.utils.data.Dataset)\n",
    "class Xdata(torch.utils.data.Dataset):\n",
    "    def __init__(self,filename,start_pos=-1,num_frames_per_clip=16,crop_size=160):\n",
    "        lines=open(filename,'r')\n",
    "        read_dirnames=[]\n",
    "        data=[]\n",
    "        label=[]\n",
    "        batch_index=0\n",
    "        next_batch_start=-1\n",
    "        lines=list(lines)\n",
    "        self.len=len(lines)\n",
    "        self.datalist=lines\n",
    "        self.num_frames_per_clip=num_frames_per_clip\n",
    "        self.crop_size=crop_size\n",
    "        \n",
    "    def __getitem__(self,index):\n",
    "        crop_size=self.crop_size\n",
    "        \n",
    "#         index=index % self.len\n",
    "        line=self.datalist[index].strip('\\n').split()\n",
    "        dirname=line[0]\n",
    "        tmp_label=line[1]\n",
    "#         use_label=[0] * 10\n",
    "#         use_label[int(tmp_label)]=1\n",
    "        tmp_data,_=get_frames_data(dirname,self.num_frames_per_clip)\n",
    "        img_datas=[]\n",
    "        if(len(tmp_data)!=0):\n",
    "            for j in xrange(len(tmp_data)):\n",
    "                img=Image.fromarray(tmp_data[j].astype(np.uint8))\n",
    "                \n",
    "                \n",
    "                if(img.width>img.height):\n",
    "                    scale=float(crop_size)/float(img.height)\n",
    "                    img=np.array(cv2.resize(np.array(img),(int(img.width*scale+1),crop_size))).astype(np.float32)\n",
    "                    \n",
    "                else:\n",
    "                    scale=float(crop_size)/float(img.width)\n",
    "                    img=np.array(cv2.resize(np.array(img),(crop_size,int(img.height*scale+1)))).astype(np.float32)\n",
    "                crop_x=int((img.shape[0]-crop_size)/2)\n",
    "                crop_y=int((img.shape[1]-crop_size)/2)\n",
    "                img=img[crop_x:crop_x+crop_size,crop_y:crop_y+crop_size,:] #-np_mean[j]\n",
    "                \n",
    "                img_datas.append(img)\n",
    "#             data.append(img_datas)\n",
    "#             label.append(int(tmp_label))\n",
    "#             batch_index=batch_index+1\n",
    "#             read_dirnames.append(dirname)\n",
    "        np_arr_data=np.array(img_datas).astype(np.float32)\n",
    "        np_arr_label=np.array(tmp_label).astype(np.int64)\n",
    "        return np_arr_data,np_arr_label,dirname\n",
    "    def __len__(self):\n",
    "        return self.len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Cell 13\n",
    "batch_size=7\n",
    "train_data=Xdata(filename='train.list',\n",
    "                 num_frames_per_clip=16,\n",
    "                crop_size=160)\n",
    "test_data=Xdata(filename='res.list',\n",
    "                num_frames_per_clip=16,\n",
    "                crop_size=160)\n",
    "#train_loader can be iterated.\n",
    "train_loader=DataLoader(train_data,batch_size=batch_size,shuffle=True,drop_last=True)\n",
    "test_loader=DataLoader(test_data,batch_size=batch_size,shuffle=True,drop_last=True)\n",
    "\n",
    "batch_iterator = iter(train_loader)\n",
    "test_iterator=iter(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cell 14 :\n",
    "model.load_state_dict(torch.load('checkpoint_ucf_iter_9600.pth'))\n",
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cell 15 :the cell for training the model:\n",
    "torch.cuda.empty_cache()\n",
    "epoch=0\n",
    "m=sys.maxsize\n",
    "lr=0\n",
    "for iteration in xrange(9601,13000):\n",
    "    epoch+=1\n",
    "    loss=0\n",
    "    #adjust learning rate\n",
    "    if(iteration % 2400==0):\n",
    "        lr=optimizer.param_groups[0]['lr']*0.2\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n",
    "    #Caution:Pytorch DOES NOT generate the repeat data!\n",
    "    try:\n",
    "        images,labels,_=next(batch_iterator)\n",
    "    except StopIteration:\n",
    "        batch_iterator = iter(train_loader)\n",
    "        images,labels,_=next(batch_iterator)\n",
    "        print(\"dataloader reloaded.\")\n",
    "    \n",
    "    \n",
    "    l_true=labels\n",
    "    #shape from(batch_size,clip_size,height,width,channel) -> (batch_size,channel,clip_size,height,width)\n",
    "    images=images.permute(0,4,1,2,3)\n",
    "    images = Variable(images.cuda())\n",
    "    labels = Variable(labels.cuda())\n",
    "    t0 = time.time()\n",
    "    predict=model(images)\n",
    "    #calculate the accuracy:\n",
    "    l_predict=predict.cpu().detach().numpy()\n",
    "    \n",
    "    k=np.argmax(l_predict,axis=-1)\n",
    "    sub=1*np.equal(k,l_true)\n",
    "    acc=np.mean(sub.cpu().detach().numpy())\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss= criterion(predict, labels)\n",
    "    \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    t1 = time.time()\n",
    "    if iteration % 10 == 0:\n",
    "        print('timer: %.4f sec.' % (t1 - t0))\n",
    "        print('iter ' + repr(iteration) + ' || Loss: %.4f ||' % (loss), end=' ')\n",
    "        print(' -> accuracy: %.2f.' % (acc))\n",
    "    if iteration % 800 ==0:\n",
    "        torch.save(model.state_dict(),'/home/gez/p3d/' + 'x_checkpoint_ucf_iter_{}'.format(iteration) + '.pth')\n",
    "        print('current lr={}'.format(lr))\n",
    "torch.save(model.state_dict(),'/home/gez/p3d/' + 'x_checkpoint_ucf_done_finetune' + '.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Cells below are provided for evaling model,run Cell 1-12 and Cells below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size=7\n",
    "train_data=Xdata(filename='train.list',   \n",
    "                      num_frames_per_clip=16,\n",
    "                      crop_size=160,\n",
    "                      )\n",
    "test_data=Xdata(filename='res.list',\n",
    "                      num_frames_per_clip=16,\n",
    "                      crop_size=160,\n",
    "                      )\n",
    "#train_loader can be iterated.\n",
    "train_loader=DataLoader(train_data,batch_size=batch_size,shuffle=False,drop_last=True)\n",
    "test_loader=DataLoader(test_data,batch_size=batch_size,shuffle=False,drop_last=True)\n",
    "\n",
    "batch_iterator = iter(train_loader)\n",
    "test_iterator=iter(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('x_checkpoint_ucf_done_finetune.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The cell for validating the model:\n",
    "acc=0\n",
    "c=0\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "l=len(open('train.list','r').readlines())\n",
    "\n",
    "print(l/batch_size)\n",
    "\n",
    "for i in range(l/batch_size):\n",
    "    try:\n",
    "        images,labels,k=next(batch_iterator)\n",
    "    except StopIteration:\n",
    "        break\n",
    "    else:\n",
    "        l_true=labels\n",
    "        \n",
    "        images=images.permute(0,4,1,2,3)\n",
    "        images=Variable(images.cuda())\n",
    "        labels = Variable(labels.cuda())\n",
    "        predict=model(images)\n",
    "        \n",
    "        l_predict=predict.cpu().detach().numpy()\n",
    "        q=np.argmax(l_predict,axis=-1)\n",
    "        \n",
    "        sub=np.equal(l_true,q)\n",
    "        acc_one=np.mean(sub.cpu().detach().numpy())\n",
    "        \n",
    "        acc=acc+acc_one\n",
    "        c=c+1\n",
    "        print('acc=',acc_one)\n",
    "acc=acc/c\n",
    "print('accuracy on test data is %.2f .' % acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The cell for validating the model:\n",
    "acc=0\n",
    "c=0\n",
    "torch.cuda.empty_cache()\n",
    "l=len(open('res.list','r').readlines())\n",
    "print(l/batch_size)\n",
    "\n",
    "\n",
    "for i in range(l/batch_size):\n",
    "    try:\n",
    "        images,labels,k=next(test_iterator)\n",
    "    except StopIteration:\n",
    "        print('Data has been all fetched out')\n",
    "        break\n",
    "    else:\n",
    "        l_true=labels\n",
    "        images=images.permute(0,4,1,2,3)\n",
    "        images=Variable(images.cuda())\n",
    "        \n",
    "        predict=model(images)\n",
    "        \n",
    "        l_predict=predict.cpu().detach().numpy()\n",
    "        q=np.argmax(l_predict,axis=-1)\n",
    "        \n",
    "        sub=np.equal(l_true,q)\n",
    "        acc_one=np.mean(sub.cpu().detach().numpy())\n",
    "        \n",
    "        acc=acc+acc_one\n",
    "        c=c+1\n",
    "        print('acc=',acc_one)\n",
    "acc=acc/c\n",
    "print('accuracy on test data is %.2f .' % acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}