WordEmbeddings_Feat_Word2Vec_Conv1D_Cross.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CONV1D RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### based on:\n",
    "https://github.com/iphysresearch/AI_Programs/blob/master/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%20Machine%20Learning/ml_lecture_3/Chinese-sentiment-analysis/chinese-sentiment-analysis_w2v_lstm.ipynb\n",
    "\n",
    "### model by\n",
    "https://github.com/leriomaggio/deep-learning-keras-tensorflow/blob/master/6.%20AutoEncoders%20and%20Embeddings/6.2%20NLP%20and%20Deep%20Learning.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "import numpy as np\n",
    "\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "from gensim.corpora.dictionary import Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers.core import Dense, Dropout,Activation\n",
    "from keras.models import model_from_yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_dim = 300\n",
    "maxlen = 300\n",
    "n_iterations = 5 \n",
    "n_exposures = 10\n",
    "window_size = 10\n",
    "batch_size = 32\n",
    "n_epoch = 8\n",
    "input_length = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def classFit(x):\n",
    "    if x['qual_a_melhor_classificao_para_esse_texto'] == \"diario\":\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def loadfile():\n",
    "    corpus = pd.read_csv('corpus.csv.gz', compression='gzip')\n",
    "    corpus = corpus[corpus['qual_a_melhor_classificao_para_esse_texto:confidence'] == 1]\n",
    "    corpus.reset_index()\n",
    "    corpus['class'] = corpus.apply(classFit,axis=1)\n",
    "    y = corpus['class'].values\n",
    "\n",
    "    combined= corpus.content\n",
    "\n",
    "    return combined,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import unicodedata\n",
    "\n",
    "def remove_accents(input_str):\n",
    "    nfkd_form = unicodedata.normalize('NFKD', input_str)\n",
    "    return u\"\".join([c for c in nfkd_form if not unicodedata.combining(c)])\n",
    "\n",
    "def tokenizer(text):\n",
    "    ''' Simple Parser converting each document to lower-case, then\n",
    "        removing the breaks for new lines and finally splitting on the\n",
    "        whitespace\n",
    "    '''\n",
    "    return_text = []\n",
    "    for sentence in text:\n",
    "        reg_tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "        tokens = reg_tokenizer.tokenize(sentence)\n",
    "        return_text.append([remove_accents(w.lower()) for w in tokens])\n",
    "        \n",
    "    return return_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_dictionaries(model=None,\n",
    "                        combined=None):\n",
    "    ''' Function does are number of Jobs:\n",
    "        1- Creates a word to index mapping\n",
    "        2- Creates a word to vector mapping\n",
    "        3- Transforms the Training and Testing Dictionaries\n",
    "\n",
    "    '''\n",
    "    if (combined is not None) and (model is not None):\n",
    "        gensim_dict = Dictionary()\n",
    "        \n",
    "        w2v = dict(zip(model.wv.index2word, model.wv.syn0))\n",
    "        \n",
    "        gensim_dict.doc2bow(w2v.keys(),\n",
    "                            allow_update=True)\n",
    "        w2indx = {v: k+1 for k, v in gensim_dict.items()}\n",
    "        w2vec = {word: model[word] for word in w2indx.keys()}\n",
    "\n",
    "        def parse_dataset(combined):\n",
    "            ''' Words become integers\n",
    "            '''\n",
    "            data=[]\n",
    "            for sentence in combined:\n",
    "                new_txt = []\n",
    "                for word in sentence:\n",
    "                    try:\n",
    "                        new_txt.append(w2indx[word])\n",
    "                    except:\n",
    "                        new_txt.append(0)\n",
    "                data.append(new_txt)\n",
    "            return data\n",
    "        \n",
    "        combined=parse_dataset(combined)\n",
    "        combined= sequence.pad_sequences(combined, maxlen=maxlen)\n",
    "        \n",
    "        return w2indx, w2vec,combined\n",
    "    else:\n",
    "        print ('No data provided...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word2vec_train(combined):    \n",
    "    model = Word2Vec(combined, size=vocab_dim, workers=16, iter=10, negative=20)\n",
    "    # trim memory\n",
    "    model.init_sims(replace=True)\n",
    "    \n",
    "    model.save('Word2vec_model.pkl')\n",
    "    index_dict, word_vectors,combined = create_dictionaries(model=model,combined=combined)\n",
    "    return   index_dict, word_vectors,combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data(index_dict,word_vectors,combined,y):\n",
    "\n",
    "    n_symbols = len(index_dict) + 1\n",
    "    embedding_weights = np.zeros((n_symbols, vocab_dim))\n",
    "    for word, index in index_dict.items():\n",
    "        embedding_weights[index, :] = word_vectors[word]\n",
    "    x_train, x_test, y_train, y_test = train_test_split(combined, y, test_size=0.2)\n",
    "    print (\"embedding_weights\", embedding_weights.shape)\n",
    "    print (x_train.shape,y_train.shape)\n",
    "    return n_symbols,embedding_weights,x_train,y_train,x_test,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, Flatten, Conv1D, MaxPooling1D\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.models import Model\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "def train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test,data,target):\n",
    "    print ('Defining a Simple Keras Model...')\n",
    "\n",
    "    def create_model():\n",
    "    \n",
    "        embedding_layer = Embedding(output_dim=vocab_dim,\n",
    "                            input_dim=n_symbols,\n",
    "                            weights=[embedding_weights],\n",
    "                            input_length=input_length)  # Adding Input Length\n",
    "\n",
    "        filter_sizes = (3, 4)\n",
    "        num_filters = 150\n",
    "\n",
    "        graph_in = Input(shape=(input_length, vocab_dim))\n",
    "        convs = []\n",
    "        for fsz in filter_sizes:\n",
    "            conv = Conv1D(filters=num_filters,\n",
    "                          kernel_size=fsz,\n",
    "                          padding='valid',\n",
    "                          activation='relu',\n",
    "                          strides=1)(graph_in)\n",
    "            pool = MaxPooling1D(pool_size=2)(conv)\n",
    "            flatten = Flatten()(pool)\n",
    "            convs.append(flatten)\n",
    "\n",
    "        out = Concatenate()(convs)\n",
    "        graph = Model(inputs=graph_in, outputs=out)\n",
    "\n",
    "        dropout_prob = (0.25, 0.5)\n",
    "        hidden_dims = 150\n",
    "\n",
    "        model = Sequential()\n",
    "        model.add(embedding_layer)\n",
    "        model.add(Dropout(dropout_prob[0], input_shape=(input_length, vocab_dim)))\n",
    "\n",
    "        model.add(graph)\n",
    "        model.add(Dense(hidden_dims))\n",
    "\n",
    "        model.add(Dropout(dropout_prob[1]))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Dense(1))\n",
    "        model.add(Activation('sigmoid'))\n",
    "\n",
    "\n",
    "\n",
    "        print ('Compiling the Model...')\n",
    "        model.compile(loss='binary_crossentropy',\n",
    "                      optimizer='adam',metrics=['accuracy'])\n",
    "        \n",
    "        return model\n",
    "\n",
    "    model = create_model();\n",
    "    estimator = KerasClassifier(build_fn=create_model, nb_epoch=n_epoch, batch_size=batch_size, verbose=1)\n",
    "    \n",
    "    print (\"10-Fold Cross Validation...\")\n",
    "    acc = cross_val_score(estimator, data, target, cv=10, scoring='accuracy').mean()\n",
    "    print (\"10-Fold Cross Acc:\", acc)\n",
    "\n",
    "    print (\"Train...\")\n",
    "    model.fit(x_train, y_train, batch_size=batch_size, epochs=n_epoch,verbose=1, validation_data=(x_test, y_test))    \n",
    "    \n",
    "    print (\"Evaluate...\")\n",
    "    score = model.evaluate(x_test, y_test,\n",
    "                                batch_size=batch_size)\n",
    "    \n",
    "    yaml_string = model.to_yaml()\n",
    "    with open('W2V_lstm.yml', 'w') as outfile:\n",
    "        outfile.write( yaml.dump(yaml_string, default_flow_style=True) )\n",
    "    model.save_weights('W2V_lstm.h5')\n",
    "    print ('Test score:', score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train():\n",
    "    print ('Loading Data...')\n",
    "    combined,y=loadfile()\n",
    "    print (len(combined),len(y))\n",
    "    print ('Tokenising...')\n",
    "    combined = tokenizer(combined)\n",
    "    print ('Training a Word2vec model...')\n",
    "    index_dict, word_vectors,combined=word2vec_train(combined)\n",
    "    print ('Setting up Arrays for Keras Embedding Layer...')\n",
    "    n_symbols,embedding_weights,x_train,y_train,x_test,y_test=get_data(index_dict, word_vectors,combined,y)\n",
    "    print (x_train.shape,y_train.shape)\n",
    "    train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test,combined,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def input_transform(string):\n",
    "    words = []\n",
    "    reg_tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "    tokens = reg_tokenizer.tokenize(string)\n",
    "    words.append([remove_accents(w.lower()) for w in tokens])\n",
    "    \n",
    "    words=np.array(words).reshape(1,-1)\n",
    "    model=Word2Vec.load('Word2vec_model.pkl')\n",
    "    _,_,combined=create_dictionaries(model,words)\n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm_predict(string):\n",
    "    print ('loading model......')\n",
    "    with open('W2V_lstm.yml', 'r') as f:\n",
    "        yaml_string = yaml.load(f)\n",
    "    model = model_from_yaml(yaml_string)\n",
    "\n",
    "    print ('loading weights......')\n",
    "    model.load_weights('W2V_lstm.h5')\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',metrics=['accuracy'])\n",
    "    data=input_transform(string)\n",
    "    data.reshape(1,-1)\n",
    "    \n",
    "    #print (data)\n",
    "    result=model.predict_classes(data, verbose=0)\n",
    "    if result[0][0]==1:\n",
    "        print (string,' positive')\n",
    "    else:\n",
    "        print (string,' negative')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Data...\n",
      "534 534\n",
      "Tokenising...\n",
      "Training a Word2vec model...\n",
      "Setting up Arrays for Keras Embedding Layer...\n",
      "embedding_weights (4525, 300)\n",
      "(427, 300) (427,)\n",
      "(427, 300) (427,)\n",
      "Defining a Simple Keras Model...\n",
      "Compiling the Model...\n",
      "10-Fold Cross Validation...\n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "480/480 [==============================] - 15s - loss: 0.7114 - acc: 0.6250    \n",
      "Epoch 2/10\n",
      "480/480 [==============================] - 14s - loss: 0.5716 - acc: 0.6687    \n",
      "Epoch 3/10\n",
      "480/480 [==============================] - 14s - loss: 0.4806 - acc: 0.7729    \n",
      "Epoch 4/10\n",
      "480/480 [==============================] - 14s - loss: 0.3210 - acc: 0.8792    \n",
      "Epoch 5/10\n",
      "480/480 [==============================] - 14s - loss: 0.1413 - acc: 0.9688    \n",
      "Epoch 6/10\n",
      "480/480 [==============================] - 14s - loss: 0.0423 - acc: 0.9958    \n",
      "Epoch 7/10\n",
      "480/480 [==============================] - 14s - loss: 0.0188 - acc: 0.9938    \n",
      "Epoch 8/10\n",
      "480/480 [==============================] - 14s - loss: 0.0096 - acc: 1.0000    \n",
      "Epoch 9/10\n",
      "480/480 [==============================] - 14s - loss: 0.0030 - acc: 1.0000    \n",
      "Epoch 10/10\n",
      "480/480 [==============================] - 14s - loss: 0.0023 - acc: 1.0000    \n",
      "54/54 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "480/480 [==============================] - 14s - loss: 0.7124 - acc: 0.6271    \n",
      "Epoch 2/10\n",
      "480/480 [==============================] - 14s - loss: 0.5876 - acc: 0.6562    \n",
      "Epoch 3/10\n",
      "480/480 [==============================] - 14s - loss: 0.5266 - acc: 0.7063    \n",
      "Epoch 4/10\n",
      "480/480 [==============================] - 14s - loss: 0.4223 - acc: 0.8333    \n",
      "Epoch 5/10\n",
      "480/480 [==============================] - 14s - loss: 0.2193 - acc: 0.9250    \n",
      "Epoch 6/10\n",
      "480/480 [==============================] - 14s - loss: 0.0999 - acc: 0.9771    \n",
      "Epoch 7/10\n",
      "480/480 [==============================] - 14s - loss: 0.0452 - acc: 0.9896    \n",
      "Epoch 8/10\n",
      "480/480 [==============================] - 14s - loss: 0.0151 - acc: 1.0000    \n",
      "Epoch 9/10\n",
      "480/480 [==============================] - 14s - loss: 0.0071 - acc: 1.0000    \n",
      "Epoch 10/10\n",
      "480/480 [==============================] - 14s - loss: 0.0048 - acc: 1.0000    \n",
      "54/54 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "480/480 [==============================] - 14s - loss: 0.7923 - acc: 0.5479    \n",
      "Epoch 2/10\n",
      "480/480 [==============================] - 14s - loss: 0.6363 - acc: 0.6479    \n",
      "Epoch 3/10\n",
      "480/480 [==============================] - 14s - loss: 0.5922 - acc: 0.6479    \n",
      "Epoch 4/10\n",
      "480/480 [==============================] - 14s - loss: 0.5164 - acc: 0.7417    \n",
      "Epoch 5/10\n",
      "480/480 [==============================] - 14s - loss: 0.3539 - acc: 0.8604    \n",
      "Epoch 6/10\n",
      "480/480 [==============================] - 14s - loss: 0.2069 - acc: 0.9271    \n",
      "Epoch 7/10\n",
      "480/480 [==============================] - 14s - loss: 0.0814 - acc: 0.9896    \n",
      "Epoch 8/10\n",
      "480/480 [==============================] - 14s - loss: 0.0286 - acc: 0.9958    \n",
      "Epoch 9/10\n",
      "480/480 [==============================] - 14s - loss: 0.0142 - acc: 0.9979    \n",
      "Epoch 10/10\n",
      "480/480 [==============================] - 14s - loss: 0.0102 - acc: 0.9958    \n",
      "54/54 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "480/480 [==============================] - 14s - loss: 0.6937 - acc: 0.6250    \n",
      "Epoch 2/10\n",
      "480/480 [==============================] - 14s - loss: 0.5718 - acc: 0.7063    \n",
      "Epoch 3/10\n",
      "480/480 [==============================] - 14s - loss: 0.4220 - acc: 0.8208    \n",
      "Epoch 4/10\n",
      "480/480 [==============================] - 14s - loss: 0.2502 - acc: 0.9021    \n",
      "Epoch 5/10\n",
      "480/480 [==============================] - 14s - loss: 0.1175 - acc: 0.9688    \n",
      "Epoch 6/10\n",
      "480/480 [==============================] - 14s - loss: 0.0531 - acc: 0.9833    \n",
      "Epoch 7/10\n",
      "480/480 [==============================] - 14s - loss: 0.0247 - acc: 0.9917    \n",
      "Epoch 8/10\n",
      "480/480 [==============================] - 14s - loss: 0.0087 - acc: 1.0000    \n",
      "Epoch 9/10\n",
      "480/480 [==============================] - 14s - loss: 0.0053 - acc: 1.0000    \n",
      "Epoch 10/10\n",
      "480/480 [==============================] - 14s - loss: 0.0047 - acc: 1.0000    \n",
      "54/54 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "481/481 [==============================] - 14s - loss: 0.7479 - acc: 0.6195    \n",
      "Epoch 2/10\n",
      "481/481 [==============================] - 14s - loss: 0.6486 - acc: 0.6466    \n",
      "Epoch 3/10\n",
      "481/481 [==============================] - 14s - loss: 0.5981 - acc: 0.6466    \n",
      "Epoch 4/10\n",
      "481/481 [==============================] - 14s - loss: 0.5358 - acc: 0.7089    \n",
      "Epoch 5/10\n",
      "481/481 [==============================] - 14s - loss: 0.4468 - acc: 0.7879    \n",
      "Epoch 6/10\n",
      "481/481 [==============================] - 14s - loss: 0.2737 - acc: 0.9044    \n",
      "Epoch 7/10\n",
      "481/481 [==============================] - 14s - loss: 0.1660 - acc: 0.9335    \n",
      "Epoch 8/10\n",
      "481/481 [==============================] - 14s - loss: 0.0699 - acc: 0.9813    \n",
      "Epoch 9/10\n",
      "481/481 [==============================] - 14s - loss: 0.0298 - acc: 0.9958    \n",
      "Epoch 10/10\n",
      "481/481 [==============================] - 14s - loss: 0.0126 - acc: 0.9958    \n",
      "53/53 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "481/481 [==============================] - 15s - loss: 0.9226 - acc: 0.5114    \n",
      "Epoch 2/10\n",
      "481/481 [==============================] - 16s - loss: 0.6725 - acc: 0.6424    \n",
      "Epoch 3/10\n",
      "481/481 [==============================] - 15s - loss: 0.6434 - acc: 0.6466    \n",
      "Epoch 4/10\n",
      "481/481 [==============================] - 14s - loss: 0.6049 - acc: 0.6466    \n",
      "Epoch 5/10\n",
      "481/481 [==============================] - 14s - loss: 0.5697 - acc: 0.7235    \n",
      "Epoch 6/10\n",
      "481/481 [==============================] - 14s - loss: 0.4925 - acc: 0.7609    \n",
      "Epoch 7/10\n",
      "481/481 [==============================] - 14s - loss: 0.3884 - acc: 0.8482    \n",
      "Epoch 8/10\n",
      "481/481 [==============================] - 14s - loss: 0.2459 - acc: 0.8940    \n",
      "Epoch 9/10\n",
      "481/481 [==============================] - 14s - loss: 0.1133 - acc: 0.9647    \n",
      "Epoch 10/10\n",
      "481/481 [==============================] - 14s - loss: 0.1152 - acc: 0.9647    \n",
      "53/53 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "481/481 [==============================] - 14s - loss: 0.7515 - acc: 0.6091    \n",
      "Epoch 2/10\n",
      "481/481 [==============================] - 14s - loss: 0.6193 - acc: 0.6507    \n",
      "Epoch 3/10\n",
      "481/481 [==============================] - 14s - loss: 0.6198 - acc: 0.7173    \n",
      "Epoch 4/10\n",
      "481/481 [==============================] - 15s - loss: 0.4793 - acc: 0.7859    \n",
      "Epoch 5/10\n",
      "481/481 [==============================] - 15s - loss: 0.3840 - acc: 0.8129    \n",
      "Epoch 6/10\n",
      "481/481 [==============================] - 15s - loss: 0.1970 - acc: 0.9418    \n",
      "Epoch 7/10\n",
      "481/481 [==============================] - 15s - loss: 0.0905 - acc: 0.9854    \n",
      "Epoch 8/10\n",
      "481/481 [==============================] - 14s - loss: 0.1481 - acc: 0.9480    \n",
      "Epoch 9/10\n",
      "481/481 [==============================] - 14s - loss: 0.0371 - acc: 0.9979    \n",
      "Epoch 10/10\n",
      "481/481 [==============================] - 14s - loss: 0.0177 - acc: 0.9979    \n",
      "53/53 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "481/481 [==============================] - 14s - loss: 0.7324 - acc: 0.6237    \n",
      "Epoch 2/10\n",
      "481/481 [==============================] - 14s - loss: 0.6298 - acc: 0.6653    \n",
      "Epoch 3/10\n",
      "481/481 [==============================] - 14s - loss: 0.5423 - acc: 0.7173    \n",
      "Epoch 4/10\n",
      "481/481 [==============================] - 15s - loss: 0.4860 - acc: 0.7588    \n",
      "Epoch 5/10\n",
      "481/481 [==============================] - 15s - loss: 0.2921 - acc: 0.9085    \n",
      "Epoch 6/10\n",
      "481/481 [==============================] - 15s - loss: 0.1441 - acc: 0.9709    \n",
      "Epoch 7/10\n",
      "481/481 [==============================] - 15s - loss: 0.0450 - acc: 0.9917    \n",
      "Epoch 8/10\n",
      "481/481 [==============================] - 15s - loss: 0.0175 - acc: 0.9979    \n",
      "Epoch 9/10\n",
      "481/481 [==============================] - 14s - loss: 0.0076 - acc: 1.0000    \n",
      "Epoch 10/10\n",
      "481/481 [==============================] - 14s - loss: 0.0042 - acc: 1.0000    \n",
      "53/53 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "481/481 [==============================] - 14s - loss: 0.7817 - acc: 0.6175    \n",
      "Epoch 2/10\n",
      "481/481 [==============================] - 14s - loss: 0.6072 - acc: 0.6466    \n",
      "Epoch 3/10\n",
      "481/481 [==============================] - 14s - loss: 0.5636 - acc: 0.7339    \n",
      "Epoch 4/10\n",
      "481/481 [==============================] - 15s - loss: 0.3974 - acc: 0.8462    \n",
      "Epoch 5/10\n",
      "481/481 [==============================] - 14s - loss: 0.2310 - acc: 0.9210    \n",
      "Epoch 6/10\n",
      "481/481 [==============================] - 14s - loss: 0.0883 - acc: 0.9751    \n",
      "Epoch 7/10\n",
      "481/481 [==============================] - 14s - loss: 0.0321 - acc: 0.9917    \n",
      "Epoch 8/10\n",
      "481/481 [==============================] - 14s - loss: 0.0152 - acc: 0.9979    \n",
      "Epoch 9/10\n",
      "481/481 [==============================] - 14s - loss: 0.0083 - acc: 0.9979    \n",
      "Epoch 10/10\n",
      "481/481 [==============================] - 14s - loss: 0.0052 - acc: 0.9979    \n",
      "53/53 [==============================] - 0s     \n",
      "Compiling the Model...\n",
      "Epoch 1/10\n",
      "481/481 [==============================] - 14s - loss: 0.7411 - acc: 0.6050    \n",
      "Epoch 2/10\n",
      "481/481 [==============================] - 14s - loss: 0.5838 - acc: 0.6528    \n",
      "Epoch 3/10\n",
      "481/481 [==============================] - 15s - loss: 0.5038 - acc: 0.7526    \n",
      "Epoch 4/10\n",
      "481/481 [==============================] - 14s - loss: 0.3561 - acc: 0.8565    \n",
      "Epoch 5/10\n",
      "481/481 [==============================] - 14s - loss: 0.1718 - acc: 0.9397    \n",
      "Epoch 6/10\n",
      "481/481 [==============================] - 14s - loss: 0.0835 - acc: 0.9792    \n",
      "Epoch 7/10\n",
      "481/481 [==============================] - 14s - loss: 0.0304 - acc: 0.9958    \n",
      "Epoch 8/10\n",
      "481/481 [==============================] - 14s - loss: 0.0120 - acc: 1.0000    \n",
      "Epoch 9/10\n",
      "481/481 [==============================] - 14s - loss: 0.0059 - acc: 0.9979    \n",
      "Epoch 10/10\n",
      "481/481 [==============================] - 14s - loss: 0.0034 - acc: 1.0000    \n",
      "53/53 [==============================] - 0s     \n",
      "10-Fold Cross Acc: 0.762473794549\n",
      "Train...\n",
      "Train on 427 samples, validate on 107 samples\n",
      "Epoch 1/8\n",
      "427/427 [==============================] - 14s - loss: 0.8904 - acc: 0.5691 - val_loss: 0.6761 - val_acc: 0.5514\n",
      "Epoch 2/8\n",
      "427/427 [==============================] - 14s - loss: 0.6213 - acc: 0.6721 - val_loss: 0.6630 - val_acc: 0.5514\n",
      "Epoch 3/8\n",
      "427/427 [==============================] - 14s - loss: 0.5879 - acc: 0.6721 - val_loss: 0.6820 - val_acc: 0.5514\n",
      "Epoch 4/8\n",
      "427/427 [==============================] - 14s - loss: 0.5695 - acc: 0.6721 - val_loss: 0.6068 - val_acc: 0.5981\n",
      "Epoch 5/8\n",
      "427/427 [==============================] - 14s - loss: 0.4788 - acc: 0.7728 - val_loss: 0.6081 - val_acc: 0.7103\n",
      "Epoch 6/8\n",
      "427/427 [==============================] - 14s - loss: 0.3632 - acc: 0.8407 - val_loss: 0.4622 - val_acc: 0.8037\n",
      "Epoch 7/8\n",
      "427/427 [==============================] - 14s - loss: 0.2013 - acc: 0.9602 - val_loss: 0.4436 - val_acc: 0.8131\n",
      "Epoch 8/8\n",
      "427/427 [==============================] - 14s - loss: 0.0868 - acc: 0.9859 - val_loss: 0.5846 - val_acc: 0.7664\n",
      "Evaluate...\n",
      "107/107 [==============================] - 1s     \n",
      "Test score: [0.58457568329628384, 0.76635514018691586]\n"
     ]
    }
   ],
   "source": [
    "train() #0.49, 0.79 ... 0.94,0.76"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading model......\n",
      "loading weights......\n",
      "Esse ano decidi eu mesma fazer os presentes que vou dar de páscoa para a minha família. Depois de muita pesquisa na net descobri o site da Cozinha da Janita que tem várias receitas diferentes de brigadeiro. vale procurar em outros sites também. Eu achei mais de 10 receitas entre brigadeiro de limão com mel, pistache, capuccino, macadâmia e até de panetone (esse vou fazer em dezembro com certeza!).   Fiquei com medo de fica muito duro e não conseguir enrolar e acabou que ficou meio mole...rs... Mas acho que ficaram gostosos!!!     Ai vão as fotos para que vocês possam ver minha aventura gastronômica que durou toda a manhã dessa sexta-feira da paixão!      Lá no fundo as caixinhas...No meio as forminhas coloridas...No cantinho esquerdo o granulado...Por último, brigadeiro de maracujá, capuccino, tradicional e de limão!     Detalhe do desenho nas caixinhas - ovinhos de páscoa     Depois de enrolados com muito capricho...Tradicional com granulado e os outros com açúcar refinado...     Os docinhos arrumadinhos na caixinhas...    O presente pronto para ser entregue!!!    Feliz Páscoa á todos!!!!  Beijos  Flavinha Cartacho      positive\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "combined,y=loadfile()\n",
    "\n",
    "idx = 466\n",
    "string = combined[idx]\n",
    "lstm_predict(string)\n",
    "\n",
    "print(y[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc 0.949438202247\n",
      "f1 0.962131837307\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "with open('W2V_lstm.yml', 'r') as f:\n",
    "    yaml_string = yaml.load(f)\n",
    "model = model_from_yaml(yaml_string)\n",
    "\n",
    "model.load_weights('W2V_lstm.h5')\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "combined,y=loadfile()\n",
    "\n",
    "y_pred = []\n",
    "\n",
    "idx = 0\n",
    "for text in combined:\n",
    "    data = input_transform(text)\n",
    "    \n",
    "    result=model.predict_classes(data, verbose=0)\n",
    "    y_pred.append(result[0][0])\n",
    "    \n",
    "    #print(result[0][0], y[idx])\n",
    "    #idx += 1\n",
    "    #if idx ==10: break\n",
    "    \n",
    "print(\"acc\", accuracy_score(y, y_pred)) #0.95\n",
    "print(\"f1\", f1_score(y, y_pred)) #0.96"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}