WordEmbeddings_Feat_GloVe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### based on: https://github.com/iphysresearch/AI_Programs/blob/master/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%20Machine%20Learning/ml_lecture_3/Chinese-sentiment-analysis/chinese-sentiment-analysis_w2v_lstm.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "import numpy as np\n",
    "\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "from gensim.corpora.dictionary import Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers.core import Dense, Dropout,Activation\n",
    "from keras.models import model_from_yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_dim = 300\n",
    "maxlen = 300\n",
    "n_iterations = 5 \n",
    "n_exposures = 10\n",
    "window_size = 10\n",
    "batch_size = 32\n",
    "n_epoch = 8\n",
    "input_length = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def classFit(x):\n",
    "    if x['qual_a_melhor_classificao_para_esse_texto'] == \"diario\":\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def loadfile():\n",
    "    corpus = pd.read_csv('corpus.csv.gz', compression='gzip')\n",
    "    corpus = corpus[corpus['qual_a_melhor_classificao_para_esse_texto:confidence'] == 1]\n",
    "    corpus.reset_index()\n",
    "    corpus['class'] = corpus.apply(classFit,axis=1)\n",
    "    y = corpus['class'].values\n",
    "\n",
    "    combined= corpus.content\n",
    "\n",
    "    return combined,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import unicodedata\n",
    "\n",
    "def remove_accents(input_str):\n",
    "    nfkd_form = unicodedata.normalize('NFKD', input_str)\n",
    "    return u\"\".join([c for c in nfkd_form if not unicodedata.combining(c)])\n",
    "\n",
    "def tokenizer(text):\n",
    "    ''' Simple Parser converting each document to lower-case, then\n",
    "        removing the breaks for new lines and finally splitting on the\n",
    "        whitespace\n",
    "    '''\n",
    "    return_text = []\n",
    "    for sentence in text:\n",
    "        reg_tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "        tokens = reg_tokenizer.tokenize(sentence)\n",
    "        return_text.append([remove_accents(w.lower()) for w in tokens])\n",
    "        \n",
    "    return return_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dictionaries(model=None,\n",
    "                        combined=None):\n",
    "    ''' Function does are number of Jobs:\n",
    "        1- Creates a word to index mapping\n",
    "        2- Creates a word to vector mapping\n",
    "        3- Transforms the Training and Testing Dictionaries\n",
    "\n",
    "    '''\n",
    "    if (combined is not None) and (model is not None):\n",
    "        gensim_dict = Dictionary()\n",
    "        \n",
    "        w2v = model.dictionary\n",
    "        \n",
    "        gensim_dict.doc2bow(w2v.keys(),\n",
    "                            allow_update=True)\n",
    "        w2indx = {v: k+1 for k, v in gensim_dict.items()}\n",
    "        w2vec = {word: w2v[word] for word in w2indx.keys()}\n",
    "\n",
    "        def parse_dataset(combined):\n",
    "            ''' Words become integers\n",
    "            '''\n",
    "            data=[]\n",
    "            for sentence in combined:\n",
    "                new_txt = []\n",
    "                for word in sentence:\n",
    "                    try:\n",
    "                        new_txt.append(w2indx[word])\n",
    "                    except:\n",
    "                        new_txt.append(0)\n",
    "                data.append(new_txt)\n",
    "            return data\n",
    "        \n",
    "        combined=parse_dataset(combined)\n",
    "        combined= sequence.pad_sequences(combined, maxlen=maxlen)\n",
    "        \n",
    "        return w2indx, w2vec,combined\n",
    "    else:\n",
    "        print ('No data provided...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glove import Corpus, Glove\n",
    "\n",
    "def glove_train(combined):    \n",
    "    \n",
    "    corpus = Corpus()\n",
    "    corpus.fit(combined, window=window_size)\n",
    "    glove = Glove(no_components=vocab_dim, learning_rate=0.05)\n",
    "    glove.fit(corpus.matrix, epochs=10, no_threads=16, verbose=True)\n",
    "    glove.add_dictionary(corpus.dictionary)\n",
    "    \n",
    "    glove.save('GloVe_model.pkl')\n",
    "    \n",
    "    index_dict, word_vectors,combined = create_dictionaries(model=glove,combined=combined)\n",
    "    return   index_dict, word_vectors,combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_data(index_dict,word_vectors,combined,y):\n",
    "\n",
    "    n_symbols = len(index_dict) + 1\n",
    "    embedding_weights = np.zeros((n_symbols, vocab_dim))\n",
    "    for word, index in index_dict.items():\n",
    "        embedding_weights[index, :] = word_vectors[word]\n",
    "    x_train, x_test, y_train, y_test = train_test_split(combined, y, test_size=0.2)\n",
    "    print (\"embedding_weights\", embedding_weights.shape)\n",
    "    print (x_train.shape,y_train.shape)\n",
    "    return n_symbols,embedding_weights,x_train,y_train,x_test,y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test):\n",
    "    print ('Defining a Simple Keras Model...')\n",
    "    model = Sequential()  # or Graph or whatever\n",
    "    model.add(Embedding(output_dim=vocab_dim,\n",
    "                        input_dim=n_symbols,\n",
    "                        mask_zero=True,\n",
    "                        weights=[embedding_weights],\n",
    "                        input_length=input_length))  # Adding Input Length\n",
    "    \n",
    "    model.add(LSTM(units=50, recurrent_activation=\"hard_sigmoid\", activation=\"sigmoid\"))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1))\n",
    "    model.add(Activation('sigmoid'))\n",
    "\n",
    "    print ('Compiling the Model...')\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "    print (\"Train...\")\n",
    "    model.fit(x_train, y_train, batch_size=batch_size, epochs=n_epoch,verbose=1, validation_data=(x_test, y_test))\n",
    "\n",
    "    print (\"Evaluate...\")\n",
    "    score = model.evaluate(x_test, y_test,\n",
    "                                batch_size=batch_size)\n",
    "\n",
    "    yaml_string = model.to_yaml()\n",
    "    with open('Glove_lstm.yml', 'w') as outfile:\n",
    "        outfile.write( yaml.dump(yaml_string, default_flow_style=True) )\n",
    "    model.save_weights('Glove_lstm.h5')\n",
    "    print ('Test score:', score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train():\n",
    "    print ('Loading Data...')\n",
    "    combined,y=loadfile()\n",
    "    print (len(combined),len(y))\n",
    "    print ('Tokenising...')\n",
    "    combined = tokenizer(combined)\n",
    "    print ('Training a Word2vec model...')\n",
    "    index_dict, word_vectors,combined=glove_train(combined)\n",
    "    print ('Setting up Arrays for Keras Embedding Layer...')\n",
    "    n_symbols,embedding_weights,x_train,y_train,x_test,y_test=get_data(index_dict, word_vectors,combined,y)\n",
    "    print (x_train.shape,y_train.shape)\n",
    "    train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def input_transform(string):\n",
    "    glove = Glove(no_components=300, learning_rate=0.05)\n",
    "    \n",
    "    words = []\n",
    "    reg_tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "    tokens = reg_tokenizer.tokenize(string)\n",
    "    words.append([remove_accents(w.lower()) for w in tokens])\n",
    "    \n",
    "    words=np.array(words).reshape(1,-1)\n",
    "    model=glove.load('GloVe_model.pkl')\n",
    "    _,_,combined=create_dictionaries(model,words)\n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_predict(string):\n",
    "    print ('loading model......')\n",
    "    with open('Glove_lstm.yml', 'r') as f:\n",
    "        yaml_string = yaml.load(f)\n",
    "    model = model_from_yaml(yaml_string)\n",
    "\n",
    "    print ('loading weights......')\n",
    "    model.load_weights('Glove_lstm.h5')\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',metrics=['accuracy'])\n",
    "    data=input_transform(string)\n",
    "    data.reshape(1,-1)\n",
    "    \n",
    "    #print (data)\n",
    "    result=model.predict_classes(data, verbose=0)\n",
    "    if result[0][0]==1:\n",
    "        print (string,' positive')\n",
    "    else:\n",
    "        print (string,' negative')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Data...\n",
      "534 534\n",
      "Tokenising...\n",
      "Training a Word2vec model...\n",
      "Performing 10 training epochs with 16 threads\n",
      "Epoch 0\n",
      "Epoch 1\n",
      "Epoch 2\n",
      "Epoch 3\n",
      "Epoch 4\n",
      "Epoch 5\n",
      "Epoch 6\n",
      "Epoch 7\n",
      "Epoch 8\n",
      "Epoch 9\n",
      "Setting up Arrays for Keras Embedding Layer...\n",
      "embedding_weights (23370, 300)\n",
      "(427, 300) (427,)\n",
      "(427, 300) (427,)\n",
      "Defining a Simple Keras Model...\n",
      "Compiling the Model...\n",
      "Train...\n",
      "Train on 427 samples, validate on 107 samples\n",
      "Epoch 1/8\n",
      "427/427 [==============================] - 6s - loss: 0.6843 - acc: 0.6347 - val_loss: 0.6554 - val_acc: 0.6449\n",
      "Epoch 2/8\n",
      "427/427 [==============================] - 6s - loss: 0.7365 - acc: 0.6417 - val_loss: 0.7074 - val_acc: 0.6449\n",
      "Epoch 3/8\n",
      "427/427 [==============================] - 6s - loss: 0.7516 - acc: 0.6323 - val_loss: 0.6771 - val_acc: 0.6449\n",
      "Epoch 4/8\n",
      "427/427 [==============================] - 6s - loss: 0.7242 - acc: 0.6159 - val_loss: 0.6735 - val_acc: 0.6449\n",
      "Epoch 5/8\n",
      "427/427 [==============================] - 6s - loss: 0.6993 - acc: 0.6230 - val_loss: 0.6553 - val_acc: 0.6449\n",
      "Epoch 6/8\n",
      "427/427 [==============================] - 6s - loss: 0.6773 - acc: 0.6253 - val_loss: 0.6509 - val_acc: 0.6449\n",
      "Epoch 7/8\n",
      "427/427 [==============================] - 6s - loss: 0.6750 - acc: 0.6230 - val_loss: 0.6502 - val_acc: 0.6449\n",
      "Epoch 8/8\n",
      "427/427 [==============================] - 6s - loss: 0.6944 - acc: 0.6042 - val_loss: 0.6499 - val_acc: 0.6449\n",
      "Evaluate...\n",
      " 96/107 [=========================>....] - ETA: 0sTest score: [0.64989185611778333, 0.6448598141982177]\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading model......\n",
      "loading weights......\n",
      "Esse ano decidi eu mesma fazer os presentes que vou dar de páscoa para a minha família. Depois de muita pesquisa na net descobri o site da Cozinha da Janita que tem várias receitas diferentes de brigadeiro. vale procurar em outros sites também. Eu achei mais de 10 receitas entre brigadeiro de limão com mel, pistache, capuccino, macadâmia e até de panetone (esse vou fazer em dezembro com certeza!).   Fiquei com medo de fica muito duro e não conseguir enrolar e acabou que ficou meio mole...rs... Mas acho que ficaram gostosos!!!     Ai vão as fotos para que vocês possam ver minha aventura gastronômica que durou toda a manhã dessa sexta-feira da paixão!      Lá no fundo as caixinhas...No meio as forminhas coloridas...No cantinho esquerdo o granulado...Por último, brigadeiro de maracujá, capuccino, tradicional e de limão!     Detalhe do desenho nas caixinhas - ovinhos de páscoa     Depois de enrolados com muito capricho...Tradicional com granulado e os outros com açúcar refinado...     Os docinhos arrumadinhos na caixinhas...    O presente pronto para ser entregue!!!    Feliz Páscoa á todos!!!!  Beijos  Flavinha Cartacho      positive\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "combined,y=loadfile()\n",
    "\n",
    "idx = 466\n",
    "string = combined[idx]\n",
    "lstm_predict(string)\n",
    "\n",
    "print(y[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc 0.64606741573\n",
      "f1 0.784982935154\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "with open('Glove_lstm.yml', 'r') as f:\n",
    "    yaml_string = yaml.load(f)\n",
    "model = model_from_yaml(yaml_string)\n",
    "\n",
    "model.load_weights('Glove_lstm.h5')\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "combined,y=loadfile()\n",
    "\n",
    "y_pred = []\n",
    "\n",
    "idx = 0\n",
    "for text in combined:\n",
    "    data = input_transform(text)\n",
    "    \n",
    "    result=model.predict_classes(data, verbose=0)\n",
    "    y_pred.append(result[0][0])\n",
    "    \n",
    "    #print(result[0][0], y[idx])\n",
    "    #idx += 1\n",
    "    #if idx ==10: break\n",
    "    \n",
    "print(\"acc\", accuracy_score(y, y_pred))\n",
    "print(\"f1\", f1_score(y, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}