Spelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# ignore some Keras warnings regarding deprecations and model saving \n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from keras.models import Model, load_model, Sequential\n",
    "from keras.layers import Input, LSTM, Dense, RepeatVector, \\\n",
    "                         TimeDistributed, Activation\n",
    "from keras.callbacks import EarlyStopping\n",
    "import pickle\n",
    "\n",
    "from helpers import *\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sentences from the [tatoeba dataset](https://tatoeba.org/eng/downloads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 100  # Number of epochs to train for.\n",
    "num_samples =220000 # Number of samples to train on.\n",
    "\n",
    "# Path to the txt file on disk.\n",
    "data_path = 'sentences.txt'\n",
    "\n",
    "epochs = 100\n",
    "noise = .05\n",
    "misspellings_count = 3\n",
    "batch_size = 128  # Batch size for training.\n",
    "\n",
    "optimizer= 'adam'\n",
    "loss_fn='categorical_crossentropy'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hand-pick maximum sequence lengths\n",
    "max_seq_length = 25 # max (allowed) input sequence length\n",
    "max_target_seq_length = max_seq_length + 2 # accomodate for the delimiters = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_preprocessed(data_path, max_len):\n",
    "    \"\"\"Dirty helper fn loading a file from disk, doing some basic preprocessing\n",
    "        and filtering out phrases that are longer than our maximum sequence length\"\"\"\n",
    "    with open(data_path) as f:\n",
    "        lines = f.readlines()\n",
    "    lines = text_preprocess(lines)\n",
    "    # allow only for a limited count of \n",
    "    allowed_chars = set(' !\"#$%&\\'()+,-./0123456789:;?[]_`abcdefghijklmnopqrstuvwxyz{}')\n",
    "    selected = []\n",
    "    for l in lines:\n",
    "        if all([c in allowed_chars for c in l.strip()]) and \\\n",
    "           len(l) <= max_len:\n",
    "            selected.append(l)\n",
    "    # suffle deterministically\n",
    "    Random(0).shuffle(selected)\n",
    "    return selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All phrases in dataset:  239820\n",
      "Training phrases:  220000\n",
      "Test phrases:  19820\n",
      "Examples:\n",
      " * tom is reading a book now\n",
      " * tom is quite strong\n",
      " * what did you just do?\n",
      " * tom unloaded the car\n",
      " * you aren't hurt\n",
      " * i had a hard time\n",
      " * tom actually likes me\n",
      " * i'm loaded\n",
      " * i began working\n",
      " * how was the interview?\n"
     ]
    }
   ],
   "source": [
    "all_phrases = load_preprocessed(data_path, max_seq_length)\n",
    "assert len(all_phrases) > num_samples\n",
    "train_phrases = all_phrases[:num_samples]\n",
    "test_phrases = all_phrases[num_samples:]\n",
    "print('All phrases in dataset: ', len(all_phrases))\n",
    "print('Training phrases: ', len(train_phrases))\n",
    "print('Test phrases: ', len(test_phrases))\n",
    "\n",
    "print(\"\\n * \".join(['Examples:'] + all_phrases[:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique tokens: 57\n"
     ]
    }
   ],
   "source": [
    "# create doken indices out of all phrases\n",
    "token_idx = token_index(all_phrases + ['\\t', '\\n'])\n",
    "# ^^ \\t and \\n are our [START] and [END] delimiters. With this trick\n",
    "# we are adding them to the token index\n",
    "\n",
    "num_encoder_tokens = len(token_idx)\n",
    "\n",
    "print('Number of unique tokens:', num_encoder_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_lstm(output_len, token_count):\n",
    "    \"\"\"Generate the model\"\"\"\n",
    "    latent_dim = 256  # Latent dimensionality of the encoding space.\n",
    "    initializer = 'he_normal'\n",
    "\n",
    "    # \"Encode\" the input sequence using an RNN, producing an output of HIDDEN_SIZE\n",
    "    # note: in a situation where your input sequences have a variable length,\n",
    "    # use input_shape=(None, nb_feature).\n",
    "    encoder = LSTM(latent_dim, input_shape=(None, token_count), \n",
    "                   kernel_initializer=initializer)\n",
    "\n",
    "    # For the decoder's input, we repeat the encoded input for each time step\n",
    "    repeater = RepeatVector(output_len)\n",
    "\n",
    "    decoder = LSTM(latent_dim, return_sequences=True, kernel_initializer=initializer)\n",
    "\n",
    "    # For each of step of the output sequence, decide which character should be chosen\n",
    "    time_dist = TimeDistributed(Dense(token_count, kernel_initializer=initializer))\n",
    "    activation = Activation('softmax')\n",
    "    \n",
    "    model = Sequential([encoder, repeater, decoder, time_dist, activation])\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train few epochs on an identity fn with a chunk of the dataset for sanity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "misspellings, correct = create_misspellings(train_phrases,\n",
    "                                            noise, misspellings_count,\n",
    "                                            max_seq_length)\n",
    "\n",
    "X = vectorize_batch(misspellings, token_idx,\n",
    "                    max_seq_length, dtype=np.bool)\n",
    "Y = vectorize_batch(wrap_with_delims(correct), token_idx,\n",
    "                    max_target_seq_length, dtype=np.bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 809600 samples, validate on 70400 samples\n",
      "Epoch 1/100\n",
      "230528/809600 [=======>......................] - ETA: 15:34 - loss: 1.4644 - acc: 0.3574"
     ]
    }
   ],
   "source": [
    "model = simple_lstm(max_target_seq_length, len(token_idx))\n",
    "early_stopping = EarlyStopping(monitor='val_loss', min_delta=.005, \n",
    "                               patience=3, verbose=0, mode='auto')\n",
    "model.fit(X,Y, batch_size=batch_size, epochs=epochs, validation_split=.08,\n",
    "         callbacks= [early_stopping])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(model.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_model = model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translator_fn(model, token_index, max_seq_len):\n",
    "    inverse_token_index = {v: k for k, v in token_index.items()}\n",
    "    def predict(in_phrase):\n",
    "        x = vectorize_phrase(in_phrase, token_index, max_seq_len)\n",
    "        pred_idxes = model.predict_classes(x, verbose=0)[0]\n",
    "        txt = ''.join([inverse_token_index[i] for i in pred_idxes])\n",
    "        end_idx = txt.find(\"\\n\")\n",
    "        return txt[1:end_idx]\n",
    "    return predict\n",
    "\n",
    "def evaluate_correct(texts, corrector):\n",
    "    errors = 0.0\n",
    "    for t in texts:\n",
    "        if t != corrector(t): errors += 1\n",
    "    return errors / len(texts)\n",
    "\n",
    "def evaluate_misspelled(texts, corrector):\n",
    "    errors = 0.0\n",
    "    for t in texts:\n",
    "        errored = add_noise_to_string(t, 0.05)\n",
    "        if t != corrector(errored): errors += 1\n",
    "    return errors / len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_vectorizer_fn(token_index, max_encoder_seq_length,\n",
    "                           max_decoder_seq_length):\n",
    "    \"\"\"Create a closure fn for vectorization that \"knows\" the \n",
    "    token index and seq lengths\"\"\" \n",
    "    def training_vectorizer(input_texts, target_texts):\n",
    "        X = vectorize_batch(input_texts, token_index,\n",
    "                            max_encoder_seq_length, dtype=np.bool)\n",
    "        Y = vectorize_batch(target_texts, token_index,\n",
    "                            max_decoder_seq_length, dtype=np.bool)\n",
    "        return X, Y\n",
    "\n",
    "    return training_vectorizer\n",
    "\n",
    "# Create a training_vectorizer that only accepts input and target texts\n",
    "training_vectorizer = training_vectorizer_fn(token_idx, max_seq_length,\n",
    "                                             max_target_seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batched_gen(phrases, chunk_size, misspellings_count, noise, max_txt_len):\n",
    "    \"\"\"Goes through the given phrases, in `chunk_size` chunks, generating \n",
    "    `misspellings_count` misspelling allongside them.\n",
    "    On each iteration it yields `batch_size`* (1+ misspellings_count) strings: \n",
    "    the original strings and the misspellings generated out of them\"\"\"\n",
    "    for i in range(0, len(phrases), chunk_size):\n",
    "        frrom = i\n",
    "        to = i+chunk_size\n",
    "        yield create_misspellings(phrases[frrom:to], noise, misspellings_count,\n",
    "                                  max_txt_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tst = batched_gen(train_phrases, 2, 2, 0.07, max_seq_length)\n",
    "inp, trgt = next(tst)\n",
    "for i,t in zip(inp, trgt):\n",
    "    print(i, '->',t)\n",
    "    \n",
    "_ = next(tst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorized_gen(phrases, batch_size, misspellings_count, noise,\n",
    "                   max_txt_len,training_vectorizer):\n",
    "    \"\"\"Creates vextorized batches of phrases (that are wrapped with delims)\"\"\"\n",
    "    # Create a generator of misspelled strings from the input phrases\n",
    "    gen = batched_gen(phrases, batch_size, misspellings_count, noise, max_txt_len)\n",
    "    \n",
    "    # Go through all the input phrases, generatiing misspellings, vectorizing them\n",
    "    # and yielding each batch\n",
    "    for input_phrases, target_phrases in gen:\n",
    "        target_phrases = wrap_with_delims(target_phrases)\n",
    "        X, Y = training_vectorizer(input_phrases, target_phrases)\n",
    "        # Yield the data in a X, Y form\n",
    "        yield (X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_size = 2000  # <- best if it divides `num_samples`\n",
    "\n",
    "# create a final generator holding all the context\n",
    "def training_generator():\n",
    "    \"\"\"Keep iterating over the training set in chunks\"\"\"\n",
    "    while True:\n",
    "        gen = vectorized_gen(train_phrases, chunk_size,\n",
    "                             misspellings_count, noise,\n",
    "                             max_seq_length,\n",
    "                             training_vectorizer)\n",
    "        yield from gen\n",
    "\n",
    "steps_per_epoch = (len(train_phrases) / chunk_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For validation, just use a fixed set of examples, not a generator\n",
    "val_wrong, val_right = create_misspellings(test_phrases[:1000], noise, 3, max_seq_length)\n",
    "wrapped_val_phrases = wrap_with_delims(val_right)\n",
    "val_X = vectorize_batch(val_wrong, token_idx, max_seq_length, dtype=np.bool)\n",
    "val_Y = vectorize_batch(wrapped_val_phrases, token_idx, max_target_seq_length, dtype=np.bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen = training_generator()\n",
    "model = simple_lstm(max_target_seq_length, len(token_idx))\n",
    "early_stopping = EarlyStopping(monitor='val_loss', min_delta=.005, \n",
    "                               patience=3, verbose=0, mode='auto')\n",
    "\n",
    "model.fit_generator(training_generator(), validation_data=(val_X, val_Y),\n",
    "                    steps_per_epoch=steps_per_epoch,\n",
    "                    verbose=2, max_queue_size=3, epochs=epochs,\n",
    "                   callbacks= [early_stopping])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(model.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translator_fn(model, token_index, max_seq_len):\n",
    "    inverse_token_index = {v: k for k, v in token_index.items()}\n",
    "    def predict(in_phrase):\n",
    "        x = vectorize_phrase(in_phrase, token_index, max_seq_len)\n",
    "        pred_idxes = model.predict_classes(x, verbose=0)[0]\n",
    "        txt = ''.join([inverse_token_index[i] for i in pred_idxes])\n",
    "        end_idx = txt.find(\"\\n\")\n",
    "        return txt[1:end_idx]\n",
    "    return predict\n",
    "\n",
    "def evaluate_correct(texts, corrector):\n",
    "    errors = 0.0\n",
    "    for t in texts:\n",
    "        if t != corrector(t): errors += 1\n",
    "    return errors / len(texts)\n",
    "\n",
    "def evaluate_misspelled(texts, corrector):\n",
    "    errors = 0.0\n",
    "    for t in texts:\n",
    "        errored = add_noise_to_string(t, 0.05)\n",
    "        if t != corrector(errored): errors += 1\n",
    "    return errors / len(texts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrector = translator_fn(model, token_idx, max_target_seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(corrector(p), p, corrector(p) == p) for p in train_phrases[:20]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(evaluate_correct(train_phrases[:1000], corrector))\n",
    "print(evaluate_correct(train_phrases[-1000:], corrector))\n",
    "print(evaluate_misspelled(train_phrases[:1000], corrector))\n",
    "print(evaluate_correct(test_phrases[:1000], corrector))\n",
    "print(evaluate_misspelled(test_phrases[:1000], corrector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(model.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.846"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.788"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.859"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fire',\n",
       " 'ss?',\n",
       " 'comen',\n",
       " 'get o',\n",
       " 'i cant  ',\n",
       " \"i'm srry\",\n",
       " 'he is bui',\n",
       " 'hess rruk',\n",
       " \"i'll be tt\",\n",
       " 'hold my berr',\n",
       " 'pust the butn',\n",
       " 'coll me on my pone',\n",
       " 'hellooyssannggrrl']"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find max encoder seq legth\n",
    "#max_encoder_seq_length = encoder_model.get_layer('encoder_inputs').input_shape[-1]\n",
    "phrases = ['fire', 'stp', 'comein', 'get ot', 'i cant go','im sorry', \n",
    "           'h is busi', 'hes drunk', 'ill be lat', 'hold mi beer', 'pus the buton', \n",
    "          'coll me on my phone', 'helo boys and girls']\n",
    "\n",
    "[corrector(phrase) for phrase in phrases]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(fname):\n",
    "    \"\"\"quick-n-dirty helper for saving models\"\"\"\n",
    "    print(\"Saving model\")\n",
    "    model.save(fname + '.h5')\n",
    "\n",
    "    model_metadata = { 'token_index': token_index, \n",
    "                       'max_seq_length': max_seq_length,\n",
    "                       'max_encoder_seq_length': max_target_seq_length }\n",
    "\n",
    "    with open(fname + '_metadata.pickle', 'wb') as f:\n",
    "        pickle.dump(model_metadata, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving model\n"
     ]
    }
   ],
   "source": [
    "save('spelling_with_generator')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}