1-Simple-N-Gram-Wikipedia-autocomplete.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "\n",
    "import bs4 as bs\n",
    "import urllib.request\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\t\t\tpages for logged out editors learn more\n",
      "deep learning is part of a broader family of machine learning methods based on artificial neural networks with representation learning. learning can be supervised, semi-supervised or unsupervised.[2]\n",
      "deep-learning architectures such as deep neural networks, deep belief networks, deep reinforcement learning, recurrent neural networks, convolutional neural networks and transformers have been applied to fields including computer vision, speech recognition, natural language processing, machine translation, bioinformatics, drug design, medical image analysis, climate science, material inspection and board game programs, where they have produced results comparable to and in some cases surpassing human expert performance.[3][4][5]\n",
      "artificial neural networks (anns) were inspired by information processing and distributed communication nodes in biological systems. anns have various differences from biological brains.  specifically, artificial neural ne\n"
     ]
    }
   ],
   "source": [
    "# Scrap sentences from wikipedia on desired topic\n",
    "raw_html = urllib.request.urlopen('https://en.wikipedia.org/wiki/Deep_learning')\n",
    "raw_html = raw_html.read()\n",
    "\n",
    "# Read complete page paragraphs\n",
    "article_html = bs.BeautifulSoup(raw_html, 'lxml')\n",
    "article_paragraphs = article_html.find_all('p')\n",
    "article_text = ''\n",
    "\n",
    "# concat all paragraphs\n",
    "for para in article_paragraphs:\n",
    "    article_text += para.text\n",
    "\n",
    "article_text = article_text.lower()\n",
    "print(article_text[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46095"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cleaning sentences - remove everything other than characters, full-stop and a space.\n",
    "article_text = re.sub(r'[^A-Za-z. ]', '', article_text)\n",
    "\n",
    "len(article_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Simple N-Gram Model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\nsser\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pages for logged\n",
      "for logged out\n",
      "logged out editors\n",
      "out editors learn\n",
      "editors learn moredeep\n",
      "learn moredeep learning\n",
      "moredeep learning is\n",
      "learning is part\n",
      "is part of\n",
      "part of a\n",
      "of a broader\n",
      "a broader family\n",
      "broader family of\n",
      "family of machine\n",
      "of machine learning\n",
      "machine learning methods\n",
      "learning methods based\n",
      "methods based on\n",
      "based on artificial\n",
      "on artificial neural\n",
      "artificial neural networks\n",
      "neural networks with\n",
      "networks with representation\n",
      "with representation learning\n",
      "representation learning .\n",
      "learning . learning\n",
      ". learning can\n",
      "learning can be\n",
      "can be supervised\n",
      "be supervised semisupervised\n",
      "supervised semisupervised or\n",
      "semisupervised or unsupervised.deeplearning\n",
      "or unsupervised.deeplearning architectures\n",
      "unsupervised.deeplearning architectures such\n",
      "architectures such as\n",
      "such as deep\n",
      "as deep neural\n",
      "deep neural networks\n",
      "neural networks deep\n",
      "networks deep belief\n",
      "deep belief networks\n",
      "belief networks deep\n",
      "networks deep reinforcement\n",
      "deep reinforcement learning\n",
      "reinforcement learning recurrent\n",
      "learning recurrent neural\n",
      "recurrent neural networks\n",
      "neural networks convolutional\n",
      "networks convolutional neural\n",
      "convolutional neural networks\n",
      "neural networks and\n",
      "networks and transformers\n",
      "and transformers have\n",
      "transformers have been\n",
      "have been applied\n",
      "been applied to\n",
      "applied to fields\n",
      "to fields including\n",
      "fields including computer\n",
      "including computer vision\n",
      "computer vision speech\n",
      "vision speech recognition\n",
      "speech recognition natural\n",
      "recognition natural language\n",
      "natural language processing\n",
      "language processing machine\n",
      "processing machine translation\n",
      "machine translation bioinformatics\n",
      "translation bioinformatics drug\n",
      "bioinformatics drug design\n",
      "drug design medical\n",
      "design medical image\n",
      "medical image analysis\n",
      "image analysis climate\n",
      "analysis climate science\n",
      "climate science material\n",
      "science material inspection\n",
      "material inspection and\n",
      "inspection and board\n",
      "and board game\n",
      "board game programs\n",
      "game programs where\n",
      "programs where they\n",
      "where they have\n",
      "they have produced\n",
      "have produced results\n",
      "produced results comparable\n",
      "results comparable to\n",
      "comparable to and\n",
      "to and in\n",
      "and in some\n",
      "in some cases\n",
      "some cases surpassing\n",
      "cases surpassing human\n",
      "surpassing human expert\n",
      "human expert performance.artificial\n",
      "expert performance.artificial neural\n",
      "performance.artificial neural networks\n",
      "neural networks anns\n",
      "networks anns were\n",
      "anns were inspired\n",
      "were inspired by\n",
      "inspired by information\n",
      "by information processing\n",
      "information processing and\n",
      "processing and distributed\n",
      "and distributed communication\n",
      "distributed communication nodes\n",
      "communication nodes in\n",
      "nodes in biological\n",
      "in biological systems\n",
      "biological systems .\n",
      "systems . anns\n",
      ". anns have\n",
      "anns have various\n",
      "have various differences\n",
      "various differences from\n",
      "differences from biological\n",
      "from biological brains\n",
      "biological brains .\n",
      "brains . specifically\n",
      ". specifically artificial\n",
      "specifically artificial neural\n",
      "artificial neural networks\n",
      "neural networks tend\n",
      "networks tend to\n",
      "tend to be\n",
      "to be static\n",
      "be static and\n",
      "static and symbolic\n",
      "and symbolic while\n",
      "symbolic while the\n",
      "while the biological\n",
      "the biological brain\n",
      "biological brain of\n",
      "brain of most\n",
      "of most living\n",
      "most living organisms\n",
      "living organisms is\n",
      "organisms is dynamic\n",
      "is dynamic plastic\n",
      "dynamic plastic and\n",
      "plastic and analog.the\n",
      "and analog.the adjective\n",
      "analog.the adjective deep\n",
      "adjective deep in\n",
      "deep in deep\n",
      "in deep learning\n",
      "deep learning refers\n",
      "learning refers to\n",
      "refers to the\n",
      "to the use\n",
      "the use of\n",
      "use of multiple\n",
      "of multiple layers\n",
      "multiple layers in\n",
      "layers in the\n",
      "in the network\n",
      "the network .\n",
      "network . early\n",
      ". early work\n",
      "early work showed\n",
      "work showed that\n",
      "showed that a\n",
      "that a linear\n",
      "a linear perceptron\n",
      "linear perceptron can\n",
      "perceptron can not\n",
      "can not be\n",
      "not be a\n",
      "be a universal\n",
      "a universal classifier\n",
      "universal classifier but\n",
      "classifier but that\n",
      "but that a\n",
      "that a network\n",
      "a network with\n",
      "network with a\n",
      "with a nonpolynomial\n",
      "a nonpolynomial activation\n",
      "nonpolynomial activation function\n",
      "activation function with\n",
      "function with one\n",
      "with one hidden\n",
      "one hidden layer\n",
      "hidden layer of\n",
      "layer of unbounded\n",
      "of unbounded width\n",
      "unbounded width can\n",
      "width can .\n",
      "can . deep\n",
      ". deep learning\n",
      "deep learning is\n",
      "learning is a\n",
      "is a modern\n",
      "a modern variation\n",
      "modern variation that\n",
      "variation that is\n",
      "that is concerned\n",
      "is concerned with\n",
      "concerned with an\n",
      "with an unbounded\n",
      "an unbounded number\n",
      "unbounded number of\n",
      "number of layers\n",
      "of layers of\n",
      "layers of bounded\n",
      "of bounded size\n",
      "bounded size which\n",
      "size which permits\n",
      "which permits practical\n",
      "permits practical application\n",
      "practical application and\n",
      "application and optimized\n",
      "and optimized implementation\n",
      "optimized implementation while\n",
      "implementation while retaining\n",
      "while retaining theoretical\n",
      "retaining theoretical universality\n",
      "theoretical universality under\n",
      "universality under mild\n",
      "under mild conditions\n",
      "mild conditions .\n",
      "conditions . in\n",
      ". in deep\n",
      "in deep learning\n",
      "deep learning the\n",
      "learning the layers\n",
      "the layers are\n",
      "layers are also\n",
      "are also permitted\n",
      "also permitted to\n",
      "permitted to be\n",
      "to be heterogeneous\n",
      "be heterogeneous and\n",
      "heterogeneous and to\n",
      "and to deviate\n",
      "to deviate widely\n",
      "deviate widely from\n",
      "widely from biologically\n",
      "from biologically informed\n",
      "biologically informed connectionist\n",
      "informed connectionist models\n",
      "connectionist models for\n",
      "models for the\n",
      "for the sake\n",
      "the sake of\n",
      "sake of efficiency\n",
      "of efficiency trainability\n",
      "efficiency trainability and\n",
      "trainability and understandability.deep\n",
      "and understandability.deep learning\n",
      "understandability.deep learning is\n",
      "learning is a\n",
      "is a class\n",
      "a class of\n",
      "class of machine\n",
      "of machine learning\n",
      "machine learning algorithms\n",
      "learning algorithms that\n",
      "algorithms that uses\n",
      "that uses multiple\n",
      "uses multiple layers\n",
      "multiple layers to\n",
      "layers to progressively\n",
      "to progressively extract\n",
      "progressively extract higherlevel\n",
      "extract higherlevel features\n",
      "higherlevel features from\n",
      "features from the\n",
      "from the raw\n",
      "the raw input\n",
      "raw input .\n",
      "input . for\n",
      ". for example\n",
      "for example in\n",
      "example in image\n",
      "in image processing\n",
      "image processing lower\n",
      "processing lower layers\n",
      "lower layers may\n",
      "layers may identify\n",
      "may identify edges\n",
      "identify edges while\n",
      "edges while higher\n",
      "while higher layers\n",
      "higher layers may\n",
      "layers may identify\n",
      "may identify the\n",
      "identify the concepts\n",
      "the concepts relevant\n",
      "concepts relevant to\n",
      "relevant to a\n",
      "to a human\n",
      "a human such\n",
      "human such as\n",
      "such as digits\n",
      "as digits or\n",
      "digits or letters\n",
      "or letters or\n",
      "letters or faces.most\n",
      "or faces.most modern\n",
      "faces.most modern deep\n",
      "modern deep learning\n",
      "deep learning models\n",
      "learning models are\n",
      "models are based\n",
      "are based on\n",
      "based on artificial\n",
      "on artificial neural\n",
      "artificial neural networks\n",
      "neural networks specifically\n",
      "networks specifically convolutional\n",
      "specifically convolutional neural\n",
      "convolutional neural networks\n",
      "neural networks cnns\n",
      "networks cnns although\n",
      "cnns although they\n",
      "although they can\n",
      "they can also\n",
      "can also include\n",
      "also include propositional\n",
      "include propositional formulas\n",
      "propositional formulas or\n",
      "formulas or latent\n",
      "or latent variables\n",
      "latent variables organized\n",
      "variables organized layerwise\n",
      "organized layerwise in\n",
      "layerwise in deep\n",
      "in deep generative\n",
      "deep generative models\n",
      "generative models such\n",
      "models such as\n",
      "such as the\n",
      "as the nodes\n",
      "the nodes in\n",
      "nodes in deep\n",
      "in deep belief\n",
      "deep belief networks\n",
      "belief networks and\n",
      "networks and deep\n",
      "and deep boltzmann\n",
      "deep boltzmann machines.in\n",
      "boltzmann machines.in deep\n",
      "machines.in deep learning\n",
      "deep learning each\n",
      "learning each level\n",
      "each level learns\n",
      "level learns to\n",
      "learns to transform\n",
      "to transform its\n",
      "transform its input\n",
      "its input data\n",
      "input data into\n",
      "data into a\n",
      "into a slightly\n",
      "a slightly more\n",
      "slightly more abstract\n",
      "more abstract and\n",
      "abstract and composite\n",
      "and composite representation\n",
      "composite representation .\n",
      "representation . in\n",
      ". in an\n",
      "in an image\n",
      "an image recognition\n",
      "image recognition application\n",
      "recognition application the\n",
      "application the raw\n",
      "the raw input\n",
      "raw input may\n",
      "input may be\n",
      "may be a\n",
      "be a matrix\n",
      "a matrix of\n",
      "matrix of pixels\n",
      "of pixels the\n",
      "pixels the first\n",
      "the first representational\n",
      "first representational layer\n",
      "representational layer may\n",
      "layer may abstract\n",
      "may abstract the\n",
      "abstract the pixels\n",
      "the pixels and\n",
      "pixels and encode\n",
      "and encode edges\n",
      "encode edges the\n",
      "edges the second\n",
      "the second layer\n",
      "second layer may\n",
      "layer may compose\n",
      "may compose and\n",
      "compose and encode\n",
      "and encode arrangements\n",
      "encode arrangements of\n",
      "arrangements of edges\n",
      "of edges the\n",
      "edges the third\n",
      "the third layer\n",
      "third layer may\n",
      "layer may encode\n",
      "may encode a\n",
      "encode a nose\n",
      "a nose and\n",
      "nose and eyes\n",
      "and eyes and\n",
      "eyes and the\n",
      "and the fourth\n",
      "the fourth layer\n",
      "fourth layer may\n",
      "layer may recognize\n",
      "may recognize that\n",
      "recognize that the\n",
      "that the image\n",
      "the image contains\n",
      "image contains a\n",
      "contains a face\n",
      "a face .\n",
      "face . importantly\n",
      ". importantly a\n",
      "importantly a deep\n",
      "a deep learning\n",
      "deep learning process\n",
      "learning process can\n",
      "process can learn\n",
      "can learn which\n",
      "learn which features\n",
      "which features to\n",
      "features to optimally\n",
      "to optimally place\n",
      "optimally place in\n",
      "place in which\n",
      "in which level\n",
      "which level on\n",
      "level on its\n",
      "on its own\n",
      "its own .\n",
      "own . this\n",
      ". this does\n",
      "this does not\n",
      "does not eliminate\n",
      "not eliminate the\n",
      "eliminate the need\n",
      "the need for\n",
      "need for handtuning\n",
      "for handtuning for\n",
      "handtuning for example\n",
      "for example varying\n",
      "example varying numbers\n",
      "varying numbers of\n",
      "numbers of layers\n",
      "of layers and\n",
      "layers and layer\n",
      "and layer sizes\n",
      "layer sizes can\n",
      "sizes can provide\n",
      "can provide different\n",
      "provide different degrees\n",
      "different degrees of\n",
      "degrees of abstraction.the\n",
      "of abstraction.the word\n",
      "abstraction.the word deep\n",
      "word deep in\n",
      "deep in deep\n",
      "in deep learning\n",
      "deep learning refers\n",
      "learning refers to\n",
      "refers to the\n",
      "to the number\n",
      "the number of\n",
      "number of layers\n",
      "of layers through\n",
      "layers through which\n",
      "through which the\n",
      "which the data\n",
      "the data is\n",
      "data is transformed\n",
      "is transformed .\n",
      "transformed . more\n",
      ". more precisely\n",
      "more precisely deep\n",
      "precisely deep learning\n",
      "deep learning systems\n",
      "learning systems have\n",
      "systems have a\n",
      "have a substantial\n",
      "a substantial credit\n",
      "substantial credit assignment\n",
      "credit assignment path\n",
      "assignment path cap\n",
      "path cap depth\n",
      "cap depth .\n",
      "depth . the\n",
      ". the cap\n",
      "the cap is\n",
      "cap is the\n",
      "is the chain\n",
      "the chain of\n",
      "chain of transformations\n",
      "of transformations from\n",
      "transformations from input\n",
      "from input to\n",
      "input to output\n",
      "to output .\n",
      "output . caps\n",
      ". caps describe\n",
      "caps describe potentially\n",
      "describe potentially causal\n",
      "potentially causal connections\n",
      "causal connections between\n",
      "connections between input\n",
      "between input and\n",
      "input and output\n",
      "and output .\n",
      "output . for\n",
      ". for a\n",
      "for a feedforward\n",
      "a feedforward neural\n",
      "feedforward neural network\n",
      "neural network the\n",
      "network the depth\n",
      "the depth of\n",
      "depth of the\n",
      "of the caps\n",
      "the caps is\n",
      "caps is that\n",
      "is that of\n",
      "that of the\n",
      "of the network\n",
      "the network and\n",
      "network and is\n",
      "and is the\n",
      "is the number\n",
      "the number of\n",
      "number of hidden\n",
      "of hidden layers\n",
      "hidden layers plus\n",
      "layers plus one\n",
      "plus one as\n",
      "one as the\n",
      "as the output\n",
      "the output layer\n",
      "output layer is\n",
      "layer is also\n",
      "is also parameterized\n",
      "also parameterized .\n",
      "parameterized . for\n",
      ". for recurrent\n",
      "for recurrent neural\n",
      "recurrent neural networks\n",
      "neural networks in\n",
      "networks in which\n",
      "in which a\n",
      "which a signal\n",
      "a signal may\n",
      "signal may propagate\n",
      "may propagate through\n",
      "propagate through a\n",
      "through a layer\n",
      "a layer more\n",
      "layer more than\n",
      "more than once\n",
      "than once the\n",
      "once the cap\n",
      "the cap depth\n",
      "cap depth is\n",
      "depth is potentially\n",
      "is potentially unlimited\n",
      "potentially unlimited .\n",
      "unlimited . no\n",
      ". no universally\n",
      "no universally agreedupon\n",
      "universally agreedupon threshold\n",
      "agreedupon threshold of\n",
      "threshold of depth\n",
      "of depth divides\n",
      "depth divides shallow\n",
      "divides shallow learning\n",
      "shallow learning from\n",
      "learning from deep\n",
      "from deep learning\n",
      "deep learning but\n",
      "learning but most\n",
      "but most researchers\n",
      "most researchers agree\n",
      "researchers agree that\n",
      "agree that deep\n",
      "that deep learning\n",
      "deep learning involves\n",
      "learning involves cap\n",
      "involves cap depth\n",
      "cap depth higher\n",
      "depth higher than\n",
      "higher than .\n",
      "than . cap\n",
      ". cap of\n",
      "cap of depth\n",
      "of depth has\n",
      "depth has been\n",
      "has been shown\n",
      "been shown to\n",
      "shown to be\n",
      "to be a\n",
      "be a universal\n",
      "a universal approximator\n",
      "universal approximator in\n",
      "approximator in the\n",
      "in the sense\n",
      "the sense that\n",
      "sense that it\n",
      "that it can\n",
      "it can emulate\n",
      "can emulate any\n",
      "emulate any function\n",
      "any function .\n",
      "function . beyond\n",
      ". beyond that\n",
      "beyond that more\n",
      "that more layers\n",
      "more layers do\n",
      "layers do not\n",
      "do not add\n",
      "not add to\n",
      "add to the\n",
      "to the function\n",
      "the function approximator\n",
      "function approximator ability\n",
      "approximator ability of\n",
      "ability of the\n",
      "of the network\n",
      "the network .\n",
      "network . deep\n",
      ". deep models\n",
      "deep models cap\n",
      "models cap are\n",
      "cap are able\n",
      "are able to\n",
      "able to extract\n",
      "to extract better\n",
      "extract better features\n",
      "better features than\n",
      "features than shallow\n",
      "than shallow models\n",
      "shallow models and\n",
      "models and hence\n",
      "and hence extra\n",
      "hence extra layers\n",
      "extra layers help\n",
      "layers help in\n",
      "help in learning\n",
      "in learning the\n",
      "learning the features\n",
      "the features effectively.deep\n",
      "features effectively.deep learning\n",
      "effectively.deep learning architectures\n",
      "learning architectures can\n",
      "architectures can be\n",
      "can be constructed\n",
      "be constructed with\n",
      "constructed with a\n",
      "with a greedy\n",
      "a greedy layerbylayer\n",
      "greedy layerbylayer method\n",
      "layerbylayer method .\n",
      "method . deep\n",
      ". deep learning\n",
      "deep learning helps\n",
      "learning helps to\n",
      "helps to disentangle\n",
      "to disentangle these\n",
      "disentangle these abstractions\n",
      "these abstractions and\n",
      "abstractions and pick\n",
      "and pick out\n",
      "pick out which\n",
      "out which features\n",
      "which features improve\n",
      "features improve performance.for\n",
      "improve performance.for supervised\n",
      "performance.for supervised learning\n",
      "supervised learning tasks\n",
      "learning tasks deep\n",
      "tasks deep learning\n",
      "deep learning methods\n",
      "learning methods eliminate\n",
      "methods eliminate feature\n",
      "eliminate feature engineering\n",
      "feature engineering by\n",
      "engineering by translating\n",
      "by translating the\n",
      "translating the data\n",
      "the data into\n",
      "data into compact\n",
      "into compact intermediate\n",
      "compact intermediate representations\n",
      "intermediate representations akin\n",
      "representations akin to\n",
      "akin to principal\n",
      "to principal components\n",
      "principal components and\n",
      "components and derive\n",
      "and derive layered\n",
      "derive layered structures\n",
      "layered structures that\n",
      "structures that remove\n",
      "that remove redundancy\n",
      "remove redundancy in\n",
      "redundancy in representation.deep\n",
      "in representation.deep learning\n",
      "representation.deep learning algorithms\n",
      "learning algorithms can\n",
      "algorithms can be\n",
      "can be applied\n",
      "be applied to\n",
      "applied to unsupervised\n",
      "to unsupervised learning\n",
      "unsupervised learning tasks\n",
      "learning tasks .\n",
      "tasks . this\n",
      ". this is\n",
      "this is an\n",
      "is an important\n",
      "an important benefit\n",
      "important benefit because\n",
      "benefit because unlabeled\n",
      "because unlabeled data\n",
      "unlabeled data are\n",
      "data are more\n",
      "are more abundant\n",
      "more abundant than\n",
      "abundant than the\n",
      "than the labeled\n",
      "the labeled data\n",
      "labeled data .\n",
      "data . examples\n",
      ". examples of\n",
      "examples of deep\n",
      "of deep structures\n",
      "deep structures that\n",
      "structures that can\n",
      "that can be\n",
      "can be trained\n",
      "be trained in\n",
      "trained in an\n",
      "in an unsupervised\n",
      "an unsupervised manner\n",
      "unsupervised manner are\n",
      "manner are deep\n",
      "are deep belief\n",
      "deep belief networks.deep\n",
      "belief networks.deep neural\n",
      "networks.deep neural networks\n",
      "neural networks are\n",
      "networks are generally\n",
      "are generally interpreted\n",
      "generally interpreted in\n",
      "interpreted in terms\n",
      "in terms of\n",
      "terms of the\n",
      "of the universal\n",
      "the universal approximation\n",
      "universal approximation theorem\n",
      "approximation theorem or\n",
      "theorem or probabilistic\n",
      "or probabilistic inference.the\n",
      "probabilistic inference.the classic\n",
      "inference.the classic universal\n",
      "classic universal approximation\n",
      "universal approximation theorem\n",
      "approximation theorem concerns\n",
      "theorem concerns the\n",
      "concerns the capacity\n",
      "the capacity of\n",
      "capacity of feedforward\n",
      "of feedforward neural\n",
      "feedforward neural networks\n",
      "neural networks with\n",
      "networks with a\n",
      "with a single\n",
      "a single hidden\n",
      "single hidden layer\n",
      "hidden layer of\n",
      "layer of finite\n",
      "of finite size\n",
      "finite size to\n",
      "size to approximate\n",
      "to approximate continuous\n",
      "approximate continuous functions\n",
      "continuous functions .\n",
      "functions . in\n",
      ". in the\n",
      "in the first\n",
      "the first proof\n",
      "first proof was\n",
      "proof was published\n",
      "was published by\n",
      "published by george\n",
      "by george cybenko\n",
      "george cybenko for\n",
      "cybenko for sigmoid\n",
      "for sigmoid activation\n",
      "sigmoid activation functions\n",
      "activation functions and\n",
      "functions and was\n",
      "and was generalised\n",
      "was generalised to\n",
      "generalised to feedforward\n",
      "to feedforward multilayer\n",
      "feedforward multilayer architectures\n",
      "multilayer architectures in\n",
      "architectures in by\n",
      "in by kurt\n",
      "by kurt hornik\n",
      "kurt hornik .\n",
      "hornik . recent\n",
      ". recent work\n",
      "recent work also\n",
      "work also showed\n",
      "also showed that\n",
      "showed that universal\n",
      "that universal approximation\n",
      "universal approximation also\n",
      "approximation also holds\n",
      "also holds for\n",
      "holds for nonbounded\n",
      "for nonbounded activation\n",
      "nonbounded activation functions\n",
      "activation functions such\n",
      "functions such as\n",
      "such as the\n",
      "as the rectified\n",
      "the rectified linear\n",
      "rectified linear unit.the\n",
      "linear unit.the universal\n",
      "unit.the universal approximation\n",
      "universal approximation theorem\n",
      "approximation theorem for\n",
      "theorem for deep\n",
      "for deep neural\n",
      "deep neural networks\n",
      "neural networks concerns\n",
      "networks concerns the\n",
      "concerns the capacity\n",
      "the capacity of\n",
      "capacity of networks\n",
      "of networks with\n",
      "networks with bounded\n",
      "with bounded width\n",
      "bounded width but\n",
      "width but the\n",
      "but the depth\n",
      "the depth is\n",
      "depth is allowed\n",
      "is allowed to\n",
      "allowed to grow\n",
      "to grow .\n",
      "grow . lu\n",
      ". lu et\n",
      "lu et al\n",
      "et al .\n",
      "al . proved\n",
      ". proved that\n",
      "proved that if\n",
      "that if the\n",
      "if the width\n",
      "the width of\n",
      "width of a\n",
      "of a deep\n",
      "a deep neural\n",
      "deep neural network\n",
      "neural network with\n",
      "network with relu\n",
      "with relu activation\n",
      "relu activation is\n",
      "activation is strictly\n",
      "is strictly larger\n",
      "strictly larger than\n",
      "larger than the\n",
      "than the input\n",
      "the input dimension\n",
      "input dimension then\n",
      "dimension then the\n",
      "then the network\n",
      "the network can\n",
      "network can approximate\n",
      "can approximate any\n",
      "approximate any lebesgue\n",
      "any lebesgue integrable\n",
      "lebesgue integrable function\n",
      "integrable function if\n",
      "function if the\n",
      "if the width\n",
      "the width is\n",
      "width is smaller\n",
      "is smaller or\n",
      "smaller or equal\n",
      "or equal to\n",
      "equal to the\n",
      "to the input\n",
      "the input dimension\n",
      "input dimension then\n",
      "dimension then a\n",
      "then a deep\n",
      "a deep neural\n",
      "deep neural network\n",
      "neural network is\n",
      "network is not\n",
      "is not a\n",
      "not a universal\n",
      "a universal approximator.the\n",
      "universal approximator.the probabilistic\n",
      "approximator.the probabilistic interpretation\n",
      "probabilistic interpretation derives\n",
      "interpretation derives from\n",
      "derives from the\n",
      "from the field\n",
      "the field of\n",
      "field of machine\n",
      "of machine learning\n",
      "machine learning .\n",
      "learning . it\n",
      ". it features\n",
      "it features inference\n",
      "features inference as\n",
      "inference as well\n",
      "as well as\n",
      "well as the\n",
      "as the optimization\n",
      "the optimization concepts\n",
      "optimization concepts of\n",
      "concepts of training\n",
      "of training and\n",
      "training and testing\n",
      "and testing related\n",
      "testing related to\n",
      "related to fitting\n",
      "to fitting and\n",
      "fitting and generalization\n",
      "and generalization respectively\n",
      "generalization respectively .\n",
      "respectively . more\n",
      ". more specifically\n",
      "more specifically the\n",
      "specifically the probabilistic\n",
      "the probabilistic interpretation\n",
      "probabilistic interpretation considers\n",
      "interpretation considers the\n",
      "considers the activation\n",
      "the activation nonlinearity\n",
      "activation nonlinearity as\n",
      "nonlinearity as a\n",
      "as a cumulative\n",
      "a cumulative distribution\n",
      "cumulative distribution function\n",
      "distribution function .\n",
      "function . the\n",
      ". the probabilistic\n",
      "the probabilistic interpretation\n",
      "probabilistic interpretation led\n",
      "interpretation led to\n",
      "led to the\n",
      "to the introduction\n",
      "the introduction of\n",
      "introduction of dropout\n",
      "of dropout as\n",
      "dropout as regularizer\n",
      "as regularizer in\n",
      "regularizer in neural\n",
      "in neural networks\n",
      "neural networks .\n",
      "networks . the\n",
      ". the probabilistic\n",
      "the probabilistic interpretation\n",
      "probabilistic interpretation was\n",
      "interpretation was introduced\n",
      "was introduced by\n",
      "introduced by researchers\n",
      "by researchers including\n",
      "researchers including hopfield\n",
      "including hopfield widrow\n",
      "hopfield widrow and\n",
      "widrow and narendra\n",
      "and narendra and\n",
      "narendra and popularized\n",
      "and popularized in\n",
      "popularized in surveys\n",
      "in surveys such\n",
      "surveys such as\n",
      "such as the\n",
      "as the one\n",
      "the one by\n",
      "one by bishop.some\n",
      "by bishop.some sources\n",
      "bishop.some sources point\n",
      "sources point out\n",
      "point out that\n",
      "out that frank\n",
      "that frank rosenblatt\n",
      "frank rosenblatt developed\n",
      "rosenblatt developed and\n",
      "developed and explored\n",
      "and explored all\n",
      "explored all of\n",
      "all of the\n",
      "of the basic\n",
      "the basic ingredients\n",
      "basic ingredients of\n",
      "ingredients of the\n",
      "of the deep\n",
      "the deep learning\n",
      "deep learning systems\n",
      "learning systems of\n",
      "systems of today\n",
      "of today .\n",
      "today . he\n",
      ". he described\n",
      "he described it\n",
      "described it in\n",
      "it in his\n",
      "in his book\n",
      "his book principles\n",
      "book principles of\n",
      "principles of neurodynamics\n",
      "of neurodynamics perceptrons\n",
      "neurodynamics perceptrons and\n",
      "perceptrons and the\n",
      "and the theory\n",
      "the theory of\n",
      "theory of brain\n",
      "of brain mechanisms\n",
      "brain mechanisms published\n",
      "mechanisms published by\n",
      "published by cornell\n",
      "by cornell aeronautical\n",
      "cornell aeronautical laboratory\n",
      "aeronautical laboratory inc.\n",
      "laboratory inc. cornell\n",
      "inc. cornell university\n",
      "cornell university in\n",
      "university in .the\n",
      "in .the first\n",
      ".the first general\n",
      "first general working\n",
      "general working learning\n",
      "working learning algorithm\n",
      "learning algorithm for\n",
      "algorithm for supervised\n",
      "for supervised deep\n",
      "supervised deep feedforward\n",
      "deep feedforward multilayer\n",
      "feedforward multilayer perceptrons\n",
      "multilayer perceptrons was\n",
      "perceptrons was published\n",
      "was published by\n",
      "published by alexey\n",
      "by alexey ivakhnenko\n",
      "alexey ivakhnenko and\n",
      "ivakhnenko and lapa\n",
      "and lapa in\n",
      "lapa in .\n",
      "in . a\n",
      ". a paper\n",
      "a paper described\n",
      "paper described a\n",
      "described a deep\n",
      "a deep network\n",
      "deep network with\n",
      "network with eight\n",
      "with eight layers\n",
      "eight layers trained\n",
      "layers trained by\n",
      "trained by the\n",
      "by the group\n",
      "the group method\n",
      "group method of\n",
      "method of data\n",
      "of data handling\n",
      "data handling .\n",
      "handling . other\n",
      ". other deep\n",
      "other deep learning\n",
      "deep learning working\n",
      "learning working architectures\n",
      "working architectures specifically\n",
      "architectures specifically those\n",
      "specifically those built\n",
      "those built for\n",
      "built for computer\n",
      "for computer vision\n",
      "computer vision began\n",
      "vision began with\n",
      "began with the\n",
      "with the neocognitron\n",
      "the neocognitron introduced\n",
      "neocognitron introduced by\n",
      "introduced by kunihiko\n",
      "by kunihiko fukushima\n",
      "kunihiko fukushima in\n",
      "fukushima in .the\n",
      "in .the term\n",
      ".the term deep\n",
      "term deep learning\n",
      "deep learning was\n",
      "learning was introduced\n",
      "was introduced to\n",
      "introduced to the\n",
      "to the machine\n",
      "the machine learning\n",
      "machine learning community\n",
      "learning community by\n",
      "community by rina\n",
      "by rina dechter\n",
      "rina dechter in\n",
      "dechter in and\n",
      "in and to\n",
      "and to artificial\n",
      "to artificial neural\n",
      "artificial neural networks\n",
      "neural networks by\n",
      "networks by igor\n",
      "by igor aizenberg\n",
      "igor aizenberg and\n",
      "aizenberg and colleagues\n",
      "and colleagues in\n",
      "colleagues in in\n",
      "in in the\n",
      "in the context\n",
      "the context of\n",
      "context of boolean\n",
      "of boolean threshold\n",
      "boolean threshold neurons.in\n",
      "threshold neurons.in yann\n",
      "neurons.in yann lecun\n",
      "yann lecun et\n",
      "lecun et al\n",
      "et al .\n",
      "al . applied\n",
      ". applied the\n",
      "applied the standard\n",
      "the standard backpropagation\n",
      "standard backpropagation algorithm\n",
      "backpropagation algorithm which\n",
      "algorithm which had\n",
      "which had been\n",
      "had been around\n",
      "been around as\n",
      "around as the\n",
      "as the reverse\n",
      "the reverse mode\n",
      "reverse mode of\n",
      "mode of automatic\n",
      "of automatic differentiation\n",
      "automatic differentiation since\n",
      "differentiation since to\n",
      "since to a\n",
      "to a deep\n",
      "a deep neural\n",
      "deep neural network\n",
      "neural network with\n",
      "network with the\n",
      "with the purpose\n",
      "the purpose of\n",
      "purpose of recognizing\n",
      "of recognizing handwritten\n",
      "recognizing handwritten zip\n",
      "handwritten zip codes\n",
      "zip codes on\n",
      "codes on mail\n",
      "on mail .\n",
      "mail . while\n",
      ". while the\n",
      "while the algorithm\n",
      "the algorithm worked\n",
      "algorithm worked training\n",
      "worked training required\n",
      "training required days.independently\n",
      "required days.independently in\n",
      "days.independently in wei\n",
      "in wei zhang\n",
      "wei zhang et\n",
      "zhang et al\n",
      "et al .\n",
      "al . applied\n",
      ". applied the\n",
      "applied the backpropagation\n",
      "the backpropagation algorithm\n",
      "backpropagation algorithm to\n",
      "algorithm to a\n",
      "to a convolutional\n",
      "a convolutional neural\n",
      "convolutional neural network\n",
      "neural network a\n",
      "network a simplified\n",
      "a simplified neocognitron\n",
      "simplified neocognitron by\n",
      "neocognitron by keeping\n",
      "by keeping only\n",
      "keeping only the\n",
      "only the convolutional\n",
      "the convolutional interconnections\n",
      "convolutional interconnections between\n",
      "interconnections between the\n",
      "between the image\n",
      "the image feature\n",
      "image feature layers\n",
      "feature layers and\n",
      "layers and the\n",
      "and the last\n",
      "the last fully\n",
      "last fully connected\n",
      "fully connected layer\n",
      "connected layer for\n",
      "layer for alphabets\n",
      "for alphabets recognition\n",
      "alphabets recognition and\n",
      "recognition and also\n",
      "and also proposed\n",
      "also proposed an\n",
      "proposed an implementation\n",
      "an implementation of\n",
      "implementation of the\n",
      "of the cnn\n",
      "the cnn with\n",
      "cnn with an\n",
      "with an optical\n",
      "an optical computing\n",
      "optical computing system\n",
      "computing system .\n",
      "system . subsequently\n",
      ". subsequently wei\n",
      "subsequently wei zhang\n",
      "wei zhang et\n",
      "zhang et al\n",
      "et al .\n",
      "al . modified\n",
      ". modified the\n",
      "modified the model\n",
      "the model by\n",
      "model by removing\n",
      "by removing the\n",
      "removing the last\n",
      "the last fully\n",
      "last fully connected\n",
      "fully connected layer\n",
      "connected layer and\n",
      "layer and applied\n",
      "and applied it\n",
      "applied it for\n",
      "it for medical\n",
      "for medical image\n",
      "medical image object\n",
      "image object segmentation\n",
      "object segmentation in\n",
      "segmentation in and\n",
      "in and breast\n",
      "and breast cancer\n",
      "breast cancer detection\n",
      "cancer detection in\n",
      "detection in mammograms\n",
      "in mammograms in\n",
      "mammograms in .in\n",
      "in .in andr\n",
      ".in andr de\n",
      "andr de carvalho\n",
      "de carvalho together\n",
      "carvalho together with\n",
      "together with mike\n",
      "with mike fairhurst\n",
      "mike fairhurst and\n",
      "fairhurst and david\n",
      "and david bisset\n",
      "david bisset published\n",
      "bisset published experimental\n",
      "published experimental results\n",
      "experimental results of\n",
      "results of a\n",
      "of a multilayer\n",
      "a multilayer boolean\n",
      "multilayer boolean neural\n",
      "boolean neural network\n",
      "neural network also\n",
      "network also known\n",
      "also known as\n",
      "known as a\n",
      "as a weightless\n",
      "a weightless neural\n",
      "weightless neural network\n",
      "neural network composed\n",
      "network composed of\n",
      "composed of a\n",
      "of a layers\n",
      "a layers selforganising\n",
      "layers selforganising feature\n",
      "selforganising feature extraction\n",
      "feature extraction neural\n",
      "extraction neural network\n",
      "neural network module\n",
      "network module soft\n",
      "module soft followed\n",
      "soft followed by\n",
      "followed by a\n",
      "by a multilayer\n",
      "a multilayer classification\n",
      "multilayer classification neural\n",
      "classification neural network\n",
      "neural network module\n",
      "network module gsn\n",
      "module gsn which\n",
      "gsn which were\n",
      "which were independently\n",
      "were independently trained\n",
      "independently trained .\n",
      "trained . each\n",
      ". each layer\n",
      "each layer in\n",
      "layer in the\n",
      "in the feature\n",
      "the feature extraction\n",
      "feature extraction module\n",
      "extraction module extracted\n",
      "module extracted features\n",
      "extracted features with\n",
      "features with growing\n",
      "with growing complexity\n",
      "growing complexity regarding\n",
      "complexity regarding the\n",
      "regarding the previous\n",
      "the previous layer.in\n",
      "previous layer.in brendan\n",
      "layer.in brendan frey\n",
      "brendan frey demonstrated\n",
      "frey demonstrated that\n",
      "demonstrated that it\n",
      "that it was\n",
      "it was possible\n",
      "was possible to\n",
      "possible to train\n",
      "to train over\n",
      "train over two\n",
      "over two days\n",
      "two days a\n",
      "days a network\n",
      "a network containing\n",
      "network containing six\n",
      "containing six fully\n",
      "six fully connected\n",
      "fully connected layers\n",
      "connected layers and\n",
      "layers and several\n",
      "and several hundred\n",
      "several hundred hidden\n",
      "hundred hidden units\n",
      "hidden units using\n",
      "units using the\n",
      "using the wakesleep\n",
      "the wakesleep algorithm\n",
      "wakesleep algorithm codeveloped\n",
      "algorithm codeveloped with\n",
      "codeveloped with peter\n",
      "with peter dayan\n",
      "peter dayan and\n",
      "dayan and hinton\n",
      "and hinton .\n",
      "hinton . many\n",
      ". many factors\n",
      "many factors contribute\n",
      "factors contribute to\n",
      "contribute to the\n",
      "to the slow\n",
      "the slow speed\n",
      "slow speed including\n",
      "speed including the\n",
      "including the vanishing\n",
      "the vanishing gradient\n",
      "vanishing gradient problem\n",
      "gradient problem analyzed\n",
      "problem analyzed in\n",
      "analyzed in by\n",
      "in by sepp\n",
      "by sepp hochreiter.since\n",
      "sepp hochreiter.since sven\n",
      "hochreiter.since sven behnke\n",
      "sven behnke extended\n",
      "behnke extended the\n",
      "extended the feedforward\n",
      "the feedforward hierarchical\n",
      "feedforward hierarchical convolutional\n",
      "hierarchical convolutional approach\n",
      "convolutional approach in\n",
      "approach in the\n",
      "in the neural\n",
      "the neural abstraction\n",
      "neural abstraction pyramid\n",
      "abstraction pyramid by\n",
      "pyramid by lateral\n",
      "by lateral and\n",
      "lateral and backward\n",
      "and backward connections\n",
      "backward connections in\n",
      "connections in order\n",
      "in order to\n",
      "order to flexibly\n",
      "to flexibly incorporate\n",
      "flexibly incorporate context\n",
      "incorporate context into\n",
      "context into decisions\n",
      "into decisions and\n",
      "decisions and iteratively\n",
      "and iteratively resolve\n",
      "iteratively resolve local\n",
      "resolve local ambiguities.simpler\n",
      "local ambiguities.simpler models\n",
      "ambiguities.simpler models that\n",
      "models that use\n",
      "that use taskspecific\n",
      "use taskspecific handcrafted\n",
      "taskspecific handcrafted features\n",
      "handcrafted features such\n",
      "features such as\n",
      "such as gabor\n",
      "as gabor filters\n",
      "gabor filters and\n",
      "filters and support\n",
      "and support vector\n",
      "support vector machines\n",
      "vector machines svms\n",
      "machines svms were\n",
      "svms were a\n",
      "were a popular\n",
      "a popular choice\n",
      "popular choice in\n",
      "choice in the\n",
      "in the s\n",
      "the s and\n",
      "s and s\n",
      "and s because\n",
      "s because of\n",
      "because of artificial\n",
      "of artificial neural\n",
      "artificial neural networks\n",
      "neural networks ann\n",
      "networks ann computational\n",
      "ann computational cost\n",
      "computational cost and\n",
      "cost and a\n",
      "and a lack\n",
      "a lack of\n",
      "lack of understanding\n",
      "of understanding of\n",
      "understanding of how\n",
      "of how the\n",
      "how the brain\n",
      "the brain wires\n",
      "brain wires its\n",
      "wires its biological\n",
      "its biological networks.both\n",
      "biological networks.both shallow\n",
      "networks.both shallow and\n",
      "shallow and deep\n",
      "and deep learning\n",
      "deep learning e.g\n",
      "learning e.g .\n",
      "e.g . recurrent\n",
      ". recurrent nets\n",
      "recurrent nets of\n",
      "nets of anns\n",
      "of anns have\n",
      "anns have been\n",
      "have been explored\n",
      "been explored for\n",
      "explored for many\n",
      "for many years\n",
      "many years .\n",
      "years . these\n",
      ". these methods\n",
      "these methods never\n",
      "methods never outperformed\n",
      "never outperformed nonuniform\n",
      "outperformed nonuniform internalhandcrafting\n",
      "nonuniform internalhandcrafting gaussian\n",
      "internalhandcrafting gaussian mixture\n",
      "gaussian mixture modelhidden\n",
      "mixture modelhidden markov\n",
      "modelhidden markov model\n",
      "markov model gmmhmm\n",
      "model gmmhmm technology\n",
      "gmmhmm technology based\n",
      "technology based on\n",
      "based on generative\n",
      "on generative models\n",
      "generative models of\n",
      "models of speech\n",
      "of speech trained\n",
      "speech trained discriminatively\n",
      "trained discriminatively .\n",
      "discriminatively . key\n",
      ". key difficulties\n",
      "key difficulties have\n",
      "difficulties have been\n",
      "have been analyzed\n",
      "been analyzed including\n",
      "analyzed including gradient\n",
      "including gradient diminishing\n",
      "gradient diminishing and\n",
      "diminishing and weak\n",
      "and weak temporal\n",
      "weak temporal correlation\n",
      "temporal correlation structure\n",
      "correlation structure in\n",
      "structure in neural\n",
      "in neural predictive\n",
      "neural predictive models\n",
      "predictive models .\n",
      "models . additional\n",
      ". additional difficulties\n",
      "additional difficulties were\n",
      "difficulties were the\n",
      "were the lack\n",
      "the lack of\n",
      "lack of training\n",
      "of training data\n",
      "training data and\n",
      "data and limited\n",
      "and limited computing\n",
      "limited computing power.most\n",
      "computing power.most speech\n",
      "power.most speech recognition\n",
      "speech recognition researchers\n",
      "recognition researchers moved\n",
      "researchers moved away\n",
      "moved away from\n",
      "away from neural\n",
      "from neural nets\n",
      "neural nets to\n",
      "nets to pursue\n",
      "to pursue generative\n",
      "pursue generative modeling\n",
      "generative modeling .\n",
      "modeling . an\n",
      ". an exception\n",
      "an exception was\n",
      "exception was at\n",
      "was at sri\n",
      "at sri international\n",
      "sri international in\n",
      "international in the\n",
      "in the late\n",
      "the late s.\n",
      "late s. funded\n",
      "s. funded by\n",
      "funded by the\n",
      "by the us\n",
      "the us governments\n",
      "us governments nsa\n",
      "governments nsa and\n",
      "nsa and darpa\n",
      "and darpa sri\n",
      "darpa sri studied\n",
      "sri studied deep\n",
      "studied deep neural\n",
      "deep neural networks\n",
      "neural networks in\n",
      "networks in speech\n",
      "in speech and\n",
      "speech and speaker\n",
      "and speaker recognition\n",
      "speaker recognition .\n",
      "recognition . the\n",
      ". the speaker\n",
      "the speaker recognition\n",
      "speaker recognition team\n",
      "recognition team led\n",
      "team led by\n",
      "led by larry\n",
      "by larry heck\n",
      "larry heck reported\n",
      "heck reported significant\n",
      "reported significant success\n",
      "significant success with\n",
      "success with deep\n",
      "with deep neural\n",
      "deep neural networks\n",
      "neural networks in\n",
      "networks in speech\n",
      "in speech processing\n",
      "speech processing in\n",
      "processing in the\n",
      "in the national\n",
      "the national institute\n",
      "national institute of\n",
      "institute of standards\n",
      "of standards and\n",
      "standards and technology\n",
      "and technology speaker\n",
      "technology speaker recognition\n",
      "speaker recognition evaluation\n",
      "recognition evaluation .\n",
      "evaluation . the\n",
      ". the sri\n",
      "the sri deep\n",
      "sri deep neural\n",
      "deep neural network\n",
      "neural network was\n",
      "network was then\n",
      "was then deployed\n",
      "then deployed in\n",
      "deployed in the\n",
      "in the nuance\n",
      "the nuance verifier\n",
      "nuance verifier representing\n",
      "verifier representing the\n",
      "representing the first\n",
      "the first major\n",
      "first major industrial\n",
      "major industrial application\n",
      "industrial application of\n",
      "application of deep\n",
      "of deep learning.the\n",
      "deep learning.the principle\n",
      "learning.the principle of\n",
      "principle of elevating\n",
      "of elevating raw\n",
      "elevating raw features\n",
      "raw features over\n",
      "features over handcrafted\n",
      "over handcrafted optimization\n",
      "handcrafted optimization was\n",
      "optimization was first\n",
      "was first explored\n",
      "first explored successfully\n",
      "explored successfully in\n",
      "successfully in the\n",
      "in the architecture\n",
      "the architecture of\n",
      "architecture of deep\n",
      "of deep autoencoder\n",
      "deep autoencoder on\n",
      "autoencoder on the\n",
      "on the raw\n",
      "the raw spectrogram\n",
      "raw spectrogram or\n",
      "spectrogram or linear\n",
      "or linear filterbank\n",
      "linear filterbank features\n",
      "filterbank features in\n",
      "features in the\n",
      "in the late\n",
      "the late s\n",
      "late s showing\n",
      "s showing its\n",
      "showing its superiority\n",
      "its superiority over\n",
      "superiority over the\n",
      "over the melcepstral\n",
      "the melcepstral features\n",
      "melcepstral features that\n",
      "features that contain\n",
      "that contain stages\n",
      "contain stages of\n",
      "stages of fixed\n",
      "of fixed transformation\n",
      "fixed transformation from\n",
      "transformation from spectrograms\n",
      "from spectrograms .\n",
      "spectrograms . the\n",
      ". the raw\n",
      "the raw features\n",
      "raw features of\n",
      "features of speech\n",
      "of speech waveforms\n",
      "speech waveforms later\n",
      "waveforms later produced\n",
      "later produced excellent\n",
      "produced excellent largerscale\n",
      "excellent largerscale results.many\n",
      "largerscale results.many aspects\n",
      "results.many aspects of\n",
      "aspects of speech\n",
      "of speech recognition\n",
      "speech recognition were\n",
      "recognition were taken\n",
      "were taken over\n",
      "taken over by\n",
      "over by a\n",
      "by a deep\n",
      "a deep learning\n",
      "deep learning method\n",
      "learning method called\n",
      "method called long\n",
      "called long shortterm\n",
      "long shortterm memory\n",
      "shortterm memory lstm\n",
      "memory lstm a\n",
      "lstm a recurrent\n",
      "a recurrent neural\n",
      "recurrent neural network\n",
      "neural network published\n",
      "network published by\n",
      "published by hochreiter\n",
      "by hochreiter and\n",
      "hochreiter and schmidhuber\n",
      "and schmidhuber in\n",
      "schmidhuber in .\n",
      "in . lstm\n",
      ". lstm rnns\n",
      "lstm rnns avoid\n",
      "rnns avoid the\n",
      "avoid the vanishing\n",
      "the vanishing gradient\n",
      "vanishing gradient problem\n",
      "gradient problem and\n",
      "problem and can\n",
      "and can learn\n",
      "can learn very\n",
      "learn very deep\n",
      "very deep learning\n",
      "deep learning tasks\n",
      "learning tasks that\n",
      "tasks that require\n",
      "that require memories\n",
      "require memories of\n",
      "memories of events\n",
      "of events that\n",
      "events that happened\n",
      "that happened thousands\n",
      "happened thousands of\n",
      "thousands of discrete\n",
      "of discrete time\n",
      "discrete time steps\n",
      "time steps before\n",
      "steps before which\n",
      "before which is\n",
      "which is important\n",
      "is important for\n",
      "important for speech\n",
      "for speech .\n",
      "speech . in\n",
      ". in lstm\n",
      "in lstm started\n",
      "lstm started to\n",
      "started to become\n",
      "to become competitive\n",
      "become competitive with\n",
      "competitive with traditional\n",
      "with traditional speech\n",
      "traditional speech recognizers\n",
      "speech recognizers on\n",
      "recognizers on certain\n",
      "on certain tasks\n",
      "certain tasks .\n",
      "tasks . later\n",
      ". later it\n",
      "later it was\n",
      "it was combined\n",
      "was combined with\n",
      "combined with connectionist\n",
      "with connectionist temporal\n",
      "connectionist temporal classification\n",
      "temporal classification ctc\n",
      "classification ctc in\n",
      "ctc in stacks\n",
      "in stacks of\n",
      "stacks of lstm\n",
      "of lstm rnns\n",
      "lstm rnns .\n",
      "rnns . in\n",
      ". in googles\n",
      "in googles speech\n",
      "googles speech recognition\n",
      "speech recognition reportedly\n",
      "recognition reportedly experienced\n",
      "reportedly experienced a\n",
      "experienced a dramatic\n",
      "a dramatic performance\n",
      "dramatic performance jump\n",
      "performance jump of\n",
      "jump of through\n",
      "of through ctctrained\n",
      "through ctctrained lstm\n",
      "ctctrained lstm which\n",
      "lstm which they\n",
      "which they made\n",
      "they made available\n",
      "made available through\n",
      "available through google\n",
      "through google voice\n",
      "google voice search.in\n",
      "voice search.in publications\n",
      "search.in publications by\n",
      "publications by geoff\n",
      "by geoff hinton\n",
      "geoff hinton ruslan\n",
      "hinton ruslan salakhutdinov\n",
      "ruslan salakhutdinov osindero\n",
      "salakhutdinov osindero and\n",
      "osindero and teh\n",
      "and teh showed\n",
      "teh showed how\n",
      "showed how a\n",
      "how a manylayered\n",
      "a manylayered feedforward\n",
      "manylayered feedforward neural\n",
      "feedforward neural network\n",
      "neural network could\n",
      "network could be\n",
      "could be effectively\n",
      "be effectively pretrained\n",
      "effectively pretrained one\n",
      "pretrained one layer\n",
      "one layer at\n",
      "layer at a\n",
      "at a time\n",
      "a time treating\n",
      "time treating each\n",
      "treating each layer\n",
      "each layer in\n",
      "layer in turn\n",
      "in turn as\n",
      "turn as an\n",
      "as an unsupervised\n",
      "an unsupervised restricted\n",
      "unsupervised restricted boltzmann\n",
      "restricted boltzmann machine\n",
      "boltzmann machine then\n",
      "machine then finetuning\n",
      "then finetuning it\n",
      "finetuning it using\n",
      "it using supervised\n",
      "using supervised backpropagation\n",
      "supervised backpropagation .\n",
      "backpropagation . the\n",
      ". the papers\n",
      "the papers referred\n",
      "papers referred to\n",
      "referred to learning\n",
      "to learning for\n",
      "learning for deep\n",
      "for deep belief\n",
      "deep belief nets.deep\n",
      "belief nets.deep learning\n",
      "nets.deep learning is\n",
      "learning is part\n",
      "is part of\n",
      "part of stateoftheart\n",
      "of stateoftheart systems\n",
      "stateoftheart systems in\n",
      "systems in various\n",
      "in various disciplines\n",
      "various disciplines particularly\n",
      "disciplines particularly computer\n",
      "particularly computer vision\n",
      "computer vision and\n",
      "vision and automatic\n",
      "and automatic speech\n",
      "automatic speech recognition\n",
      "speech recognition asr\n",
      "recognition asr .\n",
      "asr . results\n",
      ". results on\n",
      "results on commonly\n",
      "on commonly used\n",
      "commonly used evaluation\n",
      "used evaluation sets\n",
      "evaluation sets such\n",
      "sets such as\n",
      "such as timit\n",
      "as timit asr\n",
      "timit asr and\n",
      "asr and mnist\n",
      "and mnist image\n",
      "mnist image classification\n",
      "image classification as\n",
      "classification as well\n",
      "as well as\n",
      "well as a\n",
      "as a range\n",
      "a range of\n",
      "range of largevocabulary\n",
      "of largevocabulary speech\n",
      "largevocabulary speech recognition\n",
      "speech recognition tasks\n",
      "recognition tasks have\n",
      "tasks have steadily\n",
      "have steadily improved\n",
      "steadily improved .\n",
      "improved . convolutional\n",
      ". convolutional neural\n",
      "convolutional neural networks\n",
      "neural networks cnns\n",
      "networks cnns were\n",
      "cnns were superseded\n",
      "were superseded for\n",
      "superseded for asr\n",
      "for asr by\n",
      "asr by ctc\n",
      "by ctc for\n",
      "ctc for lstm\n",
      "for lstm .\n",
      "lstm . but\n",
      ". but are\n",
      "but are more\n",
      "are more successful\n",
      "more successful in\n",
      "successful in computer\n",
      "in computer vision.the\n",
      "computer vision.the impact\n",
      "vision.the impact of\n",
      "impact of deep\n",
      "of deep learning\n",
      "deep learning in\n",
      "learning in industry\n",
      "in industry began\n",
      "industry began in\n",
      "began in the\n",
      "in the early\n",
      "the early s\n",
      "early s when\n",
      "s when cnns\n",
      "when cnns already\n",
      "cnns already processed\n",
      "already processed an\n",
      "processed an estimated\n",
      "an estimated to\n",
      "estimated to of\n",
      "to of all\n",
      "of all the\n",
      "all the checks\n",
      "the checks written\n",
      "checks written in\n",
      "written in the\n",
      "in the us\n",
      "the us according\n",
      "us according to\n",
      "according to yann\n",
      "to yann lecun\n",
      "yann lecun .\n",
      "lecun . industrial\n",
      ". industrial applications\n",
      "industrial applications of\n",
      "applications of deep\n",
      "of deep learning\n",
      "deep learning to\n",
      "learning to largescale\n",
      "to largescale speech\n",
      "largescale speech recognition\n",
      "speech recognition started\n",
      "recognition started around\n",
      "started around .the\n",
      "around .the nips\n",
      ".the nips workshop\n",
      "nips workshop on\n",
      "workshop on deep\n",
      "on deep learning\n",
      "deep learning for\n",
      "learning for speech\n",
      "for speech recognition\n",
      "speech recognition was\n",
      "recognition was motivated\n",
      "was motivated by\n",
      "motivated by the\n",
      "by the limitations\n",
      "the limitations of\n",
      "limitations of deep\n",
      "of deep generative\n",
      "deep generative models\n",
      "generative models of\n",
      "models of speech\n",
      "of speech and\n",
      "speech and the\n",
      "and the possibility\n",
      "the possibility that\n",
      "possibility that given\n",
      "that given more\n",
      "given more capable\n",
      "more capable hardware\n",
      "capable hardware and\n",
      "hardware and largescale\n",
      "and largescale data\n",
      "largescale data sets\n",
      "data sets that\n",
      "sets that deep\n",
      "that deep neural\n",
      "deep neural nets\n",
      "neural nets dnn\n",
      "nets dnn might\n",
      "dnn might become\n",
      "might become practical\n",
      "become practical .\n",
      "practical . it\n",
      ". it was\n",
      "it was believed\n",
      "was believed that\n",
      "believed that pretraining\n",
      "that pretraining dnns\n",
      "pretraining dnns using\n",
      "dnns using generative\n",
      "using generative models\n",
      "generative models of\n",
      "models of deep\n",
      "of deep belief\n",
      "deep belief nets\n",
      "belief nets dbn\n",
      "nets dbn would\n",
      "dbn would overcome\n",
      "would overcome the\n",
      "overcome the main\n",
      "the main difficulties\n",
      "main difficulties of\n",
      "difficulties of neural\n",
      "of neural nets\n",
      "neural nets .\n",
      "nets . however\n",
      ". however it\n",
      "however it was\n",
      "it was discovered\n",
      "was discovered that\n",
      "discovered that replacing\n",
      "that replacing pretraining\n",
      "replacing pretraining with\n",
      "pretraining with large\n",
      "with large amounts\n",
      "large amounts of\n",
      "amounts of training\n",
      "of training data\n",
      "training data for\n",
      "data for straightforward\n",
      "for straightforward backpropagation\n",
      "straightforward backpropagation when\n",
      "backpropagation when using\n",
      "when using dnns\n",
      "using dnns with\n",
      "dnns with large\n",
      "with large contextdependent\n",
      "large contextdependent output\n",
      "contextdependent output layers\n",
      "output layers produced\n",
      "layers produced error\n",
      "produced error rates\n",
      "error rates dramatically\n",
      "rates dramatically lower\n",
      "dramatically lower than\n",
      "lower than thenstateoftheart\n",
      "than thenstateoftheart gaussian\n",
      "thenstateoftheart gaussian mixture\n",
      "gaussian mixture model\n",
      "mixture model gmmhidden\n",
      "model gmmhidden markov\n",
      "gmmhidden markov model\n",
      "markov model hmm\n",
      "model hmm and\n",
      "hmm and also\n",
      "and also than\n",
      "also than moreadvanced\n",
      "than moreadvanced generative\n",
      "moreadvanced generative modelbased\n",
      "generative modelbased systems\n",
      "modelbased systems .\n",
      "systems . the\n",
      ". the nature\n",
      "the nature of\n",
      "nature of the\n",
      "of the recognition\n",
      "the recognition errors\n",
      "recognition errors produced\n",
      "errors produced by\n",
      "produced by the\n",
      "by the two\n",
      "the two types\n",
      "two types of\n",
      "types of systems\n",
      "of systems was\n",
      "systems was characteristically\n",
      "was characteristically different\n",
      "characteristically different offering\n",
      "different offering technical\n",
      "offering technical insights\n",
      "technical insights into\n",
      "insights into how\n",
      "into how to\n",
      "how to integrate\n",
      "to integrate deep\n",
      "integrate deep learning\n",
      "deep learning into\n",
      "learning into the\n",
      "into the existing\n",
      "the existing highly\n",
      "existing highly efficient\n",
      "highly efficient runtime\n",
      "efficient runtime speech\n",
      "runtime speech decoding\n",
      "speech decoding system\n",
      "decoding system deployed\n",
      "system deployed by\n",
      "deployed by all\n",
      "by all major\n",
      "all major speech\n",
      "major speech recognition\n",
      "speech recognition systems\n",
      "recognition systems .\n",
      "systems . analysis\n",
      ". analysis around\n",
      "analysis around contrasting\n",
      "around contrasting the\n",
      "contrasting the gmm\n",
      "the gmm and\n",
      "gmm and other\n",
      "and other generative\n",
      "other generative speech\n",
      "generative speech models\n",
      "speech models vs.\n",
      "models vs. dnn\n",
      "vs. dnn models\n",
      "dnn models stimulated\n",
      "models stimulated early\n",
      "stimulated early industrial\n",
      "early industrial investment\n",
      "industrial investment in\n",
      "investment in deep\n",
      "in deep learning\n",
      "deep learning for\n",
      "learning for speech\n",
      "for speech recognition\n",
      "speech recognition eventually\n",
      "recognition eventually leading\n",
      "eventually leading to\n",
      "leading to pervasive\n",
      "to pervasive and\n",
      "pervasive and dominant\n",
      "and dominant use\n",
      "dominant use in\n",
      "use in that\n",
      "in that industry\n",
      "that industry .\n",
      "industry . that\n",
      ". that analysis\n",
      "that analysis was\n",
      "analysis was done\n",
      "was done with\n",
      "done with comparable\n",
      "with comparable performance\n",
      "comparable performance less\n",
      "performance less than\n",
      "less than .\n",
      "than . in\n",
      ". in error\n",
      "in error rate\n",
      "error rate between\n",
      "rate between discriminative\n",
      "between discriminative dnns\n",
      "discriminative dnns and\n",
      "dnns and generative\n",
      "and generative models.in\n",
      "generative models.in researchers\n",
      "models.in researchers extended\n",
      "researchers extended deep\n",
      "extended deep learning\n",
      "deep learning from\n",
      "learning from timit\n",
      "from timit to\n",
      "timit to large\n",
      "to large vocabulary\n",
      "large vocabulary speech\n",
      "vocabulary speech recognition\n",
      "speech recognition by\n",
      "recognition by adopting\n",
      "by adopting large\n",
      "adopting large output\n",
      "large output layers\n",
      "output layers of\n",
      "layers of the\n",
      "of the dnn\n",
      "the dnn based\n",
      "dnn based on\n",
      "based on contextdependent\n",
      "on contextdependent hmm\n",
      "contextdependent hmm states\n",
      "hmm states constructed\n",
      "states constructed by\n",
      "constructed by decision\n",
      "by decision trees.advances\n",
      "decision trees.advances in\n",
      "trees.advances in hardware\n",
      "in hardware have\n",
      "hardware have driven\n",
      "have driven renewed\n",
      "driven renewed interest\n",
      "renewed interest in\n",
      "interest in deep\n",
      "in deep learning\n",
      "deep learning .\n",
      "learning . in\n",
      ". in nvidia\n",
      "in nvidia was\n",
      "nvidia was involved\n",
      "was involved in\n",
      "involved in what\n",
      "in what was\n",
      "what was called\n",
      "was called the\n",
      "called the big\n",
      "the big bang\n",
      "big bang of\n",
      "bang of deep\n",
      "of deep learning\n",
      "deep learning as\n",
      "learning as deeplearning\n",
      "as deeplearning neural\n",
      "deeplearning neural networks\n",
      "neural networks were\n",
      "networks were trained\n",
      "were trained with\n",
      "trained with nvidia\n",
      "with nvidia graphics\n",
      "nvidia graphics processing\n",
      "graphics processing units\n",
      "processing units gpus\n",
      "units gpus .\n",
      "gpus . that\n",
      ". that year\n",
      "that year andrew\n",
      "year andrew ng\n",
      "andrew ng determined\n",
      "ng determined that\n",
      "determined that gpus\n",
      "that gpus could\n",
      "gpus could increase\n",
      "could increase the\n",
      "increase the speed\n",
      "the speed of\n",
      "speed of deeplearning\n",
      "of deeplearning systems\n",
      "deeplearning systems by\n",
      "systems by about\n",
      "by about times\n",
      "about times .\n",
      "times . in\n",
      ". in particular\n",
      "in particular gpus\n",
      "particular gpus are\n",
      "gpus are wellsuited\n",
      "are wellsuited for\n",
      "wellsuited for the\n",
      "for the matrixvector\n",
      "the matrixvector computations\n",
      "matrixvector computations involved\n",
      "computations involved in\n",
      "involved in machine\n",
      "in machine learning\n",
      "machine learning .\n",
      "learning . gpus\n",
      ". gpus speed\n",
      "gpus speed up\n",
      "speed up training\n",
      "up training algorithms\n",
      "training algorithms by\n",
      "algorithms by orders\n",
      "by orders of\n",
      "orders of magnitude\n",
      "of magnitude reducing\n",
      "magnitude reducing running\n",
      "reducing running times\n",
      "running times from\n",
      "times from weeks\n",
      "from weeks to\n",
      "weeks to days\n",
      "to days .\n",
      "days . further\n",
      ". further specialized\n",
      "further specialized hardware\n",
      "specialized hardware and\n",
      "hardware and algorithm\n",
      "and algorithm optimizations\n",
      "algorithm optimizations can\n",
      "optimizations can be\n",
      "can be used\n",
      "be used for\n",
      "used for efficient\n",
      "for efficient processing\n",
      "efficient processing of\n",
      "processing of deep\n",
      "of deep learning\n",
      "deep learning models.in\n",
      "learning models.in a\n",
      "models.in a team\n",
      "a team led\n",
      "team led by\n",
      "led by george\n",
      "by george e.\n",
      "george e. dahl\n",
      "e. dahl won\n",
      "dahl won the\n",
      "won the merck\n",
      "the merck molecular\n",
      "merck molecular activity\n",
      "molecular activity challenge\n",
      "activity challenge using\n",
      "challenge using multitask\n",
      "using multitask deep\n",
      "multitask deep neural\n",
      "deep neural networks\n",
      "neural networks to\n",
      "networks to predict\n",
      "to predict the\n",
      "predict the biomolecular\n",
      "the biomolecular target\n",
      "biomolecular target of\n",
      "target of one\n",
      "of one drug\n",
      "one drug .\n",
      "drug . in\n",
      ". in hochreiters\n",
      "in hochreiters group\n",
      "hochreiters group used\n",
      "group used deep\n",
      "used deep learning\n",
      "deep learning to\n",
      "learning to detect\n",
      "to detect offtarget\n",
      "detect offtarget and\n",
      "offtarget and toxic\n",
      "and toxic effects\n",
      "toxic effects of\n",
      "effects of environmental\n",
      "of environmental chemicals\n",
      "environmental chemicals in\n",
      "chemicals in nutrients\n",
      "in nutrients household\n",
      "nutrients household products\n",
      "household products and\n",
      "products and drugs\n",
      "and drugs and\n",
      "drugs and won\n",
      "and won the\n",
      "won the tox\n",
      "the tox data\n",
      "tox data challenge\n",
      "data challenge of\n",
      "challenge of nih\n",
      "of nih fda\n",
      "nih fda and\n",
      "fda and ncats.significant\n",
      "and ncats.significant additional\n",
      "ncats.significant additional impacts\n",
      "additional impacts in\n",
      "impacts in image\n",
      "in image or\n",
      "image or object\n",
      "or object recognition\n",
      "object recognition were\n",
      "recognition were felt\n",
      "were felt from\n",
      "felt from to\n",
      "from to .\n",
      "to . although\n",
      ". although cnns\n",
      "although cnns trained\n",
      "cnns trained by\n",
      "trained by backpropagation\n",
      "by backpropagation had\n",
      "backpropagation had been\n",
      "had been around\n",
      "been around for\n",
      "around for decades\n",
      "for decades and\n",
      "decades and gpu\n",
      "and gpu implementations\n",
      "gpu implementations of\n",
      "implementations of nns\n",
      "of nns for\n",
      "nns for years\n",
      "for years including\n",
      "years including cnns\n",
      "including cnns fast\n",
      "cnns fast implementations\n",
      "fast implementations of\n",
      "implementations of cnns\n",
      "of cnns on\n",
      "cnns on gpus\n",
      "on gpus were\n",
      "gpus were needed\n",
      "were needed to\n",
      "needed to progress\n",
      "to progress on\n",
      "progress on computer\n",
      "on computer vision\n",
      "computer vision .\n",
      "vision . in\n",
      ". in this\n",
      "in this approach\n",
      "this approach achieved\n",
      "approach achieved for\n",
      "achieved for the\n",
      "for the first\n",
      "the first time\n",
      "first time superhuman\n",
      "time superhuman performance\n",
      "superhuman performance in\n",
      "performance in a\n",
      "in a visual\n",
      "a visual pattern\n",
      "visual pattern recognition\n",
      "pattern recognition contest\n",
      "recognition contest .\n",
      "contest . also\n",
      ". also in\n",
      "also in it\n",
      "in it won\n",
      "it won the\n",
      "won the icdar\n",
      "the icdar chinese\n",
      "icdar chinese handwriting\n",
      "chinese handwriting contest\n",
      "handwriting contest and\n",
      "contest and in\n",
      "and in may\n",
      "in may it\n",
      "may it won\n",
      "it won the\n",
      "won the isbi\n",
      "the isbi image\n",
      "isbi image segmentation\n",
      "image segmentation contest\n",
      "segmentation contest .\n",
      "contest . until\n",
      ". until cnns\n",
      "until cnns did\n",
      "cnns did not\n",
      "did not play\n",
      "not play a\n",
      "play a major\n",
      "a major role\n",
      "major role at\n",
      "role at computer\n",
      "at computer vision\n",
      "computer vision conferences\n",
      "vision conferences but\n",
      "conferences but in\n",
      "but in june\n",
      "in june a\n",
      "june a paper\n",
      "a paper by\n",
      "paper by ciresan\n",
      "by ciresan et\n",
      "ciresan et al\n",
      "et al .\n",
      "al . at\n",
      ". at the\n",
      "at the leading\n",
      "the leading conference\n",
      "leading conference cvpr\n",
      "conference cvpr showed\n",
      "cvpr showed how\n",
      "showed how maxpooling\n",
      "how maxpooling cnns\n",
      "maxpooling cnns on\n",
      "cnns on gpu\n",
      "on gpu can\n",
      "gpu can dramatically\n",
      "can dramatically improve\n",
      "dramatically improve many\n",
      "improve many vision\n",
      "many vision benchmark\n",
      "vision benchmark records\n",
      "benchmark records .\n",
      "records . in\n",
      ". in october\n",
      "in october a\n",
      "october a similar\n",
      "a similar system\n",
      "similar system by\n",
      "system by krizhevsky\n",
      "by krizhevsky et\n",
      "krizhevsky et al\n",
      "et al .\n",
      "al . won\n",
      ". won the\n",
      "won the largescale\n",
      "the largescale imagenet\n",
      "largescale imagenet competition\n",
      "imagenet competition by\n",
      "competition by a\n",
      "by a significant\n",
      "a significant margin\n",
      "significant margin over\n",
      "margin over shallow\n",
      "over shallow machine\n",
      "shallow machine learning\n",
      "machine learning methods\n",
      "learning methods .\n",
      "methods . in\n",
      ". in november\n",
      "in november ciresan\n",
      "november ciresan et\n",
      "ciresan et al.s\n",
      "et al.s system\n",
      "al.s system also\n",
      "system also won\n",
      "also won the\n",
      "won the icpr\n",
      "the icpr contest\n",
      "icpr contest on\n",
      "contest on analysis\n",
      "on analysis of\n",
      "analysis of large\n",
      "of large medical\n",
      "large medical images\n",
      "medical images for\n",
      "images for cancer\n",
      "for cancer detection\n",
      "cancer detection and\n",
      "detection and in\n",
      "and in the\n",
      "in the following\n",
      "the following year\n",
      "following year also\n",
      "year also the\n",
      "also the miccai\n",
      "the miccai grand\n",
      "miccai grand challenge\n",
      "grand challenge on\n",
      "challenge on the\n",
      "on the same\n",
      "the same topic\n",
      "same topic .\n",
      "topic . in\n",
      ". in and\n",
      "in and the\n",
      "and the error\n",
      "the error rate\n",
      "error rate on\n",
      "rate on the\n",
      "on the imagenet\n",
      "the imagenet task\n",
      "imagenet task using\n",
      "task using deep\n",
      "using deep learning\n",
      "deep learning was\n",
      "learning was further\n",
      "was further reduced\n",
      "further reduced following\n",
      "reduced following a\n",
      "following a similar\n",
      "a similar trend\n",
      "similar trend in\n",
      "trend in largescale\n",
      "in largescale speech\n",
      "largescale speech recognition.image\n",
      "speech recognition.image classification\n",
      "recognition.image classification was\n",
      "classification was then\n",
      "was then extended\n",
      "then extended to\n",
      "extended to the\n",
      "to the more\n",
      "the more challenging\n",
      "more challenging task\n",
      "challenging task of\n",
      "task of generating\n",
      "of generating descriptions\n",
      "generating descriptions captions\n",
      "descriptions captions for\n",
      "captions for images\n",
      "for images often\n",
      "images often as\n",
      "often as a\n",
      "as a combination\n",
      "a combination of\n",
      "combination of cnns\n",
      "of cnns and\n",
      "cnns and lstms.some\n",
      "and lstms.some researchers\n",
      "lstms.some researchers state\n",
      "researchers state that\n",
      "state that the\n",
      "that the october\n",
      "the october imagenet\n",
      "october imagenet victory\n",
      "imagenet victory anchored\n",
      "victory anchored the\n",
      "anchored the start\n",
      "the start of\n",
      "start of a\n",
      "of a deep\n",
      "a deep learning\n",
      "deep learning revolution\n",
      "learning revolution that\n",
      "revolution that has\n",
      "that has transformed\n",
      "has transformed the\n",
      "transformed the ai\n",
      "the ai industry.in\n",
      "ai industry.in march\n",
      "industry.in march yoshua\n",
      "march yoshua bengio\n",
      "yoshua bengio geoffrey\n",
      "bengio geoffrey hinton\n",
      "geoffrey hinton and\n",
      "hinton and yann\n",
      "and yann lecun\n",
      "yann lecun were\n",
      "lecun were awarded\n",
      "were awarded the\n",
      "awarded the turing\n",
      "the turing award\n",
      "turing award for\n",
      "award for conceptual\n",
      "for conceptual and\n",
      "conceptual and engineering\n",
      "and engineering breakthroughs\n",
      "engineering breakthroughs that\n",
      "breakthroughs that have\n",
      "that have made\n",
      "have made deep\n",
      "made deep neural\n",
      "deep neural networks\n",
      "neural networks a\n",
      "networks a critical\n",
      "a critical component\n",
      "critical component of\n",
      "component of computing.artificial\n",
      "of computing.artificial neural\n",
      "computing.artificial neural networks\n",
      "neural networks anns\n",
      "networks anns or\n",
      "anns or connectionist\n",
      "or connectionist systems\n",
      "connectionist systems are\n",
      "systems are computing\n",
      "are computing systems\n",
      "computing systems inspired\n",
      "systems inspired by\n",
      "inspired by the\n",
      "by the biological\n",
      "the biological neural\n",
      "biological neural networks\n",
      "neural networks that\n",
      "networks that constitute\n",
      "that constitute animal\n",
      "constitute animal brains\n",
      "animal brains .\n",
      "brains . such\n",
      ". such systems\n",
      "such systems learn\n",
      "systems learn progressively\n",
      "learn progressively improve\n",
      "progressively improve their\n",
      "improve their ability\n",
      "their ability to\n",
      "ability to do\n",
      "to do tasks\n",
      "do tasks by\n",
      "tasks by considering\n",
      "by considering examples\n",
      "considering examples generally\n",
      "examples generally without\n",
      "generally without taskspecific\n",
      "without taskspecific programming\n",
      "taskspecific programming .\n",
      "programming . for\n",
      ". for example\n",
      "for example in\n",
      "example in image\n",
      "in image recognition\n",
      "image recognition they\n",
      "recognition they might\n",
      "they might learn\n",
      "might learn to\n",
      "learn to identify\n",
      "to identify images\n",
      "identify images that\n",
      "images that contain\n",
      "that contain cats\n",
      "contain cats by\n",
      "cats by analyzing\n",
      "by analyzing example\n",
      "analyzing example images\n",
      "example images that\n",
      "images that have\n",
      "that have been\n",
      "have been manually\n",
      "been manually labeled\n",
      "manually labeled as\n",
      "labeled as cat\n",
      "as cat or\n",
      "cat or no\n",
      "or no cat\n",
      "no cat and\n",
      "cat and using\n",
      "and using the\n",
      "using the analytic\n",
      "the analytic results\n",
      "analytic results to\n",
      "results to identify\n",
      "to identify cats\n",
      "identify cats in\n",
      "cats in other\n",
      "in other images\n",
      "other images .\n",
      "images . they\n",
      ". they have\n",
      "they have found\n",
      "have found most\n",
      "found most use\n",
      "most use in\n",
      "use in applications\n",
      "in applications difficult\n",
      "applications difficult to\n",
      "difficult to express\n",
      "to express with\n",
      "express with a\n",
      "with a traditional\n",
      "a traditional computer\n",
      "traditional computer algorithm\n",
      "computer algorithm using\n",
      "algorithm using rulebased\n",
      "using rulebased programming.an\n",
      "rulebased programming.an ann\n",
      "programming.an ann is\n",
      "ann is based\n",
      "is based on\n",
      "based on a\n",
      "on a collection\n",
      "a collection of\n",
      "collection of connected\n",
      "of connected units\n",
      "connected units called\n",
      "units called artificial\n",
      "called artificial neurons\n",
      "artificial neurons analogous\n",
      "neurons analogous to\n",
      "analogous to biological\n",
      "to biological neurons\n",
      "biological neurons in\n",
      "neurons in a\n",
      "in a biological\n",
      "a biological brain\n",
      "biological brain .\n",
      "brain . each\n",
      ". each connection\n",
      "each connection synapse\n",
      "connection synapse between\n",
      "synapse between neurons\n",
      "between neurons can\n",
      "neurons can transmit\n",
      "can transmit a\n",
      "transmit a signal\n",
      "a signal to\n",
      "signal to another\n",
      "to another neuron\n",
      "another neuron .\n",
      "neuron . the\n",
      ". the receiving\n",
      "the receiving postsynaptic\n",
      "receiving postsynaptic neuron\n",
      "postsynaptic neuron can\n",
      "neuron can process\n",
      "can process the\n",
      "process the signals\n",
      "the signals and\n",
      "signals and then\n",
      "and then signal\n",
      "then signal downstream\n",
      "signal downstream neurons\n",
      "downstream neurons connected\n",
      "neurons connected to\n",
      "connected to it\n",
      "to it .\n",
      "it . neurons\n",
      ". neurons may\n",
      "neurons may have\n",
      "may have state\n",
      "have state generally\n",
      "state generally represented\n",
      "generally represented by\n",
      "represented by real\n",
      "by real numbers\n",
      "real numbers typically\n",
      "numbers typically between\n",
      "typically between and\n",
      "between and .\n",
      "and . neurons\n",
      ". neurons and\n",
      "neurons and synapses\n",
      "and synapses may\n",
      "synapses may also\n",
      "may also have\n",
      "also have a\n",
      "have a weight\n",
      "a weight that\n",
      "weight that varies\n",
      "that varies as\n",
      "varies as learning\n",
      "as learning proceeds\n",
      "learning proceeds which\n",
      "proceeds which can\n",
      "which can increase\n",
      "can increase or\n",
      "increase or decrease\n",
      "or decrease the\n",
      "decrease the strength\n",
      "the strength of\n",
      "strength of the\n",
      "of the signal\n",
      "the signal that\n",
      "signal that it\n",
      "that it sends\n",
      "it sends downstream.typically\n",
      "sends downstream.typically neurons\n",
      "downstream.typically neurons are\n",
      "neurons are organized\n",
      "are organized in\n",
      "organized in layers\n",
      "in layers .\n",
      "layers . different\n",
      ". different layers\n",
      "different layers may\n",
      "layers may perform\n",
      "may perform different\n",
      "perform different kinds\n",
      "different kinds of\n",
      "kinds of transformations\n",
      "of transformations on\n",
      "transformations on their\n",
      "on their inputs\n",
      "their inputs .\n",
      "inputs . signals\n",
      ". signals travel\n",
      "signals travel from\n",
      "travel from the\n",
      "from the first\n",
      "the first input\n",
      "first input to\n",
      "input to the\n",
      "to the last\n",
      "the last output\n",
      "last output layer\n",
      "output layer possibly\n",
      "layer possibly after\n",
      "possibly after traversing\n",
      "after traversing the\n",
      "traversing the layers\n",
      "the layers multiple\n",
      "layers multiple times.the\n",
      "multiple times.the original\n",
      "times.the original goal\n",
      "original goal of\n",
      "goal of the\n",
      "of the neural\n",
      "the neural network\n",
      "neural network approach\n",
      "network approach was\n",
      "approach was to\n",
      "was to solve\n",
      "to solve problems\n",
      "solve problems in\n",
      "problems in the\n",
      "in the same\n",
      "the same way\n",
      "same way that\n",
      "way that a\n",
      "that a human\n",
      "a human brain\n",
      "human brain would\n",
      "brain would .\n",
      "would . over\n",
      ". over time\n",
      "over time attention\n",
      "time attention focused\n",
      "attention focused on\n",
      "focused on matching\n",
      "on matching specific\n",
      "matching specific mental\n",
      "specific mental abilities\n",
      "mental abilities leading\n",
      "abilities leading to\n",
      "leading to deviations\n",
      "to deviations from\n",
      "deviations from biology\n",
      "from biology such\n",
      "biology such as\n",
      "such as backpropagation\n",
      "as backpropagation or\n",
      "backpropagation or passing\n",
      "or passing information\n",
      "passing information in\n",
      "information in the\n",
      "in the reverse\n",
      "the reverse direction\n",
      "reverse direction and\n",
      "direction and adjusting\n",
      "and adjusting the\n",
      "adjusting the network\n",
      "the network to\n",
      "network to reflect\n",
      "to reflect that\n",
      "reflect that information.neural\n",
      "that information.neural networks\n",
      "information.neural networks have\n",
      "networks have been\n",
      "have been used\n",
      "been used on\n",
      "used on a\n",
      "on a variety\n",
      "a variety of\n",
      "variety of tasks\n",
      "of tasks including\n",
      "tasks including computer\n",
      "including computer vision\n",
      "computer vision speech\n",
      "vision speech recognition\n",
      "speech recognition machine\n",
      "recognition machine translation\n",
      "machine translation social\n",
      "translation social network\n",
      "social network filtering\n",
      "network filtering playing\n",
      "filtering playing board\n",
      "playing board and\n",
      "board and video\n",
      "and video games\n",
      "video games and\n",
      "games and medical\n",
      "and medical diagnosis.as\n",
      "medical diagnosis.as of\n",
      "diagnosis.as of neural\n",
      "of neural networks\n",
      "neural networks typically\n",
      "networks typically have\n",
      "typically have a\n",
      "have a few\n",
      "a few thousand\n",
      "few thousand to\n",
      "thousand to a\n",
      "to a few\n",
      "a few million\n",
      "few million units\n",
      "million units and\n",
      "units and millions\n",
      "and millions of\n",
      "millions of connections\n",
      "of connections .\n",
      "connections . despite\n",
      ". despite this\n",
      "despite this number\n",
      "this number being\n",
      "number being several\n",
      "being several order\n",
      "several order of\n",
      "order of magnitude\n",
      "of magnitude less\n",
      "magnitude less than\n",
      "less than the\n",
      "than the number\n",
      "the number of\n",
      "number of neurons\n",
      "of neurons on\n",
      "neurons on a\n",
      "on a human\n",
      "a human brain\n",
      "human brain these\n",
      "brain these networks\n",
      "these networks can\n",
      "networks can perform\n",
      "can perform many\n",
      "perform many tasks\n",
      "many tasks at\n",
      "tasks at a\n",
      "at a level\n",
      "a level beyond\n",
      "level beyond that\n",
      "beyond that of\n",
      "that of humans\n",
      "of humans e.g\n",
      "humans e.g .\n",
      "e.g . recognizing\n",
      ". recognizing faces\n",
      "recognizing faces or\n",
      "faces or playing\n",
      "or playing go\n",
      "playing go .a\n",
      "go .a deep\n",
      ".a deep neural\n",
      "deep neural network\n",
      "neural network dnn\n",
      "network dnn is\n",
      "dnn is an\n",
      "is an artificial\n",
      "an artificial neural\n",
      "artificial neural network\n",
      "neural network ann\n",
      "network ann with\n",
      "ann with multiple\n",
      "with multiple layers\n",
      "multiple layers between\n",
      "layers between the\n",
      "between the input\n",
      "the input and\n",
      "input and output\n",
      "and output layers\n",
      "output layers .\n",
      "layers . there\n",
      ". there are\n",
      "there are different\n",
      "are different types\n",
      "different types of\n",
      "types of neural\n",
      "of neural networks\n",
      "neural networks but\n",
      "networks but they\n",
      "but they always\n",
      "they always consist\n",
      "always consist of\n",
      "consist of the\n",
      "of the same\n",
      "the same components\n",
      "same components neurons\n",
      "components neurons synapses\n",
      "neurons synapses weights\n",
      "synapses weights biases\n",
      "weights biases and\n",
      "biases and functions\n",
      "and functions .\n",
      "functions . these\n",
      ". these components\n",
      "these components as\n",
      "components as a\n",
      "as a whole\n",
      "a whole function\n",
      "whole function similarly\n",
      "function similarly to\n",
      "similarly to a\n",
      "to a human\n",
      "a human brain\n",
      "human brain and\n",
      "brain and can\n",
      "and can be\n",
      "can be trained\n",
      "be trained like\n",
      "trained like any\n",
      "like any other\n",
      "any other ml\n",
      "other ml algorithm.citation\n",
      "ml algorithm.citation neededfor\n",
      "algorithm.citation neededfor example\n",
      "neededfor example a\n",
      "example a dnn\n",
      "a dnn that\n",
      "dnn that is\n",
      "that is trained\n",
      "is trained to\n",
      "trained to recognize\n",
      "to recognize dog\n",
      "recognize dog breeds\n",
      "dog breeds will\n",
      "breeds will go\n",
      "will go over\n",
      "go over the\n",
      "over the given\n",
      "the given image\n",
      "given image and\n",
      "image and calculate\n",
      "and calculate the\n",
      "calculate the probability\n",
      "the probability that\n",
      "probability that the\n",
      "that the dog\n",
      "the dog in\n",
      "dog in the\n",
      "in the image\n",
      "the image is\n",
      "image is a\n",
      "is a certain\n",
      "a certain breed\n",
      "certain breed .\n",
      "breed . the\n",
      ". the user\n",
      "the user can\n",
      "user can review\n",
      "can review the\n",
      "review the results\n",
      "the results and\n",
      "results and select\n",
      "and select which\n",
      "select which probabilities\n",
      "which probabilities the\n",
      "probabilities the network\n",
      "the network should\n",
      "network should display\n",
      "should display above\n",
      "display above a\n",
      "above a certain\n",
      "a certain threshold\n",
      "certain threshold etc\n",
      "threshold etc .\n",
      "etc . and\n",
      ". and return\n",
      "and return the\n",
      "return the proposed\n",
      "the proposed label\n",
      "proposed label .\n",
      "label . each\n",
      ". each mathematical\n",
      "each mathematical manipulation\n",
      "mathematical manipulation as\n",
      "manipulation as such\n",
      "as such is\n",
      "such is considered\n",
      "is considered a\n",
      "considered a layercitation\n",
      "a layercitation needed\n",
      "layercitation needed and\n",
      "needed and complex\n",
      "and complex dnn\n",
      "complex dnn have\n",
      "dnn have many\n",
      "have many layers\n",
      "many layers hence\n",
      "layers hence the\n",
      "hence the name\n",
      "the name deep\n",
      "name deep networks.dnns\n",
      "deep networks.dnns can\n",
      "networks.dnns can model\n",
      "can model complex\n",
      "model complex nonlinear\n",
      "complex nonlinear relationships\n",
      "nonlinear relationships .\n",
      "relationships . dnn\n",
      ". dnn architectures\n",
      "dnn architectures generate\n",
      "architectures generate compositional\n",
      "generate compositional models\n",
      "compositional models where\n",
      "models where the\n",
      "where the object\n",
      "the object is\n",
      "object is expressed\n",
      "is expressed as\n",
      "expressed as a\n",
      "as a layered\n",
      "a layered composition\n",
      "layered composition of\n",
      "composition of primitives\n",
      "of primitives .\n",
      "primitives . the\n",
      ". the extra\n",
      "the extra layers\n",
      "extra layers enable\n",
      "layers enable composition\n",
      "enable composition of\n",
      "composition of features\n",
      "of features from\n",
      "features from lower\n",
      "from lower layers\n",
      "lower layers potentially\n",
      "layers potentially modeling\n",
      "potentially modeling complex\n",
      "modeling complex data\n",
      "complex data with\n",
      "data with fewer\n",
      "with fewer units\n",
      "fewer units than\n",
      "units than a\n",
      "than a similarly\n",
      "a similarly performing\n",
      "similarly performing shallow\n",
      "performing shallow network\n",
      "shallow network .\n",
      "network . for\n",
      ". for instance\n",
      "for instance it\n",
      "instance it was\n",
      "it was proved\n",
      "was proved that\n",
      "proved that sparse\n",
      "that sparse multivariate\n",
      "sparse multivariate polynomials\n",
      "multivariate polynomials are\n",
      "polynomials are exponentially\n",
      "are exponentially easier\n",
      "exponentially easier to\n",
      "easier to approximate\n",
      "to approximate with\n",
      "approximate with dnns\n",
      "with dnns than\n",
      "dnns than with\n",
      "than with shallow\n",
      "with shallow networks.deep\n",
      "shallow networks.deep architectures\n",
      "networks.deep architectures include\n",
      "architectures include many\n",
      "include many variants\n",
      "many variants of\n",
      "variants of a\n",
      "of a few\n",
      "a few basic\n",
      "few basic approaches\n",
      "basic approaches .\n",
      "approaches . each\n",
      ". each architecture\n",
      "each architecture has\n",
      "architecture has found\n",
      "has found success\n",
      "found success in\n",
      "success in specific\n",
      "in specific domains\n",
      "specific domains .\n",
      "domains . it\n",
      ". it is\n",
      "it is not\n",
      "is not always\n",
      "not always possible\n",
      "always possible to\n",
      "possible to compare\n",
      "to compare the\n",
      "compare the performance\n",
      "the performance of\n",
      "performance of multiple\n",
      "of multiple architectures\n",
      "multiple architectures unless\n",
      "architectures unless they\n",
      "unless they have\n",
      "they have been\n",
      "have been evaluated\n",
      "been evaluated on\n",
      "evaluated on the\n",
      "on the same\n",
      "the same data\n",
      "same data sets.dnns\n",
      "data sets.dnns are\n",
      "sets.dnns are typically\n",
      "are typically feedforward\n",
      "typically feedforward networks\n",
      "feedforward networks in\n",
      "networks in which\n",
      "in which data\n",
      "which data flows\n",
      "data flows from\n",
      "flows from the\n",
      "from the input\n",
      "the input layer\n",
      "input layer to\n",
      "layer to the\n",
      "to the output\n",
      "the output layer\n",
      "output layer without\n",
      "layer without looping\n",
      "without looping back\n",
      "looping back .\n",
      "back . at\n",
      ". at first\n",
      "at first the\n",
      "first the dnn\n",
      "the dnn creates\n",
      "dnn creates a\n",
      "creates a map\n",
      "a map of\n",
      "map of virtual\n",
      "of virtual neurons\n",
      "virtual neurons and\n",
      "neurons and assigns\n",
      "and assigns random\n",
      "assigns random numerical\n",
      "random numerical values\n",
      "numerical values or\n",
      "values or weights\n",
      "or weights to\n",
      "weights to connections\n",
      "to connections between\n",
      "connections between them\n",
      "between them .\n",
      "them . the\n",
      ". the weights\n",
      "the weights and\n",
      "weights and inputs\n",
      "and inputs are\n",
      "inputs are multiplied\n",
      "are multiplied and\n",
      "multiplied and return\n",
      "and return an\n",
      "return an output\n",
      "an output between\n",
      "output between and\n",
      "between and .\n",
      "and . if\n",
      ". if the\n",
      "if the network\n",
      "the network did\n",
      "network did not\n",
      "did not accurately\n",
      "not accurately recognize\n",
      "accurately recognize a\n",
      "recognize a particular\n",
      "a particular pattern\n",
      "particular pattern an\n",
      "pattern an algorithm\n",
      "an algorithm would\n",
      "algorithm would adjust\n",
      "would adjust the\n",
      "adjust the weights\n",
      "the weights .\n",
      "weights . that\n",
      ". that way\n",
      "that way the\n",
      "way the algorithm\n",
      "the algorithm can\n",
      "algorithm can make\n",
      "can make certain\n",
      "make certain parameters\n",
      "certain parameters more\n",
      "parameters more influential\n",
      "more influential until\n",
      "influential until it\n",
      "until it determines\n",
      "it determines the\n",
      "determines the correct\n",
      "the correct mathematical\n",
      "correct mathematical manipulation\n",
      "mathematical manipulation to\n",
      "manipulation to fully\n",
      "to fully process\n",
      "fully process the\n",
      "process the data.recurrent\n",
      "the data.recurrent neural\n",
      "data.recurrent neural networks\n",
      "neural networks rnns\n",
      "networks rnns in\n",
      "rnns in which\n",
      "in which data\n",
      "which data can\n",
      "data can flow\n",
      "can flow in\n",
      "flow in any\n",
      "in any direction\n",
      "any direction are\n",
      "direction are used\n",
      "are used for\n",
      "used for applications\n",
      "for applications such\n",
      "applications such as\n",
      "such as language\n",
      "as language modeling\n",
      "language modeling .\n",
      "modeling . long\n",
      ". long shortterm\n",
      "long shortterm memory\n",
      "shortterm memory is\n",
      "memory is particularly\n",
      "is particularly effective\n",
      "particularly effective for\n",
      "effective for this\n",
      "for this use.convolutional\n",
      "this use.convolutional deep\n",
      "use.convolutional deep neural\n",
      "deep neural networks\n",
      "neural networks cnns\n",
      "networks cnns are\n",
      "cnns are used\n",
      "are used in\n",
      "used in computer\n",
      "in computer vision\n",
      "computer vision .\n",
      "vision . cnns\n",
      ". cnns also\n",
      "cnns also have\n",
      "also have been\n",
      "have been applied\n",
      "been applied to\n",
      "applied to acoustic\n",
      "to acoustic modeling\n",
      "acoustic modeling for\n",
      "modeling for automatic\n",
      "for automatic speech\n",
      "automatic speech recognition\n",
      "speech recognition asr.as\n",
      "recognition asr.as with\n",
      "asr.as with anns\n",
      "with anns many\n",
      "anns many issues\n",
      "many issues can\n",
      "issues can arise\n",
      "can arise with\n",
      "arise with naively\n",
      "with naively trained\n",
      "naively trained dnns\n",
      "trained dnns .\n",
      "dnns . two\n",
      ". two common\n",
      "two common issues\n",
      "common issues are\n",
      "issues are overfitting\n",
      "are overfitting and\n",
      "overfitting and computation\n",
      "and computation time.dnns\n",
      "computation time.dnns are\n",
      "time.dnns are prone\n",
      "are prone to\n",
      "prone to overfitting\n",
      "to overfitting because\n",
      "overfitting because of\n",
      "because of the\n",
      "of the added\n",
      "the added layers\n",
      "added layers of\n",
      "layers of abstraction\n",
      "of abstraction which\n",
      "abstraction which allow\n",
      "which allow them\n",
      "allow them to\n",
      "them to model\n",
      "to model rare\n",
      "model rare dependencies\n",
      "rare dependencies in\n",
      "dependencies in the\n",
      "in the training\n",
      "the training data\n",
      "training data .\n",
      "data . regularization\n",
      ". regularization methods\n",
      "regularization methods such\n",
      "methods such as\n",
      "such as ivakhnenkos\n",
      "as ivakhnenkos unit\n",
      "ivakhnenkos unit pruning\n",
      "unit pruning or\n",
      "pruning or weight\n",
      "or weight decay\n",
      "weight decay displaystyle\n",
      "decay displaystyle ell\n",
      "displaystyle ell regularization\n",
      "ell regularization or\n",
      "regularization or sparsity\n",
      "or sparsity displaystyle\n",
      "sparsity displaystyle ell\n",
      "displaystyle ell regularization\n",
      "ell regularization can\n",
      "regularization can be\n",
      "can be applied\n",
      "be applied during\n",
      "applied during training\n",
      "during training to\n",
      "training to combat\n",
      "to combat overfitting\n",
      "combat overfitting .\n",
      "overfitting . alternatively\n",
      ". alternatively dropout\n",
      "alternatively dropout regularization\n",
      "dropout regularization randomly\n",
      "regularization randomly omits\n",
      "randomly omits units\n",
      "omits units from\n",
      "units from the\n",
      "from the hidden\n",
      "the hidden layers\n",
      "hidden layers during\n",
      "layers during training\n",
      "during training .\n",
      "training . this\n",
      ". this helps\n",
      "this helps to\n",
      "helps to exclude\n",
      "to exclude rare\n",
      "exclude rare dependencies\n",
      "rare dependencies .\n",
      "dependencies . finally\n",
      ". finally data\n",
      "finally data can\n",
      "data can be\n",
      "can be augmented\n",
      "be augmented via\n",
      "augmented via methods\n",
      "via methods such\n",
      "methods such as\n",
      "such as cropping\n",
      "as cropping and\n",
      "cropping and rotating\n",
      "and rotating such\n",
      "rotating such that\n",
      "such that smaller\n",
      "that smaller training\n",
      "smaller training sets\n",
      "training sets can\n",
      "sets can be\n",
      "can be increased\n",
      "be increased in\n",
      "increased in size\n",
      "in size to\n",
      "size to reduce\n",
      "to reduce the\n",
      "reduce the chances\n",
      "the chances of\n",
      "chances of overfitting.dnns\n",
      "of overfitting.dnns must\n",
      "overfitting.dnns must consider\n",
      "must consider many\n",
      "consider many training\n",
      "many training parameters\n",
      "training parameters such\n",
      "parameters such as\n",
      "such as the\n",
      "as the size\n",
      "the size number\n",
      "size number of\n",
      "number of layers\n",
      "of layers and\n",
      "layers and number\n",
      "and number of\n",
      "number of units\n",
      "of units per\n",
      "units per layer\n",
      "per layer the\n",
      "layer the learning\n",
      "the learning rate\n",
      "learning rate and\n",
      "rate and initial\n",
      "and initial weights\n",
      "initial weights .\n",
      "weights . sweeping\n",
      ". sweeping through\n",
      "sweeping through the\n",
      "through the parameter\n",
      "the parameter space\n",
      "parameter space for\n",
      "space for optimal\n",
      "for optimal parameters\n",
      "optimal parameters may\n",
      "parameters may not\n",
      "may not be\n",
      "not be feasible\n",
      "be feasible due\n",
      "feasible due to\n",
      "due to the\n",
      "to the cost\n",
      "the cost in\n",
      "cost in time\n",
      "in time and\n",
      "time and computational\n",
      "and computational resources\n",
      "computational resources .\n",
      "resources . various\n",
      ". various tricks\n",
      "various tricks such\n",
      "tricks such as\n",
      "such as batching\n",
      "as batching computing\n",
      "batching computing the\n",
      "computing the gradient\n",
      "the gradient on\n",
      "gradient on several\n",
      "on several training\n",
      "several training examples\n",
      "training examples at\n",
      "examples at once\n",
      "at once rather\n",
      "once rather than\n",
      "rather than individual\n",
      "than individual examples\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "individual examples speed\n",
      "examples speed up\n",
      "speed up computation\n",
      "up computation .\n",
      "computation . large\n",
      ". large processing\n",
      "large processing capabilities\n",
      "processing capabilities of\n",
      "capabilities of manycore\n",
      "of manycore architectures\n",
      "manycore architectures such\n",
      "architectures such as\n",
      "such as gpus\n",
      "as gpus or\n",
      "gpus or the\n",
      "or the intel\n",
      "the intel xeon\n",
      "intel xeon phi\n",
      "xeon phi have\n",
      "phi have produced\n",
      "have produced significant\n",
      "produced significant speedups\n",
      "significant speedups in\n",
      "speedups in training\n",
      "in training because\n",
      "training because of\n",
      "because of the\n",
      "of the suitability\n",
      "the suitability of\n",
      "suitability of such\n",
      "of such processing\n",
      "such processing architectures\n",
      "processing architectures for\n",
      "architectures for the\n",
      "for the matrix\n",
      "the matrix and\n",
      "matrix and vector\n",
      "and vector computations.alternatively\n",
      "vector computations.alternatively engineers\n",
      "computations.alternatively engineers may\n",
      "engineers may look\n",
      "may look for\n",
      "look for other\n",
      "for other types\n",
      "other types of\n",
      "types of neural\n",
      "of neural networks\n",
      "neural networks with\n",
      "networks with more\n",
      "with more straightforward\n",
      "more straightforward and\n",
      "straightforward and convergent\n",
      "and convergent training\n",
      "convergent training algorithms\n",
      "training algorithms .\n",
      "algorithms . cmac\n",
      ". cmac cerebellar\n",
      "cmac cerebellar model\n",
      "cerebellar model articulation\n",
      "model articulation controller\n",
      "articulation controller is\n",
      "controller is one\n",
      "is one such\n",
      "one such kind\n",
      "such kind of\n",
      "kind of neural\n",
      "of neural network\n",
      "neural network .\n",
      "network . it\n",
      ". it doesnt\n",
      "it doesnt require\n",
      "doesnt require learning\n",
      "require learning rates\n",
      "learning rates or\n",
      "rates or randomized\n",
      "or randomized initial\n",
      "randomized initial weights\n",
      "initial weights for\n",
      "weights for cmac\n",
      "for cmac .\n",
      "cmac . the\n",
      ". the training\n",
      "the training process\n",
      "training process can\n",
      "process can be\n",
      "can be guaranteed\n",
      "be guaranteed to\n",
      "guaranteed to converge\n",
      "to converge in\n",
      "converge in one\n",
      "in one step\n",
      "one step with\n",
      "step with a\n",
      "with a new\n",
      "a new batch\n",
      "new batch of\n",
      "batch of data\n",
      "of data and\n",
      "data and the\n",
      "and the computational\n",
      "the computational complexity\n",
      "computational complexity of\n",
      "complexity of the\n",
      "of the training\n",
      "the training algorithm\n",
      "training algorithm is\n",
      "algorithm is linear\n",
      "is linear with\n",
      "linear with respect\n",
      "with respect to\n",
      "respect to the\n",
      "to the number\n",
      "the number of\n",
      "number of neurons\n",
      "of neurons involved.since\n",
      "neurons involved.since the\n",
      "involved.since the s\n",
      "the s advances\n",
      "s advances in\n",
      "advances in both\n",
      "in both machine\n",
      "both machine learning\n",
      "machine learning algorithms\n",
      "learning algorithms and\n",
      "algorithms and computer\n",
      "and computer hardware\n",
      "computer hardware have\n",
      "hardware have led\n",
      "have led to\n",
      "led to more\n",
      "to more efficient\n",
      "more efficient methods\n",
      "efficient methods for\n",
      "methods for training\n",
      "for training deep\n",
      "training deep neural\n",
      "deep neural networks\n",
      "neural networks that\n",
      "networks that contain\n",
      "that contain many\n",
      "contain many layers\n",
      "many layers of\n",
      "layers of nonlinear\n",
      "of nonlinear hidden\n",
      "nonlinear hidden units\n",
      "hidden units and\n",
      "units and a\n",
      "and a very\n",
      "a very large\n",
      "very large output\n",
      "large output layer\n",
      "output layer .\n",
      "layer . by\n",
      ". by graphic\n",
      "by graphic processing\n",
      "graphic processing units\n",
      "processing units gpus\n",
      "units gpus often\n",
      "gpus often with\n",
      "often with aispecific\n",
      "with aispecific enhancements\n",
      "aispecific enhancements had\n",
      "enhancements had displaced\n",
      "had displaced cpus\n",
      "displaced cpus as\n",
      "cpus as the\n",
      "as the dominant\n",
      "the dominant method\n",
      "dominant method of\n",
      "method of training\n",
      "of training largescale\n",
      "training largescale commercial\n",
      "largescale commercial cloud\n",
      "commercial cloud ai\n",
      "cloud ai .\n",
      "ai . openai\n",
      ". openai estimated\n",
      "openai estimated the\n",
      "estimated the hardware\n",
      "the hardware computation\n",
      "hardware computation used\n",
      "computation used in\n",
      "used in the\n",
      "in the largest\n",
      "the largest deep\n",
      "largest deep learning\n",
      "deep learning projects\n",
      "learning projects from\n",
      "projects from alexnet\n",
      "from alexnet to\n",
      "alexnet to alphazero\n",
      "to alphazero and\n",
      "alphazero and found\n",
      "and found a\n",
      "found a fold\n",
      "a fold increase\n",
      "fold increase in\n",
      "increase in the\n",
      "in the amount\n",
      "the amount of\n",
      "amount of computation\n",
      "of computation required\n",
      "computation required with\n",
      "required with a\n",
      "with a doublingtime\n",
      "a doublingtime trendline\n",
      "doublingtime trendline of\n",
      "trendline of .\n",
      "of . months.special\n",
      ". months.special electronic\n",
      "months.special electronic circuits\n",
      "electronic circuits called\n",
      "circuits called deep\n",
      "called deep learning\n",
      "deep learning processors\n",
      "learning processors were\n",
      "processors were designed\n",
      "were designed to\n",
      "designed to speed\n",
      "to speed up\n",
      "speed up deep\n",
      "up deep learning\n",
      "deep learning algorithms\n",
      "learning algorithms .\n",
      "algorithms . deep\n",
      ". deep learning\n",
      "deep learning processors\n",
      "learning processors include\n",
      "processors include neural\n",
      "include neural processing\n",
      "neural processing units\n",
      "processing units npus\n",
      "units npus in\n",
      "npus in huawei\n",
      "in huawei cellphones\n",
      "huawei cellphones and\n",
      "cellphones and cloud\n",
      "and cloud computing\n",
      "cloud computing servers\n",
      "computing servers such\n",
      "servers such as\n",
      "such as tensor\n",
      "as tensor processing\n",
      "tensor processing units\n",
      "processing units tpu\n",
      "units tpu in\n",
      "tpu in the\n",
      "in the google\n",
      "the google cloud\n",
      "google cloud platform\n",
      "cloud platform .\n",
      "platform . cerebras\n",
      ". cerebras systems\n",
      "cerebras systems has\n",
      "systems has also\n",
      "has also built\n",
      "also built a\n",
      "built a dedicated\n",
      "a dedicated system\n",
      "dedicated system to\n",
      "system to handle\n",
      "to handle large\n",
      "handle large deep\n",
      "large deep learning\n",
      "deep learning models\n",
      "learning models the\n",
      "models the cs\n",
      "the cs based\n",
      "cs based on\n",
      "based on the\n",
      "on the largest\n",
      "the largest processor\n",
      "largest processor in\n",
      "processor in the\n",
      "in the industry\n",
      "the industry the\n",
      "industry the secondgeneration\n",
      "the secondgeneration wafer\n",
      "secondgeneration wafer scale\n",
      "wafer scale engine\n",
      "scale engine wse.atomically\n",
      "engine wse.atomically thin\n",
      "wse.atomically thin semiconductors\n",
      "thin semiconductors are\n",
      "semiconductors are considered\n",
      "are considered promising\n",
      "considered promising for\n",
      "promising for energyefficient\n",
      "for energyefficient deep\n",
      "energyefficient deep learning\n",
      "deep learning hardware\n",
      "learning hardware where\n",
      "hardware where the\n",
      "where the same\n",
      "the same basic\n",
      "same basic device\n",
      "basic device structure\n",
      "device structure is\n",
      "structure is used\n",
      "is used for\n",
      "used for both\n",
      "for both logic\n",
      "both logic operations\n",
      "logic operations and\n",
      "operations and data\n",
      "and data storage.in\n",
      "data storage.in marega\n",
      "storage.in marega et\n",
      "marega et al\n",
      "et al .\n",
      "al . published\n",
      ". published experiments\n",
      "published experiments with\n",
      "experiments with a\n",
      "with a largearea\n",
      "a largearea active\n",
      "largearea active channel\n",
      "active channel material\n",
      "channel material for\n",
      "material for developing\n",
      "for developing logicinmemory\n",
      "developing logicinmemory devices\n",
      "logicinmemory devices and\n",
      "devices and circuits\n",
      "and circuits based\n",
      "circuits based on\n",
      "based on floatinggate\n",
      "on floatinggate fieldeffect\n",
      "floatinggate fieldeffect transistors\n",
      "fieldeffect transistors fgfets.in\n",
      "transistors fgfets.in j.\n",
      "fgfets.in j. feldmann\n",
      "j. feldmann et\n",
      "feldmann et al\n",
      "et al .\n",
      "al . proposed\n",
      ". proposed an\n",
      "proposed an integrated\n",
      "an integrated photonic\n",
      "integrated photonic hardware\n",
      "photonic hardware accelerator\n",
      "hardware accelerator for\n",
      "accelerator for parallel\n",
      "for parallel convolutional\n",
      "parallel convolutional processing\n",
      "convolutional processing .\n",
      "processing . the\n",
      ". the authors\n",
      "the authors identify\n",
      "authors identify two\n",
      "identify two key\n",
      "two key advantages\n",
      "key advantages of\n",
      "advantages of integrated\n",
      "of integrated photonics\n",
      "integrated photonics over\n",
      "photonics over its\n",
      "over its electronic\n",
      "its electronic counterparts\n",
      "electronic counterparts massively\n",
      "counterparts massively parallel\n",
      "massively parallel data\n",
      "parallel data transfer\n",
      "data transfer through\n",
      "transfer through wavelength\n",
      "through wavelength division\n",
      "wavelength division multiplexing\n",
      "division multiplexing in\n",
      "multiplexing in conjunction\n",
      "in conjunction with\n",
      "conjunction with frequency\n",
      "with frequency combs\n",
      "frequency combs and\n",
      "combs and extremely\n",
      "and extremely high\n",
      "extremely high data\n",
      "high data modulation\n",
      "data modulation speeds\n",
      "modulation speeds .\n",
      "speeds . their\n",
      ". their system\n",
      "their system can\n",
      "system can execute\n",
      "can execute trillions\n",
      "execute trillions of\n",
      "trillions of multiplyaccumulate\n",
      "of multiplyaccumulate operations\n",
      "multiplyaccumulate operations per\n",
      "operations per second\n",
      "per second indicating\n",
      "second indicating the\n",
      "indicating the potential\n",
      "the potential of\n",
      "potential of integrated\n",
      "of integrated photonics\n",
      "integrated photonics in\n",
      "photonics in dataheavy\n",
      "in dataheavy ai\n",
      "dataheavy ai applications.largescale\n",
      "ai applications.largescale automatic\n",
      "applications.largescale automatic speech\n",
      "automatic speech recognition\n",
      "speech recognition is\n",
      "recognition is the\n",
      "is the first\n",
      "the first and\n",
      "first and most\n",
      "and most convincing\n",
      "most convincing successful\n",
      "convincing successful case\n",
      "successful case of\n",
      "case of deep\n",
      "of deep learning\n",
      "deep learning .\n",
      "learning . lstm\n",
      ". lstm rnns\n",
      "lstm rnns can\n",
      "rnns can learn\n",
      "can learn very\n",
      "learn very deep\n",
      "very deep learning\n",
      "deep learning tasks\n",
      "learning tasks that\n",
      "tasks that involve\n",
      "that involve multisecond\n",
      "involve multisecond intervals\n",
      "multisecond intervals containing\n",
      "intervals containing speech\n",
      "containing speech events\n",
      "speech events separated\n",
      "events separated by\n",
      "separated by thousands\n",
      "by thousands of\n",
      "thousands of discrete\n",
      "of discrete time\n",
      "discrete time steps\n",
      "time steps where\n",
      "steps where one\n",
      "where one time\n",
      "one time step\n",
      "time step corresponds\n",
      "step corresponds to\n",
      "corresponds to about\n",
      "to about ms.\n",
      "about ms. lstm\n",
      "ms. lstm with\n",
      "lstm with forget\n",
      "with forget gates\n",
      "forget gates is\n",
      "gates is competitive\n",
      "is competitive with\n",
      "competitive with traditional\n",
      "with traditional speech\n",
      "traditional speech recognizers\n",
      "speech recognizers on\n",
      "recognizers on certain\n",
      "on certain tasks.the\n",
      "certain tasks.the initial\n",
      "tasks.the initial success\n",
      "initial success in\n",
      "success in speech\n",
      "in speech recognition\n",
      "speech recognition was\n",
      "recognition was based\n",
      "was based on\n",
      "based on smallscale\n",
      "on smallscale recognition\n",
      "smallscale recognition tasks\n",
      "recognition tasks based\n",
      "tasks based on\n",
      "based on timit\n",
      "on timit .\n",
      "timit . the\n",
      ". the data\n",
      "the data set\n",
      "data set contains\n",
      "set contains speakers\n",
      "contains speakers from\n",
      "speakers from eight\n",
      "from eight major\n",
      "eight major dialects\n",
      "major dialects of\n",
      "dialects of american\n",
      "of american english\n",
      "american english where\n",
      "english where each\n",
      "where each speaker\n",
      "each speaker reads\n",
      "speaker reads sentences\n",
      "reads sentences .\n",
      "sentences . its\n",
      ". its small\n",
      "its small size\n",
      "small size lets\n",
      "size lets many\n",
      "lets many configurations\n",
      "many configurations be\n",
      "configurations be tried\n",
      "be tried .\n",
      "tried . more\n",
      ". more importantly\n",
      "more importantly the\n",
      "importantly the timit\n",
      "the timit task\n",
      "timit task concerns\n",
      "task concerns phonesequence\n",
      "concerns phonesequence recognition\n",
      "phonesequence recognition which\n",
      "recognition which unlike\n",
      "which unlike wordsequence\n",
      "unlike wordsequence recognition\n",
      "wordsequence recognition allows\n",
      "recognition allows weak\n",
      "allows weak phone\n",
      "weak phone bigram\n",
      "phone bigram language\n",
      "bigram language models\n",
      "language models .\n",
      "models . this\n",
      ". this lets\n",
      "this lets the\n",
      "lets the strength\n",
      "the strength of\n",
      "strength of the\n",
      "of the acoustic\n",
      "the acoustic modeling\n",
      "acoustic modeling aspects\n",
      "modeling aspects of\n",
      "aspects of speech\n",
      "of speech recognition\n",
      "speech recognition be\n",
      "recognition be more\n",
      "be more easily\n",
      "more easily analyzed\n",
      "easily analyzed .\n",
      "analyzed . the\n",
      ". the error\n",
      "the error rates\n",
      "error rates listed\n",
      "rates listed below\n",
      "listed below including\n",
      "below including these\n",
      "including these early\n",
      "these early results\n",
      "early results and\n",
      "results and measured\n",
      "and measured as\n",
      "measured as percent\n",
      "as percent phone\n",
      "percent phone error\n",
      "phone error rates\n",
      "error rates per\n",
      "rates per have\n",
      "per have been\n",
      "have been summarized\n",
      "been summarized since\n",
      "summarized since .the\n",
      "since .the debut\n",
      ".the debut of\n",
      "debut of dnns\n",
      "of dnns for\n",
      "dnns for speaker\n",
      "for speaker recognition\n",
      "speaker recognition in\n",
      "recognition in the\n",
      "in the late\n",
      "the late s\n",
      "late s and\n",
      "s and speech\n",
      "and speech recognition\n",
      "speech recognition around\n",
      "recognition around and\n",
      "around and of\n",
      "and of lstm\n",
      "of lstm around\n",
      "lstm around accelerated\n",
      "around accelerated progress\n",
      "accelerated progress in\n",
      "progress in eight\n",
      "in eight major\n",
      "eight major areasall\n",
      "major areasall major\n",
      "areasall major commercial\n",
      "major commercial speech\n",
      "commercial speech recognition\n",
      "speech recognition systems\n",
      "recognition systems e.g\n",
      "systems e.g .\n",
      "e.g . microsoft\n",
      ". microsoft cortana\n",
      "microsoft cortana xbox\n",
      "cortana xbox skype\n",
      "xbox skype translator\n",
      "skype translator amazon\n",
      "translator amazon alexa\n",
      "amazon alexa google\n",
      "alexa google now\n",
      "google now apple\n",
      "now apple siri\n",
      "apple siri baidu\n",
      "siri baidu and\n",
      "baidu and iflytek\n",
      "and iflytek voice\n",
      "iflytek voice search\n",
      "voice search and\n",
      "search and a\n",
      "and a range\n",
      "a range of\n",
      "range of nuance\n",
      "of nuance speech\n",
      "nuance speech products\n",
      "speech products etc\n",
      "products etc .\n",
      "etc . are\n",
      ". are based\n",
      "are based on\n",
      "based on deep\n",
      "on deep learning.a\n",
      "deep learning.a common\n",
      "learning.a common evaluation\n",
      "common evaluation set\n",
      "evaluation set for\n",
      "set for image\n",
      "for image classification\n",
      "image classification is\n",
      "classification is the\n",
      "is the mnist\n",
      "the mnist database\n",
      "mnist database data\n",
      "database data set\n",
      "data set .\n",
      "set . mnist\n",
      ". mnist is\n",
      "mnist is composed\n",
      "is composed of\n",
      "composed of handwritten\n",
      "of handwritten digits\n",
      "handwritten digits and\n",
      "digits and includes\n",
      "and includes training\n",
      "includes training examples\n",
      "training examples and\n",
      "examples and test\n",
      "and test examples\n",
      "test examples .\n",
      "examples . as\n",
      ". as with\n",
      "as with timit\n",
      "with timit its\n",
      "timit its small\n",
      "its small size\n",
      "small size lets\n",
      "size lets users\n",
      "lets users test\n",
      "users test multiple\n",
      "test multiple configurations\n",
      "multiple configurations .\n",
      "configurations . a\n",
      ". a comprehensive\n",
      "a comprehensive list\n",
      "comprehensive list of\n",
      "list of results\n",
      "of results on\n",
      "results on this\n",
      "on this set\n",
      "this set is\n",
      "set is available.deep\n",
      "is available.deep learningbased\n",
      "available.deep learningbased image\n",
      "learningbased image recognition\n",
      "image recognition has\n",
      "recognition has become\n",
      "has become superhuman\n",
      "become superhuman producing\n",
      "superhuman producing more\n",
      "producing more accurate\n",
      "more accurate results\n",
      "accurate results than\n",
      "results than human\n",
      "than human contestants\n",
      "human contestants .\n",
      "contestants . this\n",
      ". this first\n",
      "this first occurred\n",
      "first occurred in\n",
      "occurred in in\n",
      "in in recognition\n",
      "in recognition of\n",
      "recognition of traffic\n",
      "of traffic signs\n",
      "traffic signs and\n",
      "signs and in\n",
      "and in with\n",
      "in with recognition\n",
      "with recognition of\n",
      "recognition of human\n",
      "of human faces.deep\n",
      "human faces.deep learningtrained\n",
      "faces.deep learningtrained vehicles\n",
      "learningtrained vehicles now\n",
      "vehicles now interpret\n",
      "now interpret camera\n",
      "interpret camera views\n",
      "camera views .\n",
      "views . another\n",
      ". another example\n",
      "another example is\n",
      "example is facial\n",
      "is facial dysmorphology\n",
      "facial dysmorphology novel\n",
      "dysmorphology novel analysis\n",
      "novel analysis fdna\n",
      "analysis fdna used\n",
      "fdna used to\n",
      "used to analyze\n",
      "to analyze cases\n",
      "analyze cases of\n",
      "cases of human\n",
      "of human malformation\n",
      "human malformation connected\n",
      "malformation connected to\n",
      "connected to a\n",
      "to a large\n",
      "a large database\n",
      "large database of\n",
      "database of genetic\n",
      "of genetic syndromes.closely\n",
      "genetic syndromes.closely related\n",
      "syndromes.closely related to\n",
      "related to the\n",
      "to the progress\n",
      "the progress that\n",
      "progress that has\n",
      "that has been\n",
      "has been made\n",
      "been made in\n",
      "made in image\n",
      "in image recognition\n",
      "image recognition is\n",
      "recognition is the\n",
      "is the increasing\n",
      "the increasing application\n",
      "increasing application of\n",
      "application of deep\n",
      "of deep learning\n",
      "deep learning techniques\n",
      "learning techniques to\n",
      "techniques to various\n",
      "to various visual\n",
      "various visual art\n",
      "visual art tasks\n",
      "art tasks .\n",
      "tasks . dnns\n",
      ". dnns have\n",
      "dnns have proven\n",
      "have proven themselves\n",
      "proven themselves capable\n",
      "themselves capable for\n",
      "capable for example\n",
      "for example ofneural\n",
      "example ofneural networks\n",
      "ofneural networks have\n",
      "networks have been\n",
      "have been used\n",
      "been used for\n",
      "used for implementing\n",
      "for implementing language\n",
      "implementing language models\n",
      "language models since\n",
      "models since the\n",
      "since the early\n",
      "the early s.\n",
      "early s. lstm\n",
      "s. lstm helped\n",
      "lstm helped to\n",
      "helped to improve\n",
      "to improve machine\n",
      "improve machine translation\n",
      "machine translation and\n",
      "translation and language\n",
      "and language modeling.other\n",
      "language modeling.other key\n",
      "modeling.other key techniques\n",
      "key techniques in\n",
      "techniques in this\n",
      "in this field\n",
      "this field are\n",
      "field are negative\n",
      "are negative sampling\n",
      "negative sampling and\n",
      "sampling and word\n",
      "and word embedding\n",
      "word embedding .\n",
      "embedding . word\n",
      ". word embedding\n",
      "word embedding such\n",
      "embedding such as\n",
      "such as wordvec\n",
      "as wordvec can\n",
      "wordvec can be\n",
      "can be thought\n",
      "be thought of\n",
      "thought of as\n",
      "of as a\n",
      "as a representational\n",
      "a representational layer\n",
      "representational layer in\n",
      "layer in a\n",
      "in a deep\n",
      "a deep learning\n",
      "deep learning architecture\n",
      "learning architecture that\n",
      "architecture that transforms\n",
      "that transforms an\n",
      "transforms an atomic\n",
      "an atomic word\n",
      "atomic word into\n",
      "word into a\n",
      "into a positional\n",
      "a positional representation\n",
      "positional representation of\n",
      "representation of the\n",
      "of the word\n",
      "the word relative\n",
      "word relative to\n",
      "relative to other\n",
      "to other words\n",
      "other words in\n",
      "words in the\n",
      "in the dataset\n",
      "the dataset the\n",
      "dataset the position\n",
      "the position is\n",
      "position is represented\n",
      "is represented as\n",
      "represented as a\n",
      "as a point\n",
      "a point in\n",
      "point in a\n",
      "in a vector\n",
      "a vector space\n",
      "vector space .\n",
      "space . using\n",
      ". using word\n",
      "using word embedding\n",
      "word embedding as\n",
      "embedding as an\n",
      "as an rnn\n",
      "an rnn input\n",
      "rnn input layer\n",
      "input layer allows\n",
      "layer allows the\n",
      "allows the network\n",
      "the network to\n",
      "network to parse\n",
      "to parse sentences\n",
      "parse sentences and\n",
      "sentences and phrases\n",
      "and phrases using\n",
      "phrases using an\n",
      "using an effective\n",
      "an effective compositional\n",
      "effective compositional vector\n",
      "compositional vector grammar\n",
      "vector grammar .\n",
      "grammar . a\n",
      ". a compositional\n",
      "a compositional vector\n",
      "compositional vector grammar\n",
      "vector grammar can\n",
      "grammar can be\n",
      "can be thought\n",
      "be thought of\n",
      "thought of as\n",
      "of as probabilistic\n",
      "as probabilistic context\n",
      "probabilistic context free\n",
      "context free grammar\n",
      "free grammar pcfg\n",
      "grammar pcfg implemented\n",
      "pcfg implemented by\n",
      "implemented by an\n",
      "by an rnn\n",
      "an rnn .\n",
      "rnn . recursive\n",
      ". recursive autoencoders\n",
      "recursive autoencoders built\n",
      "autoencoders built atop\n",
      "built atop word\n",
      "atop word embeddings\n",
      "word embeddings can\n",
      "embeddings can assess\n",
      "can assess sentence\n",
      "assess sentence similarity\n",
      "sentence similarity and\n",
      "similarity and detect\n",
      "and detect paraphrasing\n",
      "detect paraphrasing .\n",
      "paraphrasing . deep\n",
      ". deep neural\n",
      "deep neural architectures\n",
      "neural architectures provide\n",
      "architectures provide the\n",
      "provide the best\n",
      "the best results\n",
      "best results for\n",
      "results for constituency\n",
      "for constituency parsing\n",
      "constituency parsing sentiment\n",
      "parsing sentiment analysis\n",
      "sentiment analysis information\n",
      "analysis information retrieval\n",
      "information retrieval spoken\n",
      "retrieval spoken language\n",
      "spoken language understanding\n",
      "language understanding machine\n",
      "understanding machine translation\n",
      "machine translation contextual\n",
      "translation contextual entity\n",
      "contextual entity linking\n",
      "entity linking writing\n",
      "linking writing style\n",
      "writing style recognition\n",
      "style recognition text\n",
      "recognition text classification\n",
      "text classification and\n",
      "classification and others.recent\n",
      "and others.recent developments\n",
      "others.recent developments generalize\n",
      "developments generalize word\n",
      "generalize word embedding\n",
      "word embedding to\n",
      "embedding to sentence\n",
      "to sentence embedding.google\n",
      "sentence embedding.google translate\n",
      "embedding.google translate gt\n",
      "translate gt uses\n",
      "gt uses a\n",
      "uses a large\n",
      "a large endtoend\n",
      "large endtoend long\n",
      "endtoend long shortterm\n",
      "long shortterm memory\n",
      "shortterm memory lstm\n",
      "memory lstm network\n",
      "lstm network .\n",
      "network . google\n",
      ". google neural\n",
      "google neural machine\n",
      "neural machine translation\n",
      "machine translation gnmt\n",
      "translation gnmt uses\n",
      "gnmt uses an\n",
      "uses an examplebased\n",
      "an examplebased machine\n",
      "examplebased machine translation\n",
      "machine translation method\n",
      "translation method in\n",
      "method in which\n",
      "in which the\n",
      "which the system\n",
      "the system learns\n",
      "system learns from\n",
      "learns from millions\n",
      "from millions of\n",
      "millions of examples\n",
      "of examples .\n",
      "examples . it\n",
      ". it translates\n",
      "it translates whole\n",
      "translates whole sentences\n",
      "whole sentences at\n",
      "sentences at a\n",
      "at a time\n",
      "a time rather\n",
      "time rather than\n",
      "rather than pieces\n",
      "than pieces .\n",
      "pieces . google\n",
      ". google translate\n",
      "google translate supports\n",
      "translate supports over\n",
      "supports over one\n",
      "over one hundred\n",
      "one hundred languages\n",
      "hundred languages .\n",
      "languages . the\n",
      ". the network\n",
      "the network encodes\n",
      "network encodes the\n",
      "encodes the semantics\n",
      "the semantics of\n",
      "semantics of the\n",
      "of the sentence\n",
      "the sentence rather\n",
      "sentence rather than\n",
      "rather than simply\n",
      "than simply memorizing\n",
      "simply memorizing phrasetophrase\n",
      "memorizing phrasetophrase translations\n",
      "phrasetophrase translations .\n",
      "translations . gt\n",
      ". gt uses\n",
      "gt uses english\n",
      "uses english as\n",
      "english as an\n",
      "as an intermediate\n",
      "an intermediate between\n",
      "intermediate between most\n",
      "between most language\n",
      "most language pairs.a\n",
      "language pairs.a large\n",
      "pairs.a large percentage\n",
      "large percentage of\n",
      "percentage of candidate\n",
      "of candidate drugs\n",
      "candidate drugs fail\n",
      "drugs fail to\n",
      "fail to win\n",
      "to win regulatory\n",
      "win regulatory approval\n",
      "regulatory approval .\n",
      "approval . these\n",
      ". these failures\n",
      "these failures are\n",
      "failures are caused\n",
      "are caused by\n",
      "caused by insufficient\n",
      "by insufficient efficacy\n",
      "insufficient efficacy ontarget\n",
      "efficacy ontarget effect\n",
      "ontarget effect undesired\n",
      "effect undesired interactions\n",
      "undesired interactions offtarget\n",
      "interactions offtarget effects\n",
      "offtarget effects or\n",
      "effects or unanticipated\n",
      "or unanticipated toxic\n",
      "unanticipated toxic effects\n",
      "toxic effects .\n",
      "effects . research\n",
      ". research has\n",
      "research has explored\n",
      "has explored use\n",
      "explored use of\n",
      "use of deep\n",
      "of deep learning\n",
      "deep learning to\n",
      "learning to predict\n",
      "to predict the\n",
      "predict the biomolecular\n",
      "the biomolecular targets\n",
      "biomolecular targets offtargets\n",
      "targets offtargets and\n",
      "offtargets and toxic\n",
      "and toxic effects\n",
      "toxic effects of\n",
      "effects of environmental\n",
      "of environmental chemicals\n",
      "environmental chemicals in\n",
      "chemicals in nutrients\n",
      "in nutrients household\n",
      "nutrients household products\n",
      "household products and\n",
      "products and drugs.atomnet\n",
      "and drugs.atomnet is\n",
      "drugs.atomnet is a\n",
      "is a deep\n",
      "a deep learning\n",
      "deep learning system\n",
      "learning system for\n",
      "system for structurebased\n",
      "for structurebased rational\n",
      "structurebased rational drug\n",
      "rational drug design\n",
      "drug design .\n",
      "design . atomnet\n",
      ". atomnet was\n",
      "atomnet was used\n",
      "was used to\n",
      "used to predict\n",
      "to predict novel\n",
      "predict novel candidate\n",
      "novel candidate biomolecules\n",
      "candidate biomolecules for\n",
      "biomolecules for disease\n",
      "for disease targets\n",
      "disease targets such\n",
      "targets such as\n",
      "such as the\n",
      "as the ebola\n",
      "the ebola virus\n",
      "ebola virus and\n",
      "virus and multiple\n",
      "and multiple sclerosis.in\n",
      "multiple sclerosis.in graph\n",
      "sclerosis.in graph neural\n",
      "graph neural networks\n",
      "neural networks were\n",
      "networks were used\n",
      "were used for\n",
      "used for the\n",
      "for the first\n",
      "the first time\n",
      "first time to\n",
      "time to predict\n",
      "to predict various\n",
      "predict various properties\n",
      "various properties of\n",
      "properties of molecules\n",
      "of molecules in\n",
      "molecules in a\n",
      "in a large\n",
      "a large toxicology\n",
      "large toxicology data\n",
      "toxicology data set\n",
      "data set .\n",
      "set . in\n",
      ". in generative\n",
      "in generative neural\n",
      "generative neural networks\n",
      "neural networks were\n",
      "networks were used\n",
      "were used to\n",
      "used to produce\n",
      "to produce molecules\n",
      "produce molecules that\n",
      "molecules that were\n",
      "that were validated\n",
      "were validated experimentally\n",
      "validated experimentally all\n",
      "experimentally all the\n",
      "all the way\n",
      "the way into\n",
      "way into mice.deep\n",
      "into mice.deep reinforcement\n",
      "mice.deep reinforcement learning\n",
      "reinforcement learning has\n",
      "learning has been\n",
      "has been used\n",
      "been used to\n",
      "used to approximate\n",
      "to approximate the\n",
      "approximate the value\n",
      "the value of\n",
      "value of possible\n",
      "of possible direct\n",
      "possible direct marketing\n",
      "direct marketing actions\n",
      "marketing actions defined\n",
      "actions defined in\n",
      "defined in terms\n",
      "in terms of\n",
      "terms of rfm\n",
      "of rfm variables\n",
      "rfm variables .\n",
      "variables . the\n",
      ". the estimated\n",
      "the estimated value\n",
      "estimated value function\n",
      "value function was\n",
      "function was shown\n",
      "was shown to\n",
      "shown to have\n",
      "to have a\n",
      "have a natural\n",
      "a natural interpretation\n",
      "natural interpretation as\n",
      "interpretation as customer\n",
      "as customer lifetime\n",
      "customer lifetime value.recommendation\n",
      "lifetime value.recommendation systems\n",
      "value.recommendation systems have\n",
      "systems have used\n",
      "have used deep\n",
      "used deep learning\n",
      "deep learning to\n",
      "learning to extract\n",
      "to extract meaningful\n",
      "extract meaningful features\n",
      "meaningful features for\n",
      "features for a\n",
      "for a latent\n",
      "a latent factor\n",
      "latent factor model\n",
      "factor model for\n",
      "model for contentbased\n",
      "for contentbased music\n",
      "contentbased music and\n",
      "music and journal\n",
      "and journal recommendations\n",
      "journal recommendations .\n",
      "recommendations . multiview\n",
      ". multiview deep\n",
      "multiview deep learning\n",
      "deep learning has\n",
      "learning has been\n",
      "has been applied\n",
      "been applied for\n",
      "applied for learning\n",
      "for learning user\n",
      "learning user preferences\n",
      "user preferences from\n",
      "preferences from multiple\n",
      "from multiple domains\n",
      "multiple domains .\n",
      "domains . the\n",
      ". the model\n",
      "the model uses\n",
      "model uses a\n",
      "uses a hybrid\n",
      "a hybrid collaborative\n",
      "hybrid collaborative and\n",
      "collaborative and contentbased\n",
      "and contentbased approach\n",
      "contentbased approach and\n",
      "approach and enhances\n",
      "and enhances recommendations\n",
      "enhances recommendations in\n",
      "recommendations in multiple\n",
      "in multiple tasks.an\n",
      "multiple tasks.an autoencoder\n",
      "tasks.an autoencoder ann\n",
      "autoencoder ann was\n",
      "ann was used\n",
      "was used in\n",
      "used in bioinformatics\n",
      "in bioinformatics to\n",
      "bioinformatics to predict\n",
      "to predict gene\n",
      "predict gene ontology\n",
      "gene ontology annotations\n",
      "ontology annotations and\n",
      "annotations and genefunction\n",
      "and genefunction relationships.in\n",
      "genefunction relationships.in medical\n",
      "relationships.in medical informatics\n",
      "medical informatics deep\n",
      "informatics deep learning\n",
      "deep learning was\n",
      "learning was used\n",
      "was used to\n",
      "used to predict\n",
      "to predict sleep\n",
      "predict sleep quality\n",
      "sleep quality based\n",
      "quality based on\n",
      "based on data\n",
      "on data from\n",
      "data from wearables\n",
      "from wearables and\n",
      "wearables and predictions\n",
      "and predictions of\n",
      "predictions of health\n",
      "of health complications\n",
      "health complications from\n",
      "complications from electronic\n",
      "from electronic health\n",
      "electronic health record\n",
      "health record data.deep\n",
      "record data.deep learning\n",
      "data.deep learning has\n",
      "learning has been\n",
      "has been shown\n",
      "been shown to\n",
      "shown to produce\n",
      "to produce competitive\n",
      "produce competitive results\n",
      "competitive results in\n",
      "results in medical\n",
      "in medical application\n",
      "medical application such\n",
      "application such as\n",
      "such as cancer\n",
      "as cancer cell\n",
      "cancer cell classification\n",
      "cell classification lesion\n",
      "classification lesion detection\n",
      "lesion detection organ\n",
      "detection organ segmentation\n",
      "organ segmentation and\n",
      "segmentation and image\n",
      "and image enhancement\n",
      "image enhancement .\n",
      "enhancement . modern\n",
      ". modern deep\n",
      "modern deep learning\n",
      "deep learning tools\n",
      "learning tools demonstrate\n",
      "tools demonstrate the\n",
      "demonstrate the high\n",
      "the high accuracy\n",
      "high accuracy of\n",
      "accuracy of detecting\n",
      "of detecting various\n",
      "detecting various diseases\n",
      "various diseases and\n",
      "diseases and the\n",
      "and the helpfulness\n",
      "the helpfulness of\n",
      "helpfulness of their\n",
      "of their use\n",
      "their use by\n",
      "use by specialists\n",
      "by specialists to\n",
      "specialists to improve\n",
      "to improve the\n",
      "improve the diagnosis\n",
      "the diagnosis efficiency.finding\n",
      "diagnosis efficiency.finding the\n",
      "efficiency.finding the appropriate\n",
      "the appropriate mobile\n",
      "appropriate mobile audience\n",
      "mobile audience for\n",
      "audience for mobile\n",
      "for mobile advertising\n",
      "mobile advertising is\n",
      "advertising is always\n",
      "is always challenging\n",
      "always challenging since\n",
      "challenging since many\n",
      "since many data\n",
      "many data points\n",
      "data points must\n",
      "points must be\n",
      "must be considered\n",
      "be considered and\n",
      "considered and analyzed\n",
      "and analyzed before\n",
      "analyzed before a\n",
      "before a target\n",
      "a target segment\n",
      "target segment can\n",
      "segment can be\n",
      "can be created\n",
      "be created and\n",
      "created and used\n",
      "and used in\n",
      "used in ad\n",
      "in ad serving\n",
      "ad serving by\n",
      "serving by any\n",
      "by any ad\n",
      "any ad server\n",
      "ad server .\n",
      "server . deep\n",
      ". deep learning\n",
      "deep learning has\n",
      "learning has been\n",
      "has been used\n",
      "been used to\n",
      "used to interpret\n",
      "to interpret large\n",
      "interpret large manydimensioned\n",
      "large manydimensioned advertising\n",
      "manydimensioned advertising datasets\n",
      "advertising datasets .\n",
      "datasets . many\n",
      ". many data\n",
      "many data points\n",
      "data points are\n",
      "points are collected\n",
      "are collected during\n",
      "collected during the\n",
      "during the requestserveclick\n",
      "the requestserveclick internet\n",
      "requestserveclick internet advertising\n",
      "internet advertising cycle\n",
      "advertising cycle .\n",
      "cycle . this\n",
      ". this information\n",
      "this information can\n",
      "information can form\n",
      "can form the\n",
      "form the basis\n",
      "the basis of\n",
      "basis of machine\n",
      "of machine learning\n",
      "machine learning to\n",
      "learning to improve\n",
      "to improve ad\n",
      "improve ad selection.deep\n",
      "ad selection.deep learning\n",
      "selection.deep learning has\n",
      "learning has been\n",
      "has been successfully\n",
      "been successfully applied\n",
      "successfully applied to\n",
      "applied to inverse\n",
      "to inverse problems\n",
      "inverse problems such\n",
      "problems such as\n",
      "such as denoising\n",
      "as denoising superresolution\n",
      "denoising superresolution inpainting\n",
      "superresolution inpainting and\n",
      "inpainting and film\n",
      "and film colorization\n",
      "film colorization .\n",
      "colorization . these\n",
      ". these applications\n",
      "these applications include\n",
      "applications include learning\n",
      "include learning methods\n",
      "learning methods such\n",
      "methods such as\n",
      "such as shrinkage\n",
      "as shrinkage fields\n",
      "shrinkage fields for\n",
      "fields for effective\n",
      "for effective image\n",
      "effective image restoration\n",
      "image restoration which\n",
      "restoration which trains\n",
      "which trains on\n",
      "trains on an\n",
      "on an image\n",
      "an image dataset\n",
      "image dataset and\n",
      "dataset and deep\n",
      "and deep image\n",
      "deep image prior\n",
      "image prior which\n",
      "prior which trains\n",
      "which trains on\n",
      "trains on the\n",
      "on the image\n",
      "the image that\n",
      "image that needs\n",
      "that needs restoration.deep\n",
      "needs restoration.deep learning\n",
      "restoration.deep learning is\n",
      "learning is being\n",
      "is being successfully\n",
      "being successfully applied\n",
      "successfully applied to\n",
      "applied to financial\n",
      "to financial fraud\n",
      "financial fraud detection\n",
      "fraud detection tax\n",
      "detection tax evasion\n",
      "tax evasion detection\n",
      "evasion detection and\n",
      "detection and antimoney\n",
      "and antimoney laundering\n",
      "antimoney laundering .\n",
      "laundering . a\n",
      ". a potentially\n",
      "a potentially impressive\n",
      "potentially impressive demonstration\n",
      "impressive demonstration of\n",
      "demonstration of unsupervised\n",
      "of unsupervised learning\n",
      "unsupervised learning as\n",
      "learning as prosecution\n",
      "as prosecution of\n",
      "prosecution of financial\n",
      "of financial crime\n",
      "financial crime is\n",
      "crime is required\n",
      "is required to\n",
      "required to produce\n",
      "to produce training\n",
      "produce training data.also\n",
      "training data.also of\n",
      "data.also of note\n",
      "of note is\n",
      "note is that\n",
      "is that while\n",
      "that while the\n",
      "while the state\n",
      "the state of\n",
      "state of the\n",
      "of the art\n",
      "the art model\n",
      "art model in\n",
      "model in automated\n",
      "in automated financial\n",
      "automated financial crime\n",
      "financial crime detection\n",
      "crime detection has\n",
      "detection has existed\n",
      "has existed for\n",
      "existed for quite\n",
      "for quite some\n",
      "quite some time\n",
      "some time the\n",
      "time the applications\n",
      "the applications for\n",
      "applications for deep\n",
      "for deep learning\n",
      "deep learning referred\n",
      "learning referred to\n",
      "referred to here\n",
      "to here dramatically\n",
      "here dramatically under\n",
      "dramatically under perform\n",
      "under perform much\n",
      "perform much simpler\n",
      "much simpler theoretical\n",
      "simpler theoretical models\n",
      "theoretical models .\n",
      "models . one\n",
      ". one such\n",
      "one such yet\n",
      "such yet to\n",
      "yet to be\n",
      "to be implemented\n",
      "be implemented model\n",
      "implemented model the\n",
      "model the sensor\n",
      "the sensor location\n",
      "sensor location heuristic\n",
      "location heuristic and\n",
      "heuristic and simple\n",
      "and simple any\n",
      "simple any human\n",
      "any human detection\n",
      "human detection for\n",
      "detection for financial\n",
      "for financial crimes\n",
      "financial crimes slhsahdfc\n",
      "crimes slhsahdfc is\n",
      "slhsahdfc is an\n",
      "is an example.the\n",
      "an example.the model\n",
      "example.the model works\n",
      "model works with\n",
      "works with the\n",
      "with the simple\n",
      "the simple heuristic\n",
      "simple heuristic of\n",
      "heuristic of choosing\n",
      "of choosing where\n",
      "choosing where it\n",
      "where it gets\n",
      "it gets its\n",
      "gets its input\n",
      "its input data\n",
      "input data .\n",
      "data . by\n",
      ". by placing\n",
      "by placing the\n",
      "placing the sensors\n",
      "the sensors by\n",
      "sensors by places\n",
      "by places frequented\n",
      "places frequented by\n",
      "frequented by large\n",
      "by large concentrations\n",
      "large concentrations of\n",
      "concentrations of wealth\n",
      "of wealth and\n",
      "wealth and power\n",
      "and power and\n",
      "power and then\n",
      "and then simply\n",
      "then simply identifying\n",
      "simply identifying any\n",
      "identifying any live\n",
      "any live human\n",
      "live human being\n",
      "human being it\n",
      "being it turns\n",
      "it turns out\n",
      "turns out that\n",
      "out that the\n",
      "that the automated\n",
      "the automated detection\n",
      "automated detection of\n",
      "detection of financial\n",
      "of financial crime\n",
      "financial crime is\n",
      "crime is accomplished\n",
      "is accomplished at\n",
      "accomplished at very\n",
      "at very high\n",
      "very high accuracies\n",
      "high accuracies and\n",
      "accuracies and very\n",
      "and very high\n",
      "very high confidence\n",
      "high confidence levels\n",
      "confidence levels .\n",
      "levels . even\n",
      ". even better\n",
      "even better the\n",
      "better the model\n",
      "the model has\n",
      "model has shown\n",
      "has shown to\n",
      "shown to be\n",
      "to be extremely\n",
      "be extremely effective\n",
      "extremely effective at\n",
      "effective at identifying\n",
      "at identifying not\n",
      "identifying not just\n",
      "not just crime\n",
      "just crime but\n",
      "crime but large\n",
      "but large very\n",
      "large very destructive\n",
      "very destructive and\n",
      "destructive and egregious\n",
      "and egregious crime\n",
      "egregious crime .\n",
      "crime . due\n",
      ". due to\n",
      "due to the\n",
      "to the effectiveness\n",
      "the effectiveness of\n",
      "effectiveness of such\n",
      "of such models\n",
      "such models it\n",
      "models it is\n",
      "it is highly\n",
      "is highly likely\n",
      "highly likely that\n",
      "likely that applications\n",
      "that applications to\n",
      "applications to financial\n",
      "to financial crime\n",
      "financial crime detection\n",
      "crime detection by\n",
      "detection by deep\n",
      "by deep learning\n",
      "deep learning will\n",
      "learning will never\n",
      "will never be\n",
      "never be able\n",
      "be able to\n",
      "able to compete.the\n",
      "to compete.the united\n",
      "compete.the united states\n",
      "united states department\n",
      "states department of\n",
      "department of defense\n",
      "of defense applied\n",
      "defense applied deep\n",
      "applied deep learning\n",
      "deep learning to\n",
      "learning to train\n",
      "to train robots\n",
      "train robots in\n",
      "robots in new\n",
      "in new tasks\n",
      "new tasks through\n",
      "tasks through observation.physics\n",
      "through observation.physics informed\n",
      "observation.physics informed neural\n",
      "informed neural networks\n",
      "neural networks have\n",
      "networks have been\n",
      "have been used\n",
      "been used to\n",
      "used to solve\n",
      "to solve partial\n",
      "solve partial differential\n",
      "partial differential equations\n",
      "differential equations in\n",
      "equations in both\n",
      "in both forward\n",
      "both forward and\n",
      "forward and inverse\n",
      "and inverse problems\n",
      "inverse problems in\n",
      "problems in a\n",
      "in a data\n",
      "a data driven\n",
      "data driven manner\n",
      "driven manner .\n",
      "manner . one\n",
      ". one example\n",
      "one example is\n",
      "example is the\n",
      "is the reconstructing\n",
      "the reconstructing fluid\n",
      "reconstructing fluid flow\n",
      "fluid flow governed\n",
      "flow governed by\n",
      "governed by the\n",
      "by the navierstokes\n",
      "the navierstokes equations\n",
      "navierstokes equations .\n",
      "equations . using\n",
      ". using physics\n",
      "using physics informed\n",
      "physics informed neural\n",
      "informed neural networks\n",
      "neural networks does\n",
      "networks does not\n",
      "does not require\n",
      "not require the\n",
      "require the often\n",
      "the often expensive\n",
      "often expensive mesh\n",
      "expensive mesh generation\n",
      "mesh generation that\n",
      "generation that conventional\n",
      "that conventional cfd\n",
      "conventional cfd methods\n",
      "cfd methods relies\n",
      "methods relies on.image\n",
      "relies on.image reconstruction\n",
      "on.image reconstruction is\n",
      "reconstruction is the\n",
      "is the reconstruction\n",
      "the reconstruction of\n",
      "reconstruction of the\n",
      "of the underlying\n",
      "the underlying images\n",
      "underlying images from\n",
      "images from the\n",
      "from the imagerelated\n",
      "the imagerelated measurements\n",
      "imagerelated measurements .\n",
      "measurements . several\n",
      ". several works\n",
      "several works showed\n",
      "works showed the\n",
      "showed the better\n",
      "the better and\n",
      "better and superior\n",
      "and superior performance\n",
      "superior performance of\n",
      "performance of the\n",
      "of the deep\n",
      "the deep learning\n",
      "deep learning methods\n",
      "learning methods compared\n",
      "methods compared to\n",
      "compared to analytical\n",
      "to analytical methods\n",
      "analytical methods for\n",
      "methods for various\n",
      "for various applications\n",
      "various applications e.g\n",
      "applications e.g .\n",
      "e.g . spectral\n",
      ". spectral imaging\n",
      "spectral imaging and\n",
      "imaging and ultrasound\n",
      "and ultrasound imaging.epigenetic\n",
      "ultrasound imaging.epigenetic clockfor\n",
      "imaging.epigenetic clockfor more\n",
      "clockfor more information\n",
      "more information see\n",
      "information see epigenetic\n",
      "see epigenetic clock.an\n",
      "epigenetic clock.an epigenetic\n",
      "clock.an epigenetic clock\n",
      "epigenetic clock is\n",
      "clock is a\n",
      "is a biochemical\n",
      "a biochemical test\n",
      "biochemical test that\n",
      "test that can\n",
      "that can be\n",
      "can be used\n",
      "be used to\n",
      "used to measure\n",
      "to measure age\n",
      "measure age .\n",
      "age . galkin\n",
      ". galkin et\n",
      "galkin et al\n",
      "et al .\n",
      "al . used\n",
      ". used deep\n",
      "used deep neural\n",
      "deep neural networks\n",
      "neural networks to\n",
      "networks to train\n",
      "to train an\n",
      "train an epigenetic\n",
      "an epigenetic aging\n",
      "epigenetic aging clock\n",
      "aging clock of\n",
      "clock of unprecedented\n",
      "of unprecedented accuracy\n",
      "unprecedented accuracy using\n",
      "accuracy using blood\n",
      "using blood samples\n",
      "blood samples .\n",
      "samples . the\n",
      ". the clock\n",
      "the clock uses\n",
      "clock uses information\n",
      "uses information from\n",
      "information from cpg\n",
      "from cpg sites\n",
      "cpg sites and\n",
      "sites and predicts\n",
      "and predicts people\n",
      "predicts people with\n",
      "people with certain\n",
      "with certain conditions\n",
      "certain conditions older\n",
      "conditions older than\n",
      "older than healthy\n",
      "than healthy controls\n",
      "healthy controls ibd\n",
      "controls ibd frontotemporal\n",
      "ibd frontotemporal dementia\n",
      "frontotemporal dementia ovarian\n",
      "dementia ovarian cancer\n",
      "ovarian cancer obesity\n",
      "cancer obesity .\n",
      "obesity . the\n",
      ". the aging\n",
      "the aging clock\n",
      "aging clock is\n",
      "clock is planned\n",
      "is planned to\n",
      "planned to be\n",
      "to be released\n",
      "be released for\n",
      "released for public\n",
      "for public use\n",
      "public use in\n",
      "use in by\n",
      "in by an\n",
      "by an insilico\n",
      "an insilico medicine\n",
      "insilico medicine spinoff\n",
      "medicine spinoff company\n",
      "spinoff company deep\n",
      "company deep longevity.deep\n",
      "deep longevity.deep learning\n",
      "longevity.deep learning is\n",
      "learning is closely\n",
      "is closely related\n",
      "closely related to\n",
      "related to a\n",
      "to a class\n",
      "a class of\n",
      "class of theories\n",
      "of theories of\n",
      "theories of brain\n",
      "of brain development\n",
      "brain development specifically\n",
      "development specifically neocortical\n",
      "specifically neocortical development\n",
      "neocortical development proposed\n",
      "development proposed by\n",
      "proposed by cognitive\n",
      "by cognitive neuroscientists\n",
      "cognitive neuroscientists in\n",
      "neuroscientists in the\n",
      "in the early\n",
      "the early s.\n",
      "early s. these\n",
      "s. these developmental\n",
      "these developmental theories\n",
      "developmental theories were\n",
      "theories were instantiated\n",
      "were instantiated in\n",
      "instantiated in computational\n",
      "in computational models\n",
      "computational models making\n",
      "models making them\n",
      "making them predecessors\n",
      "them predecessors of\n",
      "predecessors of deep\n",
      "of deep learning\n",
      "deep learning systems\n",
      "learning systems .\n",
      "systems . these\n",
      ". these developmental\n",
      "these developmental models\n",
      "developmental models share\n",
      "models share the\n",
      "share the property\n",
      "the property that\n",
      "property that various\n",
      "that various proposed\n",
      "various proposed learning\n",
      "proposed learning dynamics\n",
      "learning dynamics in\n",
      "dynamics in the\n",
      "in the brain\n",
      "the brain e.g\n",
      "brain e.g .\n",
      "e.g . a\n",
      ". a wave\n",
      "a wave of\n",
      "wave of nerve\n",
      "of nerve growth\n",
      "nerve growth factor\n",
      "growth factor support\n",
      "factor support the\n",
      "support the selforganization\n",
      "the selforganization somewhat\n",
      "selforganization somewhat analogous\n",
      "somewhat analogous to\n",
      "analogous to the\n",
      "to the neural\n",
      "the neural networks\n",
      "neural networks utilized\n",
      "networks utilized in\n",
      "utilized in deep\n",
      "in deep learning\n",
      "deep learning models\n",
      "learning models .\n",
      "models . like\n",
      ". like the\n",
      "like the neocortex\n",
      "the neocortex neural\n",
      "neocortex neural networks\n",
      "neural networks employ\n",
      "networks employ a\n",
      "employ a hierarchy\n",
      "a hierarchy of\n",
      "hierarchy of layered\n",
      "of layered filters\n",
      "layered filters in\n",
      "filters in which\n",
      "in which each\n",
      "which each layer\n",
      "each layer considers\n",
      "layer considers information\n",
      "considers information from\n",
      "information from a\n",
      "from a prior\n",
      "a prior layer\n",
      "prior layer or\n",
      "layer or the\n",
      "or the operating\n",
      "the operating environment\n",
      "operating environment and\n",
      "environment and then\n",
      "and then passes\n",
      "then passes its\n",
      "passes its output\n",
      "its output and\n",
      "output and possibly\n",
      "and possibly the\n",
      "possibly the original\n",
      "the original input\n",
      "original input to\n",
      "input to other\n",
      "to other layers\n",
      "other layers .\n",
      "layers . this\n",
      ". this process\n",
      "this process yields\n",
      "process yields a\n",
      "yields a selforganizing\n",
      "a selforganizing stack\n",
      "selforganizing stack of\n",
      "stack of transducers\n",
      "of transducers welltuned\n",
      "transducers welltuned to\n",
      "welltuned to their\n",
      "to their operating\n",
      "their operating environment\n",
      "operating environment .\n",
      "environment . a\n",
      ". a description\n",
      "a description stated\n",
      "description stated ...\n",
      "stated ... the\n",
      "... the infants\n",
      "the infants brain\n",
      "infants brain seems\n",
      "brain seems to\n",
      "seems to organize\n",
      "to organize itself\n",
      "organize itself under\n",
      "itself under the\n",
      "under the influence\n",
      "the influence of\n",
      "influence of waves\n",
      "of waves of\n",
      "waves of socalled\n",
      "of socalled trophicfactors\n",
      "socalled trophicfactors ...\n",
      "trophicfactors ... different\n",
      "... different regions\n",
      "different regions of\n",
      "regions of the\n",
      "of the brain\n",
      "the brain become\n",
      "brain become connected\n",
      "become connected sequentially\n",
      "connected sequentially with\n",
      "sequentially with one\n",
      "with one layer\n",
      "one layer of\n",
      "layer of tissue\n",
      "of tissue maturing\n",
      "tissue maturing before\n",
      "maturing before another\n",
      "before another and\n",
      "another and so\n",
      "and so on\n",
      "so on until\n",
      "on until the\n",
      "until the whole\n",
      "the whole brain\n",
      "whole brain is\n",
      "brain is mature.a\n",
      "is mature.a variety\n",
      "mature.a variety of\n",
      "variety of approaches\n",
      "of approaches have\n",
      "approaches have been\n",
      "have been used\n",
      "been used to\n",
      "used to investigate\n",
      "to investigate the\n",
      "investigate the plausibility\n",
      "the plausibility of\n",
      "plausibility of deep\n",
      "of deep learning\n",
      "deep learning models\n",
      "learning models from\n",
      "models from a\n",
      "from a neurobiological\n",
      "a neurobiological perspective\n",
      "neurobiological perspective .\n",
      "perspective . on\n",
      ". on the\n",
      "on the one\n",
      "the one hand\n",
      "one hand several\n",
      "hand several variants\n",
      "several variants of\n",
      "variants of the\n",
      "of the backpropagation\n",
      "the backpropagation algorithm\n",
      "backpropagation algorithm have\n",
      "algorithm have been\n",
      "have been proposed\n",
      "been proposed in\n",
      "proposed in order\n",
      "in order to\n",
      "order to increase\n",
      "to increase its\n",
      "increase its processing\n",
      "its processing realism\n",
      "processing realism .\n",
      "realism . other\n",
      ". other researchers\n",
      "other researchers have\n",
      "researchers have argued\n",
      "have argued that\n",
      "argued that unsupervised\n",
      "that unsupervised forms\n",
      "unsupervised forms of\n",
      "forms of deep\n",
      "of deep learning\n",
      "deep learning such\n",
      "learning such as\n",
      "such as those\n",
      "as those based\n",
      "those based on\n",
      "based on hierarchical\n",
      "on hierarchical generative\n",
      "hierarchical generative models\n",
      "generative models and\n",
      "models and deep\n",
      "and deep belief\n",
      "deep belief networks\n",
      "belief networks may\n",
      "networks may be\n",
      "may be closer\n",
      "be closer to\n",
      "closer to biological\n",
      "to biological reality\n",
      "biological reality .\n",
      "reality . in\n",
      ". in this\n",
      "in this respect\n",
      "this respect generative\n",
      "respect generative neural\n",
      "generative neural network\n",
      "neural network models\n",
      "network models have\n",
      "models have been\n",
      "have been related\n",
      "been related to\n",
      "related to neurobiological\n",
      "to neurobiological evidence\n",
      "neurobiological evidence about\n",
      "evidence about samplingbased\n",
      "about samplingbased processing\n",
      "samplingbased processing in\n",
      "processing in the\n",
      "in the cerebral\n",
      "the cerebral cortex.although\n",
      "cerebral cortex.although a\n",
      "cortex.although a systematic\n",
      "a systematic comparison\n",
      "systematic comparison between\n",
      "comparison between the\n",
      "between the human\n",
      "the human brain\n",
      "human brain organization\n",
      "brain organization and\n",
      "organization and the\n",
      "and the neuronal\n",
      "the neuronal encoding\n",
      "neuronal encoding in\n",
      "encoding in deep\n",
      "in deep networks\n",
      "deep networks has\n",
      "networks has not\n",
      "has not yet\n",
      "not yet been\n",
      "yet been established\n",
      "been established several\n",
      "established several analogies\n",
      "several analogies have\n",
      "analogies have been\n",
      "have been reported\n",
      "been reported .\n",
      "reported . for\n",
      ". for example\n",
      "for example the\n",
      "example the computations\n",
      "the computations performed\n",
      "computations performed by\n",
      "performed by deep\n",
      "by deep learning\n",
      "deep learning units\n",
      "learning units could\n",
      "units could be\n",
      "could be similar\n",
      "be similar to\n",
      "similar to those\n",
      "to those of\n",
      "those of actual\n",
      "of actual neurons\n",
      "actual neurons and\n",
      "neurons and neural\n",
      "and neural populations\n",
      "neural populations .\n",
      "populations . similarly\n",
      ". similarly the\n",
      "similarly the representations\n",
      "the representations developed\n",
      "representations developed by\n",
      "developed by deep\n",
      "by deep learning\n",
      "deep learning models\n",
      "learning models are\n",
      "models are similar\n",
      "are similar to\n",
      "similar to those\n",
      "to those measured\n",
      "those measured in\n",
      "measured in the\n",
      "in the primate\n",
      "the primate visual\n",
      "primate visual system\n",
      "visual system both\n",
      "system both at\n",
      "both at the\n",
      "at the singleunit\n",
      "the singleunit and\n",
      "singleunit and at\n",
      "and at the\n",
      "at the population\n",
      "the population levels.facebooks\n",
      "population levels.facebooks ai\n",
      "levels.facebooks ai lab\n",
      "ai lab performs\n",
      "lab performs tasks\n",
      "performs tasks such\n",
      "tasks such as\n",
      "such as automatically\n",
      "as automatically tagging\n",
      "automatically tagging uploaded\n",
      "tagging uploaded pictures\n",
      "uploaded pictures with\n",
      "pictures with the\n",
      "with the names\n",
      "the names of\n",
      "names of the\n",
      "of the people\n",
      "the people in\n",
      "people in them.googles\n",
      "in them.googles deepmind\n",
      "them.googles deepmind technologies\n",
      "deepmind technologies developed\n",
      "technologies developed a\n",
      "developed a system\n",
      "a system capable\n",
      "system capable of\n",
      "capable of learning\n",
      "of learning how\n",
      "learning how to\n",
      "how to play\n",
      "to play atari\n",
      "play atari video\n",
      "atari video games\n",
      "video games using\n",
      "games using only\n",
      "using only pixels\n",
      "only pixels as\n",
      "pixels as data\n",
      "as data input\n",
      "data input .\n",
      "input . in\n",
      ". in they\n",
      "in they demonstrated\n",
      "they demonstrated their\n",
      "demonstrated their alphago\n",
      "their alphago system\n",
      "alphago system which\n",
      "system which learned\n",
      "which learned the\n",
      "learned the game\n",
      "the game of\n",
      "game of go\n",
      "of go well\n",
      "go well enough\n",
      "well enough to\n",
      "enough to beat\n",
      "to beat a\n",
      "beat a professional\n",
      "a professional go\n",
      "professional go player\n",
      "go player .\n",
      "player . google\n",
      ". google translate\n",
      "google translate uses\n",
      "translate uses a\n",
      "uses a neural\n",
      "a neural network\n",
      "neural network to\n",
      "network to translate\n",
      "to translate between\n",
      "translate between more\n",
      "between more than\n",
      "more than languages.in\n",
      "than languages.in covariant.ai\n",
      "languages.in covariant.ai was\n",
      "covariant.ai was launched\n",
      "was launched which\n",
      "launched which focuses\n",
      "which focuses on\n",
      "focuses on integrating\n",
      "on integrating deep\n",
      "integrating deep learning\n",
      "deep learning into\n",
      "learning into factories.as\n",
      "into factories.as of\n",
      "factories.as of researchers\n",
      "of researchers at\n",
      "researchers at the\n",
      "at the university\n",
      "the university of\n",
      "university of texas\n",
      "of texas at\n",
      "texas at austin\n",
      "at austin ut\n",
      "austin ut developed\n",
      "ut developed a\n",
      "developed a machine\n",
      "a machine learning\n",
      "machine learning framework\n",
      "learning framework called\n",
      "framework called training\n",
      "called training an\n",
      "training an agent\n",
      "an agent manually\n",
      "agent manually via\n",
      "manually via evaluative\n",
      "via evaluative reinforcement\n",
      "evaluative reinforcement or\n",
      "reinforcement or tamer\n",
      "or tamer which\n",
      "tamer which proposed\n",
      "which proposed new\n",
      "proposed new methods\n",
      "new methods for\n",
      "methods for robots\n",
      "for robots or\n",
      "robots or computer\n",
      "or computer programs\n",
      "computer programs to\n",
      "programs to learn\n",
      "to learn how\n",
      "learn how to\n",
      "how to perform\n",
      "to perform tasks\n",
      "perform tasks by\n",
      "tasks by interacting\n",
      "by interacting with\n",
      "interacting with a\n",
      "with a human\n",
      "a human instructor\n",
      "human instructor .\n",
      "instructor . first\n",
      ". first developed\n",
      "first developed as\n",
      "developed as tamer\n",
      "as tamer a\n",
      "tamer a new\n",
      "a new algorithm\n",
      "new algorithm called\n",
      "algorithm called deep\n",
      "called deep tamer\n",
      "deep tamer was\n",
      "tamer was later\n",
      "was later introduced\n",
      "later introduced in\n",
      "introduced in during\n",
      "in during a\n",
      "during a collaboration\n",
      "a collaboration between\n",
      "collaboration between u.s.\n",
      "between u.s. army\n",
      "u.s. army research\n",
      "army research laboratory\n",
      "research laboratory arl\n",
      "laboratory arl and\n",
      "arl and ut\n",
      "and ut researchers\n",
      "ut researchers .\n",
      "researchers . deep\n",
      ". deep tamer\n",
      "deep tamer used\n",
      "tamer used deep\n",
      "used deep learning\n",
      "deep learning to\n",
      "learning to provide\n",
      "to provide a\n",
      "provide a robot\n",
      "a robot the\n",
      "robot the ability\n",
      "the ability to\n",
      "ability to learn\n",
      "to learn new\n",
      "learn new tasks\n",
      "new tasks through\n",
      "tasks through observation\n",
      "through observation .\n",
      "observation . using\n",
      ". using deep\n",
      "using deep tamer\n",
      "deep tamer a\n",
      "tamer a robot\n",
      "a robot learned\n",
      "robot learned a\n",
      "learned a task\n",
      "a task with\n",
      "task with a\n",
      "with a human\n",
      "a human trainer\n",
      "human trainer watching\n",
      "trainer watching video\n",
      "watching video streams\n",
      "video streams or\n",
      "streams or observing\n",
      "or observing a\n",
      "observing a human\n",
      "a human perform\n",
      "human perform a\n",
      "perform a task\n",
      "a task inperson\n",
      "task inperson .\n",
      "inperson . the\n",
      ". the robot\n",
      "the robot later\n",
      "robot later practiced\n",
      "later practiced the\n",
      "practiced the task\n",
      "the task with\n",
      "task with the\n",
      "with the help\n",
      "the help of\n",
      "help of some\n",
      "of some coaching\n",
      "some coaching from\n",
      "coaching from the\n",
      "from the trainer\n",
      "the trainer who\n",
      "trainer who provided\n",
      "who provided feedback\n",
      "provided feedback such\n",
      "feedback such as\n",
      "such as good\n",
      "as good job\n",
      "good job and\n",
      "job and bad\n",
      "and bad job.deep\n",
      "bad job.deep learning\n",
      "job.deep learning has\n",
      "learning has attracted\n",
      "has attracted both\n",
      "attracted both criticism\n",
      "both criticism and\n",
      "criticism and comment\n",
      "and comment in\n",
      "comment in some\n",
      "in some cases\n",
      "some cases from\n",
      "cases from outside\n",
      "from outside the\n",
      "outside the field\n",
      "the field of\n",
      "field of computer\n",
      "of computer science.a\n",
      "computer science.a main\n",
      "science.a main criticism\n",
      "main criticism concerns\n",
      "criticism concerns the\n",
      "concerns the lack\n",
      "the lack of\n",
      "lack of theory\n",
      "of theory surrounding\n",
      "theory surrounding some\n",
      "surrounding some methods\n",
      "some methods .\n",
      "methods . learning\n",
      ". learning in\n",
      "learning in the\n",
      "in the most\n",
      "the most common\n",
      "most common deep\n",
      "common deep architectures\n",
      "deep architectures is\n",
      "architectures is implemented\n",
      "is implemented using\n",
      "implemented using wellunderstood\n",
      "using wellunderstood gradient\n",
      "wellunderstood gradient descent\n",
      "gradient descent .\n",
      "descent . however\n",
      ". however the\n",
      "however the theory\n",
      "the theory surrounding\n",
      "theory surrounding other\n",
      "surrounding other algorithms\n",
      "other algorithms such\n",
      "algorithms such as\n",
      "such as contrastive\n",
      "as contrastive divergence\n",
      "contrastive divergence is\n",
      "divergence is less\n",
      "is less clear.citation\n",
      "less clear.citation needed\n",
      "clear.citation needed e.g\n",
      "needed e.g .\n",
      "e.g . does\n",
      ". does it\n",
      "does it converge\n",
      "it converge if\n",
      "converge if so\n",
      "if so how\n",
      "so how fast\n",
      "how fast what\n",
      "fast what is\n",
      "what is it\n",
      "is it approximating\n",
      "it approximating deep\n",
      "approximating deep learning\n",
      "deep learning methods\n",
      "learning methods are\n",
      "methods are often\n",
      "are often looked\n",
      "often looked at\n",
      "looked at as\n",
      "at as a\n",
      "as a black\n",
      "a black box\n",
      "black box with\n",
      "box with most\n",
      "with most confirmations\n",
      "most confirmations done\n",
      "confirmations done empirically\n",
      "done empirically rather\n",
      "empirically rather than\n",
      "rather than theoretically.others\n",
      "than theoretically.others point\n",
      "theoretically.others point out\n",
      "point out that\n",
      "out that deep\n",
      "that deep learning\n",
      "deep learning should\n",
      "learning should be\n",
      "should be looked\n",
      "be looked at\n",
      "looked at as\n",
      "at as a\n",
      "as a step\n",
      "a step towards\n",
      "step towards realizing\n",
      "towards realizing strong\n",
      "realizing strong ai\n",
      "strong ai not\n",
      "ai not as\n",
      "not as an\n",
      "as an allencompassing\n",
      "an allencompassing solution\n",
      "allencompassing solution .\n",
      "solution . despite\n",
      ". despite the\n",
      "despite the power\n",
      "the power of\n",
      "power of deep\n",
      "of deep learning\n",
      "deep learning methods\n",
      "learning methods they\n",
      "methods they still\n",
      "they still lack\n",
      "still lack much\n",
      "lack much of\n",
      "much of the\n",
      "of the functionality\n",
      "the functionality needed\n",
      "functionality needed for\n",
      "needed for realizing\n",
      "for realizing this\n",
      "realizing this goal\n",
      "this goal entirely\n",
      "goal entirely .\n",
      "entirely . research\n",
      ". research psychologist\n",
      "research psychologist gary\n",
      "psychologist gary marcus\n",
      "gary marcus notedrealistically\n",
      "marcus notedrealistically deep\n",
      "notedrealistically deep learning\n",
      "deep learning is\n",
      "learning is only\n",
      "is only part\n",
      "only part of\n",
      "part of the\n",
      "of the larger\n",
      "the larger challenge\n",
      "larger challenge of\n",
      "challenge of building\n",
      "of building intelligent\n",
      "building intelligent machines\n",
      "intelligent machines .\n",
      "machines . such\n",
      ". such techniques\n",
      "such techniques lack\n",
      "techniques lack ways\n",
      "lack ways of\n",
      "ways of representing\n",
      "of representing causal\n",
      "representing causal relationships\n",
      "causal relationships ...\n",
      "relationships ... have\n",
      "... have no\n",
      "have no obvious\n",
      "no obvious ways\n",
      "obvious ways of\n",
      "ways of performing\n",
      "of performing logical\n",
      "performing logical inferences\n",
      "logical inferences and\n",
      "inferences and they\n",
      "and they are\n",
      "they are also\n",
      "are also still\n",
      "also still a\n",
      "still a long\n",
      "a long way\n",
      "long way from\n",
      "way from integrating\n",
      "from integrating abstract\n",
      "integrating abstract knowledge\n",
      "abstract knowledge such\n",
      "knowledge such as\n",
      "such as information\n",
      "as information about\n",
      "information about what\n",
      "about what objects\n",
      "what objects are\n",
      "objects are what\n",
      "are what they\n",
      "what they are\n",
      "they are for\n",
      "are for and\n",
      "for and how\n",
      "and how they\n",
      "how they are\n",
      "they are typically\n",
      "are typically used\n",
      "typically used .\n",
      "used . the\n",
      ". the most\n",
      "the most powerful\n",
      "most powerful a.i\n",
      "powerful a.i .\n",
      "a.i . systems\n",
      ". systems like\n",
      "systems like watson\n",
      "like watson ...\n",
      "watson ... use\n",
      "... use techniques\n",
      "use techniques like\n",
      "techniques like deep\n",
      "like deep learning\n",
      "deep learning as\n",
      "learning as just\n",
      "as just one\n",
      "just one element\n",
      "one element in\n",
      "element in a\n",
      "in a very\n",
      "a very complicated\n",
      "very complicated ensemble\n",
      "complicated ensemble of\n",
      "ensemble of techniques\n",
      "of techniques ranging\n",
      "techniques ranging from\n",
      "ranging from the\n",
      "from the statistical\n",
      "the statistical technique\n",
      "statistical technique of\n",
      "technique of bayesian\n",
      "of bayesian inference\n",
      "bayesian inference to\n",
      "inference to deductive\n",
      "to deductive reasoning.in\n",
      "deductive reasoning.in further\n",
      "reasoning.in further reference\n",
      "further reference to\n",
      "reference to the\n",
      "to the idea\n",
      "the idea that\n",
      "idea that artistic\n",
      "that artistic sensitivity\n",
      "artistic sensitivity might\n",
      "sensitivity might be\n",
      "might be inherent\n",
      "be inherent in\n",
      "inherent in relatively\n",
      "in relatively low\n",
      "relatively low levels\n",
      "low levels of\n",
      "levels of the\n",
      "of the cognitive\n",
      "the cognitive hierarchy\n",
      "cognitive hierarchy a\n",
      "hierarchy a published\n",
      "a published series\n",
      "published series of\n",
      "series of graphic\n",
      "of graphic representations\n",
      "graphic representations of\n",
      "representations of the\n",
      "of the internal\n",
      "the internal states\n",
      "internal states of\n",
      "states of deep\n",
      "of deep layers\n",
      "deep layers neural\n",
      "layers neural networks\n",
      "neural networks attempting\n",
      "networks attempting to\n",
      "attempting to discern\n",
      "to discern within\n",
      "discern within essentially\n",
      "within essentially random\n",
      "essentially random data\n",
      "random data the\n",
      "data the images\n",
      "the images on\n",
      "images on which\n",
      "on which they\n",
      "which they were\n",
      "they were trained\n",
      "were trained demonstrate\n",
      "trained demonstrate a\n",
      "demonstrate a visual\n",
      "a visual appeal\n",
      "visual appeal the\n",
      "appeal the original\n",
      "the original research\n",
      "original research notice\n",
      "research notice received\n",
      "notice received well\n",
      "received well over\n",
      "well over comments\n",
      "over comments and\n",
      "comments and was\n",
      "and was the\n",
      "was the subject\n",
      "the subject of\n",
      "subject of what\n",
      "of what was\n",
      "what was for\n",
      "was for a\n",
      "for a time\n",
      "a time the\n",
      "time the most\n",
      "the most frequently\n",
      "most frequently accessed\n",
      "frequently accessed article\n",
      "accessed article on\n",
      "article on the\n",
      "on the guardians\n",
      "the guardians website.some\n",
      "guardians website.some deep\n",
      "website.some deep learning\n",
      "deep learning architectures\n",
      "learning architectures display\n",
      "architectures display problematic\n",
      "display problematic behaviors\n",
      "problematic behaviors such\n",
      "behaviors such as\n",
      "such as confidently\n",
      "as confidently classifying\n",
      "confidently classifying unrecognizable\n",
      "classifying unrecognizable images\n",
      "unrecognizable images as\n",
      "images as belonging\n",
      "as belonging to\n",
      "belonging to a\n",
      "to a familiar\n",
      "a familiar category\n",
      "familiar category of\n",
      "category of ordinary\n",
      "of ordinary images\n",
      "ordinary images and\n",
      "images and misclassifying\n",
      "and misclassifying minuscule\n",
      "misclassifying minuscule perturbations\n",
      "minuscule perturbations of\n",
      "perturbations of correctly\n",
      "of correctly classified\n",
      "correctly classified images\n",
      "classified images .\n",
      "images . goertzel\n",
      ". goertzel hypothesized\n",
      "goertzel hypothesized that\n",
      "hypothesized that these\n",
      "that these behaviors\n",
      "these behaviors are\n",
      "behaviors are due\n",
      "are due to\n",
      "due to limitations\n",
      "to limitations in\n",
      "limitations in their\n",
      "in their internal\n",
      "their internal representations\n",
      "internal representations and\n",
      "representations and that\n",
      "and that these\n",
      "that these limitations\n",
      "these limitations would\n",
      "limitations would inhibit\n",
      "would inhibit integration\n",
      "inhibit integration into\n",
      "integration into heterogeneous\n",
      "into heterogeneous multicomponent\n",
      "heterogeneous multicomponent artificial\n",
      "multicomponent artificial general\n",
      "artificial general intelligence\n",
      "general intelligence agi\n",
      "intelligence agi architectures\n",
      "agi architectures .\n",
      "architectures . these\n",
      ". these issues\n",
      "these issues may\n",
      "issues may possibly\n",
      "may possibly be\n",
      "possibly be addressed\n",
      "be addressed by\n",
      "addressed by deep\n",
      "by deep learning\n",
      "deep learning architectures\n",
      "learning architectures that\n",
      "architectures that internally\n",
      "that internally form\n",
      "internally form states\n",
      "form states homologous\n",
      "states homologous to\n",
      "homologous to imagegrammar\n",
      "to imagegrammar decompositions\n",
      "imagegrammar decompositions of\n",
      "decompositions of observed\n",
      "of observed entities\n",
      "observed entities and\n",
      "entities and events\n",
      "and events .\n",
      "events . learning\n",
      ". learning a\n",
      "learning a grammar\n",
      "a grammar visual\n",
      "grammar visual or\n",
      "visual or linguistic\n",
      "or linguistic from\n",
      "linguistic from training\n",
      "from training data\n",
      "training data would\n",
      "data would be\n",
      "would be equivalent\n",
      "be equivalent to\n",
      "equivalent to restricting\n",
      "to restricting the\n",
      "restricting the system\n",
      "the system to\n",
      "system to commonsense\n",
      "to commonsense reasoning\n",
      "commonsense reasoning that\n",
      "reasoning that operates\n",
      "that operates on\n",
      "operates on concepts\n",
      "on concepts in\n",
      "concepts in terms\n",
      "in terms of\n",
      "terms of grammatical\n",
      "of grammatical production\n",
      "grammatical production rules\n",
      "production rules and\n",
      "rules and is\n",
      "and is a\n",
      "is a basic\n",
      "a basic goal\n",
      "basic goal of\n",
      "goal of both\n",
      "of both human\n",
      "both human language\n",
      "human language acquisition\n",
      "language acquisition and\n",
      "acquisition and artificial\n",
      "and artificial intelligence\n",
      "artificial intelligence ai.as\n",
      "intelligence ai.as deep\n",
      "ai.as deep learning\n",
      "deep learning moves\n",
      "learning moves from\n",
      "moves from the\n",
      "from the lab\n",
      "the lab into\n",
      "lab into the\n",
      "into the world\n",
      "the world research\n",
      "world research and\n",
      "research and experience\n",
      "and experience show\n",
      "experience show that\n",
      "show that artificial\n",
      "that artificial neural\n",
      "artificial neural networks\n",
      "neural networks are\n",
      "networks are vulnerable\n",
      "are vulnerable to\n",
      "vulnerable to hacks\n",
      "to hacks and\n",
      "hacks and deception\n",
      "and deception .\n",
      "deception . by\n",
      ". by identifying\n",
      "by identifying patterns\n",
      "identifying patterns that\n",
      "patterns that these\n",
      "that these systems\n",
      "these systems use\n",
      "systems use to\n",
      "use to function\n",
      "to function attackers\n",
      "function attackers can\n",
      "attackers can modify\n",
      "can modify inputs\n",
      "modify inputs to\n",
      "inputs to anns\n",
      "to anns in\n",
      "anns in such\n",
      "in such a\n",
      "such a way\n",
      "a way that\n",
      "way that the\n",
      "that the ann\n",
      "the ann finds\n",
      "ann finds a\n",
      "finds a match\n",
      "a match that\n",
      "match that human\n",
      "that human observers\n",
      "human observers would\n",
      "observers would not\n",
      "would not recognize\n",
      "not recognize .\n",
      "recognize . for\n",
      ". for example\n",
      "for example an\n",
      "example an attacker\n",
      "an attacker can\n",
      "attacker can make\n",
      "can make subtle\n",
      "make subtle changes\n",
      "subtle changes to\n",
      "changes to an\n",
      "to an image\n",
      "an image such\n",
      "image such that\n",
      "such that the\n",
      "that the ann\n",
      "the ann finds\n",
      "ann finds a\n",
      "finds a match\n",
      "a match even\n",
      "match even though\n",
      "even though the\n",
      "though the image\n",
      "the image looks\n",
      "image looks to\n",
      "looks to a\n",
      "to a human\n",
      "a human nothing\n",
      "human nothing like\n",
      "nothing like the\n",
      "like the search\n",
      "the search target\n",
      "search target .\n",
      "target . such\n",
      ". such manipulation\n",
      "such manipulation is\n",
      "manipulation is termed\n",
      "is termed an\n",
      "termed an adversarial\n",
      "an adversarial attack.in\n",
      "adversarial attack.in researchers\n",
      "attack.in researchers used\n",
      "researchers used one\n",
      "used one ann\n",
      "one ann to\n",
      "ann to doctor\n",
      "to doctor images\n",
      "doctor images in\n",
      "images in trial\n",
      "in trial and\n",
      "trial and error\n",
      "and error fashion\n",
      "error fashion identify\n",
      "fashion identify anothers\n",
      "identify anothers focal\n",
      "anothers focal points\n",
      "focal points and\n",
      "points and thereby\n",
      "and thereby generate\n",
      "thereby generate images\n",
      "generate images that\n",
      "images that deceived\n",
      "that deceived it\n",
      "deceived it .\n",
      "it . the\n",
      ". the modified\n",
      "the modified images\n",
      "modified images looked\n",
      "images looked no\n",
      "looked no different\n",
      "no different to\n",
      "different to human\n",
      "to human eyes\n",
      "human eyes .\n",
      "eyes . another\n",
      ". another group\n",
      "another group showed\n",
      "group showed that\n",
      "showed that printouts\n",
      "that printouts of\n",
      "printouts of doctored\n",
      "of doctored images\n",
      "doctored images then\n",
      "images then photographed\n",
      "then photographed successfully\n",
      "photographed successfully tricked\n",
      "successfully tricked an\n",
      "tricked an image\n",
      "an image classification\n",
      "image classification system\n",
      "classification system .\n",
      "system . one\n",
      ". one defense\n",
      "one defense is\n",
      "defense is reverse\n",
      "is reverse image\n",
      "reverse image search\n",
      "image search in\n",
      "search in which\n",
      "in which a\n",
      "which a possible\n",
      "a possible fake\n",
      "possible fake image\n",
      "fake image is\n",
      "image is submitted\n",
      "is submitted to\n",
      "submitted to a\n",
      "to a site\n",
      "a site such\n",
      "site such as\n",
      "such as tineye\n",
      "as tineye that\n",
      "tineye that can\n",
      "that can then\n",
      "can then find\n",
      "then find other\n",
      "find other instances\n",
      "other instances of\n",
      "instances of it\n",
      "of it .\n",
      "it . a\n",
      ". a refinement\n",
      "a refinement is\n",
      "refinement is to\n",
      "is to search\n",
      "to search using\n",
      "search using only\n",
      "using only parts\n",
      "only parts of\n",
      "parts of the\n",
      "of the image\n",
      "the image to\n",
      "image to identify\n",
      "to identify images\n",
      "identify images from\n",
      "images from which\n",
      "from which that\n",
      "which that piece\n",
      "that piece may\n",
      "piece may have\n",
      "may have been\n",
      "have been taken.another\n",
      "been taken.another group\n",
      "taken.another group showed\n",
      "group showed that\n",
      "showed that certain\n",
      "that certain psychedelic\n",
      "certain psychedelic spectacles\n",
      "psychedelic spectacles could\n",
      "spectacles could fool\n",
      "could fool a\n",
      "fool a facial\n",
      "a facial recognition\n",
      "facial recognition system\n",
      "recognition system into\n",
      "system into thinking\n",
      "into thinking ordinary\n",
      "thinking ordinary people\n",
      "ordinary people were\n",
      "people were celebrities\n",
      "were celebrities potentially\n",
      "celebrities potentially allowing\n",
      "potentially allowing one\n",
      "allowing one person\n",
      "one person to\n",
      "person to impersonate\n",
      "to impersonate another\n",
      "impersonate another .\n",
      "another . in\n",
      ". in researchers\n",
      "in researchers added\n",
      "researchers added stickers\n",
      "added stickers to\n",
      "stickers to stop\n",
      "to stop signs\n",
      "stop signs and\n",
      "signs and caused\n",
      "and caused an\n",
      "caused an ann\n",
      "an ann to\n",
      "ann to misclassify\n",
      "to misclassify them.anns\n",
      "misclassify them.anns can\n",
      "them.anns can however\n",
      "can however be\n",
      "however be further\n",
      "be further trained\n",
      "further trained to\n",
      "trained to detect\n",
      "to detect attempts\n",
      "detect attempts at\n",
      "attempts at deception\n",
      "at deception potentially\n",
      "deception potentially leading\n",
      "potentially leading attackers\n",
      "leading attackers and\n",
      "attackers and defenders\n",
      "and defenders into\n",
      "defenders into an\n",
      "into an arms\n",
      "an arms race\n",
      "arms race similar\n",
      "race similar to\n",
      "similar to the\n",
      "to the kind\n",
      "the kind that\n",
      "kind that already\n",
      "that already defines\n",
      "already defines the\n",
      "defines the malware\n",
      "the malware defense\n",
      "malware defense industry\n",
      "defense industry .\n",
      "industry . anns\n",
      ". anns have\n",
      "anns have been\n",
      "have been trained\n",
      "been trained to\n",
      "trained to defeat\n",
      "to defeat annbased\n",
      "defeat annbased antimalware\n",
      "annbased antimalware software\n",
      "antimalware software by\n",
      "software by repeatedly\n",
      "by repeatedly attacking\n",
      "repeatedly attacking a\n",
      "attacking a defense\n",
      "a defense with\n",
      "defense with malware\n",
      "with malware that\n",
      "malware that was\n",
      "that was continually\n",
      "was continually altered\n",
      "continually altered by\n",
      "altered by a\n",
      "by a genetic\n",
      "a genetic algorithm\n",
      "genetic algorithm until\n",
      "algorithm until it\n",
      "until it tricked\n",
      "it tricked the\n",
      "tricked the antimalware\n",
      "the antimalware while\n",
      "antimalware while retaining\n",
      "while retaining its\n",
      "retaining its ability\n",
      "its ability to\n",
      "ability to damage\n",
      "to damage the\n",
      "damage the target.in\n",
      "the target.in another\n",
      "target.in another group\n",
      "another group demonstrated\n",
      "group demonstrated that\n",
      "demonstrated that certain\n",
      "that certain sounds\n",
      "certain sounds could\n",
      "sounds could make\n",
      "could make the\n",
      "make the google\n",
      "the google now\n",
      "google now voice\n",
      "now voice command\n",
      "voice command system\n",
      "command system open\n",
      "system open a\n",
      "open a particular\n",
      "a particular web\n",
      "particular web address\n",
      "web address and\n",
      "address and hypothesized\n",
      "and hypothesized that\n",
      "hypothesized that this\n",
      "that this could\n",
      "this could serve\n",
      "could serve as\n",
      "serve as a\n",
      "as a stepping\n",
      "a stepping stone\n",
      "stepping stone for\n",
      "stone for further\n",
      "for further attacks\n",
      "further attacks e.g\n",
      "attacks e.g .\n",
      "e.g . opening\n",
      ". opening a\n",
      "opening a web\n",
      "a web page\n",
      "web page hosting\n",
      "page hosting driveby\n",
      "hosting driveby malware.in\n",
      "driveby malware.in data\n",
      "malware.in data poisoning\n",
      "data poisoning false\n",
      "poisoning false data\n",
      "false data is\n",
      "data is continually\n",
      "is continually smuggled\n",
      "continually smuggled into\n",
      "smuggled into a\n",
      "into a machine\n",
      "a machine learning\n",
      "machine learning systems\n",
      "learning systems training\n",
      "systems training set\n",
      "training set to\n",
      "set to prevent\n",
      "to prevent it\n",
      "prevent it from\n",
      "it from achieving\n",
      "from achieving mastery.most\n",
      "achieving mastery.most deep\n",
      "mastery.most deep learning\n",
      "deep learning systems\n",
      "learning systems rely\n",
      "systems rely on\n",
      "rely on training\n",
      "on training and\n",
      "training and verification\n",
      "and verification data\n",
      "verification data that\n",
      "data that is\n",
      "that is generated\n",
      "is generated andor\n",
      "generated andor annotated\n",
      "andor annotated by\n",
      "annotated by humans\n",
      "by humans .\n",
      "humans . it\n",
      ". it has\n",
      "it has been\n",
      "has been argued\n",
      "been argued in\n",
      "argued in media\n",
      "in media philosophy\n",
      "media philosophy that\n",
      "philosophy that not\n",
      "that not only\n",
      "not only lowpaid\n",
      "only lowpaid clickwork\n",
      "lowpaid clickwork e.g\n",
      "clickwork e.g .\n",
      "e.g . on\n",
      ". on amazon\n",
      "on amazon mechanical\n",
      "amazon mechanical turk\n",
      "mechanical turk is\n",
      "turk is regularly\n",
      "is regularly deployed\n",
      "regularly deployed for\n",
      "deployed for this\n",
      "for this purpose\n",
      "this purpose but\n",
      "purpose but also\n",
      "but also implicit\n",
      "also implicit forms\n",
      "implicit forms of\n",
      "forms of human\n",
      "of human microwork\n",
      "human microwork that\n",
      "microwork that are\n",
      "that are often\n",
      "are often not\n",
      "often not recognized\n",
      "not recognized as\n",
      "recognized as such\n",
      "as such .\n",
      "such . the\n",
      ". the philosopher\n",
      "the philosopher rainer\n",
      "philosopher rainer mhlhoff\n",
      "rainer mhlhoff distinguishes\n",
      "mhlhoff distinguishes five\n",
      "distinguishes five types\n",
      "five types of\n",
      "types of machinic\n",
      "of machinic capture\n",
      "machinic capture of\n",
      "capture of human\n",
      "of human microwork\n",
      "human microwork to\n",
      "microwork to generate\n",
      "to generate training\n",
      "generate training data\n",
      "training data gamification\n",
      "data gamification the\n",
      "gamification the embedding\n",
      "the embedding of\n",
      "embedding of annotation\n",
      "of annotation or\n",
      "annotation or computation\n",
      "or computation tasks\n",
      "computation tasks in\n",
      "tasks in the\n",
      "in the flow\n",
      "the flow of\n",
      "flow of a\n",
      "of a game\n",
      "a game trapping\n",
      "game trapping and\n",
      "trapping and tracking\n",
      "and tracking e.g\n",
      "tracking e.g .\n",
      "e.g . captchas\n",
      ". captchas for\n",
      "captchas for image\n",
      "for image recognition\n",
      "image recognition or\n",
      "recognition or clicktracking\n",
      "or clicktracking on\n",
      "clicktracking on google\n",
      "on google search\n",
      "google search results\n",
      "search results pages\n",
      "results pages exploitation\n",
      "pages exploitation of\n",
      "exploitation of social\n",
      "of social motivations\n",
      "social motivations e.g\n",
      "motivations e.g .\n",
      "e.g . tagging\n",
      ". tagging faces\n",
      "tagging faces on\n",
      "faces on facebook\n",
      "on facebook to\n",
      "facebook to obtain\n",
      "to obtain labeled\n",
      "obtain labeled facial\n",
      "labeled facial images\n",
      "facial images information\n",
      "images information mining\n",
      "information mining e.g\n",
      "mining e.g .\n",
      "e.g . by\n",
      ". by leveraging\n",
      "by leveraging quantifiedself\n",
      "leveraging quantifiedself devices\n",
      "quantifiedself devices such\n",
      "devices such as\n",
      "such as activity\n",
      "as activity trackers\n",
      "activity trackers and\n",
      "trackers and clickwork.mhlhoff\n",
      "and clickwork.mhlhoff argues\n",
      "clickwork.mhlhoff argues that\n",
      "argues that in\n",
      "that in most\n",
      "in most commercial\n",
      "most commercial enduser\n",
      "commercial enduser applications\n",
      "enduser applications of\n",
      "applications of deep\n",
      "of deep learning\n",
      "deep learning such\n",
      "learning such as\n",
      "such as facebooks\n",
      "as facebooks face\n",
      "facebooks face recognition\n",
      "face recognition system\n",
      "recognition system the\n",
      "system the need\n",
      "the need for\n",
      "need for training\n",
      "for training data\n",
      "training data does\n",
      "data does not\n",
      "does not stop\n",
      "not stop once\n",
      "stop once an\n",
      "once an ann\n",
      "an ann is\n",
      "ann is trained\n",
      "is trained .\n",
      "trained . rather\n",
      ". rather there\n",
      "rather there is\n",
      "there is a\n",
      "is a continued\n",
      "a continued demand\n",
      "continued demand for\n",
      "demand for humangenerated\n",
      "for humangenerated verification\n",
      "humangenerated verification datato\n",
      "verification datato constantly\n",
      "datato constantly calibrate\n",
      "constantly calibrate and\n",
      "calibrate and update\n",
      "and update the\n",
      "update the ann\n",
      "the ann .\n",
      "ann . for\n",
      ". for this\n",
      "for this purpose\n",
      "this purpose facebook\n",
      "purpose facebook introduced\n",
      "facebook introduced the\n",
      "introduced the feature\n",
      "the feature that\n",
      "feature that once\n",
      "that once a\n",
      "once a user\n",
      "a user is\n",
      "user is automatically\n",
      "is automatically recognized\n",
      "automatically recognized in\n",
      "recognized in an\n",
      "in an image\n",
      "an image they\n",
      "image they receive\n",
      "they receive a\n",
      "receive a notification\n",
      "a notification .\n",
      "notification . they\n",
      ". they can\n",
      "they can choose\n",
      "can choose whether\n",
      "choose whether of\n",
      "whether of not\n",
      "of not they\n",
      "not they like\n",
      "they like to\n",
      "like to be\n",
      "to be publicly\n",
      "be publicly labeled\n",
      "publicly labeled on\n",
      "labeled on the\n",
      "on the image\n",
      "the image or\n",
      "image or tell\n",
      "or tell facebook\n",
      "tell facebook that\n",
      "facebook that it\n",
      "that it is\n",
      "it is not\n",
      "is not them\n",
      "not them in\n",
      "them in the\n",
      "in the picture\n",
      "the picture .\n",
      "picture . this\n",
      ". this user\n",
      "this user interface\n",
      "user interface is\n",
      "interface is a\n",
      "is a mechanism\n",
      "a mechanism to\n",
      "mechanism to generate\n",
      "to generate a\n",
      "generate a constant\n",
      "a constant stream\n",
      "constant stream of\n",
      "stream of verification\n",
      "of verification data\n",
      "verification data to\n",
      "data to further\n",
      "to further train\n",
      "further train the\n",
      "train the network\n",
      "the network in\n",
      "network in realtime\n",
      "in realtime .\n",
      "realtime . as\n",
      ". as mhlhoff\n",
      "as mhlhoff argues\n",
      "mhlhoff argues involvement\n",
      "argues involvement of\n",
      "involvement of human\n",
      "of human users\n",
      "human users to\n",
      "users to generate\n",
      "to generate training\n",
      "generate training and\n",
      "training and verification\n",
      "and verification data\n",
      "verification data is\n",
      "data is so\n",
      "is so typical\n",
      "so typical for\n",
      "typical for most\n",
      "for most commercial\n",
      "most commercial enduser\n",
      "commercial enduser applications\n",
      "enduser applications of\n",
      "applications of deep\n",
      "of deep learning\n",
      "deep learning that\n",
      "learning that such\n",
      "that such systems\n",
      "such systems may\n",
      "systems may be\n",
      "may be referred\n",
      "be referred to\n",
      "referred to as\n",
      "to as humanaided\n",
      "as humanaided artificial\n",
      "humanaided artificial intelligence\n"
     ]
    }
   ],
   "source": [
    "# ngram dictionary to keep the possible ngrams as keys and next occuring words as list items\n",
    "ngrams = {}\n",
    "\n",
    "# words in a single ngram\n",
    "words = 3\n",
    "\n",
    "# Word Tokenization\n",
    "words_tokens = nltk.word_tokenize(article_text)\n",
    "\n",
    "# Iterate over words tokens\n",
    "for i in range(len(words_tokens)-words):\n",
    "    # incrementaly find ngram word sequences  \n",
    "    seq = ' '.join(words_tokens[i:i+words])\n",
    "    print(seq)\n",
    "\n",
    "    # insert in dictionary as key\n",
    "    if  seq not in ngrams.keys():\n",
    "        # prepare empty list to insert next possible chars\n",
    "        ngrams[seq] = []\n",
    "\n",
    "    # push a next word\n",
    "    ngrams[seq].append(words_tokens[i+words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# assuming very first ngram as a search sequence\n",
    "search_sequence = ' '.join(words_tokens[0:words])\n",
    "\n",
    "# init the suggestion output\n",
    "output = search_sequence\n",
    "\n",
    "next_word = \"\"\n",
    "\n",
    "# lenth of max words in obtained suggestion\n",
    "while next_word != \".\":\n",
    "    # break, if search sequence is not present in prepared ngram dictionary\n",
    "    if search_sequence not in ngrams.keys():\n",
    "        break\n",
    "    \n",
    "    # if ngram key is available, then find the list of next possible characters\n",
    "    possible_words = ngrams[search_sequence]\n",
    "    \n",
    "    # Randomly select the next possbile character, most common will have more chances\n",
    "    next_word = possible_words[random.randrange(len(possible_words))]\n",
    "\n",
    "    # Update the suggestion output\n",
    "    output += ' ' + next_word\n",
    "    seq_words = nltk.word_tokenize(output)\n",
    "\n",
    "    # update the search sequence now, excluding first char as we move forward\n",
    "    search_sequence = ' '.join(seq_words[len(seq_words)-words:len(seq_words)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search sequence:  pages for logged\n",
      "\n",
      "Suggestion: pages for logged out editors learn moredeep learning is part of stateoftheart systems in various disciplines particularly computer vision and automatic speech recognition is the first and most convincing successful case of deep learning in industry began in the early s.\n"
     ]
    }
   ],
   "source": [
    "print(f'Search sequence: ',' '.join(words_tokens[0:words]))\n",
    "print(f'\\nSuggestion: {output}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}