talk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computación Distribuida con Dask\n",
    "\n",
    "Si ... pero ...\n",
    "\n",
    "![](dask_def.svg)\n",
    "\n",
    "En esta charla vamos a hablar sobre computación distribuida.\n",
    "\n",
    "-------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## La Arquitectura de Dask\n",
    "\n",
    "![](https://docs.dask.org/en/latest/_images/collections-schedulers.png)\n",
    "\n",
    "------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `dask.distributed`\n",
    "\n",
    "### El Modelo de `dask.distributed`\n",
    "\n",
    "![](https://blog.dask.org/images/distributed-layout.png)\n",
    "\n",
    "---------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cómo Armar un Cluster Local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask.distributed import Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/remote/tcadprj2/cganterh/python_envs/conda_dask/.venv/lib/python3.7/site-packages/distributed/dashboard/core.py:72: UserWarning: \n",
      "Port 8787 is already in use. \n",
      "Perhaps you already have a cluster running?\n",
      "Hosting the diagnostics dashboard on a random port instead.\n",
      "  warnings.warn(\"\\n\" + msg)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://127.0.0.1:42525</li>\n",
       "  <li><b>Dashboard: </b><a href='http://127.0.0.1:34752/status' target='_blank'>http://127.0.0.1:34752/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>8</li>\n",
       "  <li><b>Cores: </b>40</li>\n",
       "  <li><b>Memory: </b>270.18 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://127.0.0.1:42525' processes=8 cores=40>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Client()\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Y ahora podemos realizar cómputo:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def squared(n):\n",
    "    return n**2\n",
    "\n",
    "client.submit(squared, 3).result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--------------------------------------------\n",
    "\n",
    "### Bags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask import bag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('In the beginning God created the heaven and the earth. And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters. \\n',\n",
       " 'And God said, Let there be light: and there was light. \\n',\n",
       " 'And God saw the light, that it was good: and God divided the light from the darkness. \\n')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible_bag = bag.read_text('bible.txt', blocksize='100KB')\n",
    "bible_bag.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30383"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_lines = bible_bag.count()\n",
    "client.compute(total_lines).result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3513"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "god_lines = bible_bag.filter(lambda l: 'God' in l).count()\n",
    "client.compute(god_lines).result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------------------------\n",
    "\n",
    "### Clusters Distribuidos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://10.65.8.34:8786</li>\n",
       "  <li><b>Dashboard: </b><a href='http://10.65.8.34:8787/status' target='_blank'>http://10.65.8.34:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>160</li>\n",
       "  <li><b>Memory: </b>886.64 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://10.65.8.34:8786' processes=3 cores=136>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Client('tcp://10.65.8.34:8786')\n",
    "client.restart()\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.submit(squared, 4).result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--------------------------------------------\n",
    "\n",
    "### Ejemplo: Generando Datos en un Cluster Distribuido"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask.delayed import delayed\n",
    "from random import gauss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "numbers = 10**8\n",
    "partitions = 160\n",
    "numbers_per_partition = numbers // partitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "@delayed\n",
    "def get_partition_numbers():\n",
    "    return (\n",
    "        gauss(4, 2) for j in range(numbers_per_partition)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.8324762002123967, 4.593674992130431, 5.864875979169428)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gauss_bag = bag.from_delayed(\n",
    "    get_partition_numbers() for i in range(partitions)\n",
    ")\n",
    "\n",
    "gauss_bag.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.832476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.593675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5.864876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.540659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.502452</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a\n",
       "0  3.832476\n",
       "1  4.593675\n",
       "2  5.864876\n",
       "3  6.540659\n",
       "4  3.502452"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gauss_dataframe = gauss_bag.to_dataframe([('a', float)])\n",
    "gauss_dataframe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><strong>Dask DataFrame Structure:</strong></div>\n",
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>npartitions=160</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>float64</td>\n",
       "      <td>float64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "<div>Dask Name: assign, 1120 tasks</div>"
      ],
      "text/plain": [
       "Dask DataFrame Structure:\n",
       "                       a        b\n",
       "npartitions=160                  \n",
       "                 float64  float64\n",
       "                     ...      ...\n",
       "...                  ...      ...\n",
       "                     ...      ...\n",
       "                     ...      ...\n",
       "Dask Name: assign, 1120 tasks"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gauss_dataframe['b'] = 4*gauss_dataframe['a'] + 3\n",
    "gauss_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-000.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-001.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-002.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-003.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-004.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-005.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-006.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-007.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-008.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-009.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-010.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-011.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-012.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-013.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-014.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-015.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-016.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-017.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-018.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-019.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-020.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-021.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-022.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-023.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-024.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-025.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-026.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-027.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-028.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-029.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-030.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-031.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-032.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-033.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-034.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-035.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-036.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-037.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-038.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-039.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-040.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-041.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-042.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-043.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-044.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-045.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-046.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-047.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-048.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-049.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-050.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-051.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-052.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-053.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-054.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-055.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-056.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-057.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-058.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-059.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-060.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-061.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-062.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-063.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-064.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-065.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-066.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-067.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-068.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-069.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-070.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-071.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-072.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-073.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-074.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-075.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-076.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-077.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-078.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-079.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-080.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-081.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-082.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-083.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-084.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-085.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-086.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-087.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-088.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-089.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-090.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-091.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-092.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-093.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-094.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-095.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-096.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-097.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-098.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-099.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-100.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-101.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-102.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-103.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-104.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-105.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-106.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-107.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-108.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-109.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-110.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-111.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-112.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-113.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-114.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-115.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-116.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-117.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-118.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-119.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-120.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-121.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-122.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-123.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-124.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-125.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-126.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-127.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-128.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-129.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-130.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-131.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-132.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-133.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-134.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-135.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-136.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-137.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-138.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-139.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-140.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-141.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-142.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-143.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-144.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-145.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-146.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-147.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-148.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-149.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-150.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-151.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-152.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-153.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-154.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-155.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-156.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-157.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-158.csv',\n",
       " '/remote/tcadprj/cganterh/services/jupyterlab/dask_talk/gen-159.csv']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gauss_dataframe.to_csv('gen-*.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------------------------\n",
    "\n",
    "### Leyendo los Datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask import dataframe\n",
    "from dask_ml.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><strong>Dask DataFrame Structure:</strong></div>\n",
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>npartitions=160</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>int64</td>\n",
       "      <td>float64</td>\n",
       "      <td>float64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "<div>Dask Name: from-delayed, 160 tasks</div>"
      ],
      "text/plain": [
       "Dask DataFrame Structure:\n",
       "                Unnamed: 0        a        b\n",
       "npartitions=160                             \n",
       "                     int64  float64  float64\n",
       "                       ...      ...      ...\n",
       "...                    ...      ...      ...\n",
       "                       ...      ...      ...\n",
       "                       ...      ...      ...\n",
       "Dask Name: from-delayed, 160 tasks"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dataframe.read_csv('gen-*.csv')\n",
    "client.persist(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------------------------\n",
    "\n",
    "### Regresión Lineal Distribuida"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr>\n",
       "<td>\n",
       "<table>\n",
       "  <thead>\n",
       "    <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr><th> Bytes </th><td> 2.40 GB </td> <td> 15.00 MB </td></tr>\n",
       "    <tr><th> Shape </th><td> (100000000, 3) </td> <td> (625000, 3) </td></tr>\n",
       "    <tr><th> Count </th><td> 640 Tasks </td><td> 160 Chunks </td></tr>\n",
       "    <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</td>\n",
       "<td>\n",
       "<svg width=\"75\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
       "\n",
       "  <!-- Horizontal lines -->\n",
       "  <line x1=\"0\" y1=\"0\" x2=\"25\" y2=\"0\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"0\" y1=\"0\" x2=\"25\" y2=\"0\" />\n",
       "  <line x1=\"0\" y1=\"1\" x2=\"25\" y2=\"1\" />\n",
       "  <line x1=\"0\" y1=\"2\" x2=\"25\" y2=\"2\" />\n",
       "  <line x1=\"0\" y1=\"3\" x2=\"25\" y2=\"3\" />\n",
       "  <line x1=\"0\" y1=\"3\" x2=\"25\" y2=\"3\" />\n",
       "  <line x1=\"0\" y1=\"4\" x2=\"25\" y2=\"4\" />\n",
       "  <line x1=\"0\" y1=\"5\" x2=\"25\" y2=\"5\" />\n",
       "  <line x1=\"0\" y1=\"6\" x2=\"25\" y2=\"6\" />\n",
       "  <line x1=\"0\" y1=\"6\" x2=\"25\" y2=\"6\" />\n",
       "  <line x1=\"0\" y1=\"7\" x2=\"25\" y2=\"7\" />\n",
       "  <line x1=\"0\" y1=\"8\" x2=\"25\" y2=\"8\" />\n",
       "  <line x1=\"0\" y1=\"9\" x2=\"25\" y2=\"9\" />\n",
       "  <line x1=\"0\" y1=\"9\" x2=\"25\" y2=\"9\" />\n",
       "  <line x1=\"0\" y1=\"10\" x2=\"25\" y2=\"10\" />\n",
       "  <line x1=\"0\" y1=\"11\" x2=\"25\" y2=\"11\" />\n",
       "  <line x1=\"0\" y1=\"12\" x2=\"25\" y2=\"12\" />\n",
       "  <line x1=\"0\" y1=\"12\" x2=\"25\" y2=\"12\" />\n",
       "  <line x1=\"0\" y1=\"13\" x2=\"25\" y2=\"13\" />\n",
       "  <line x1=\"0\" y1=\"14\" x2=\"25\" y2=\"14\" />\n",
       "  <line x1=\"0\" y1=\"15\" x2=\"25\" y2=\"15\" />\n",
       "  <line x1=\"0\" y1=\"15\" x2=\"25\" y2=\"15\" />\n",
       "  <line x1=\"0\" y1=\"16\" x2=\"25\" y2=\"16\" />\n",
       "  <line x1=\"0\" y1=\"17\" x2=\"25\" y2=\"17\" />\n",
       "  <line x1=\"0\" y1=\"18\" x2=\"25\" y2=\"18\" />\n",
       "  <line x1=\"0\" y1=\"18\" x2=\"25\" y2=\"18\" />\n",
       "  <line x1=\"0\" y1=\"19\" x2=\"25\" y2=\"19\" />\n",
       "  <line x1=\"0\" y1=\"20\" x2=\"25\" y2=\"20\" />\n",
       "  <line x1=\"0\" y1=\"21\" x2=\"25\" y2=\"21\" />\n",
       "  <line x1=\"0\" y1=\"21\" x2=\"25\" y2=\"21\" />\n",
       "  <line x1=\"0\" y1=\"22\" x2=\"25\" y2=\"22\" />\n",
       "  <line x1=\"0\" y1=\"23\" x2=\"25\" y2=\"23\" />\n",
       "  <line x1=\"0\" y1=\"24\" x2=\"25\" y2=\"24\" />\n",
       "  <line x1=\"0\" y1=\"24\" x2=\"25\" y2=\"24\" />\n",
       "  <line x1=\"0\" y1=\"25\" x2=\"25\" y2=\"25\" />\n",
       "  <line x1=\"0\" y1=\"26\" x2=\"25\" y2=\"26\" />\n",
       "  <line x1=\"0\" y1=\"27\" x2=\"25\" y2=\"27\" />\n",
       "  <line x1=\"0\" y1=\"27\" x2=\"25\" y2=\"27\" />\n",
       "  <line x1=\"0\" y1=\"28\" x2=\"25\" y2=\"28\" />\n",
       "  <line x1=\"0\" y1=\"29\" x2=\"25\" y2=\"29\" />\n",
       "  <line x1=\"0\" y1=\"30\" x2=\"25\" y2=\"30\" />\n",
       "  <line x1=\"0\" y1=\"30\" x2=\"25\" y2=\"30\" />\n",
       "  <line x1=\"0\" y1=\"31\" x2=\"25\" y2=\"31\" />\n",
       "  <line x1=\"0\" y1=\"32\" x2=\"25\" y2=\"32\" />\n",
       "  <line x1=\"0\" y1=\"33\" x2=\"25\" y2=\"33\" />\n",
       "  <line x1=\"0\" y1=\"33\" x2=\"25\" y2=\"33\" />\n",
       "  <line x1=\"0\" y1=\"34\" x2=\"25\" y2=\"34\" />\n",
       "  <line x1=\"0\" y1=\"35\" x2=\"25\" y2=\"35\" />\n",
       "  <line x1=\"0\" y1=\"36\" x2=\"25\" y2=\"36\" />\n",
       "  <line x1=\"0\" y1=\"36\" x2=\"25\" y2=\"36\" />\n",
       "  <line x1=\"0\" y1=\"37\" x2=\"25\" y2=\"37\" />\n",
       "  <line x1=\"0\" y1=\"38\" x2=\"25\" y2=\"38\" />\n",
       "  <line x1=\"0\" y1=\"39\" x2=\"25\" y2=\"39\" />\n",
       "  <line x1=\"0\" y1=\"39\" x2=\"25\" y2=\"39\" />\n",
       "  <line x1=\"0\" y1=\"40\" x2=\"25\" y2=\"40\" />\n",
       "  <line x1=\"0\" y1=\"41\" x2=\"25\" y2=\"41\" />\n",
       "  <line x1=\"0\" y1=\"42\" x2=\"25\" y2=\"42\" />\n",
       "  <line x1=\"0\" y1=\"42\" x2=\"25\" y2=\"42\" />\n",
       "  <line x1=\"0\" y1=\"43\" x2=\"25\" y2=\"43\" />\n",
       "  <line x1=\"0\" y1=\"44\" x2=\"25\" y2=\"44\" />\n",
       "  <line x1=\"0\" y1=\"45\" x2=\"25\" y2=\"45\" />\n",
       "  <line x1=\"0\" y1=\"45\" x2=\"25\" y2=\"45\" />\n",
       "  <line x1=\"0\" y1=\"46\" x2=\"25\" y2=\"46\" />\n",
       "  <line x1=\"0\" y1=\"47\" x2=\"25\" y2=\"47\" />\n",
       "  <line x1=\"0\" y1=\"48\" x2=\"25\" y2=\"48\" />\n",
       "  <line x1=\"0\" y1=\"48\" x2=\"25\" y2=\"48\" />\n",
       "  <line x1=\"0\" y1=\"49\" x2=\"25\" y2=\"49\" />\n",
       "  <line x1=\"0\" y1=\"50\" x2=\"25\" y2=\"50\" />\n",
       "  <line x1=\"0\" y1=\"51\" x2=\"25\" y2=\"51\" />\n",
       "  <line x1=\"0\" y1=\"51\" x2=\"25\" y2=\"51\" />\n",
       "  <line x1=\"0\" y1=\"52\" x2=\"25\" y2=\"52\" />\n",
       "  <line x1=\"0\" y1=\"53\" x2=\"25\" y2=\"53\" />\n",
       "  <line x1=\"0\" y1=\"54\" x2=\"25\" y2=\"54\" />\n",
       "  <line x1=\"0\" y1=\"54\" x2=\"25\" y2=\"54\" />\n",
       "  <line x1=\"0\" y1=\"55\" x2=\"25\" y2=\"55\" />\n",
       "  <line x1=\"0\" y1=\"56\" x2=\"25\" y2=\"56\" />\n",
       "  <line x1=\"0\" y1=\"57\" x2=\"25\" y2=\"57\" />\n",
       "  <line x1=\"0\" y1=\"57\" x2=\"25\" y2=\"57\" />\n",
       "  <line x1=\"0\" y1=\"58\" x2=\"25\" y2=\"58\" />\n",
       "  <line x1=\"0\" y1=\"59\" x2=\"25\" y2=\"59\" />\n",
       "  <line x1=\"0\" y1=\"60\" x2=\"25\" y2=\"60\" />\n",
       "  <line x1=\"0\" y1=\"60\" x2=\"25\" y2=\"60\" />\n",
       "  <line x1=\"0\" y1=\"61\" x2=\"25\" y2=\"61\" />\n",
       "  <line x1=\"0\" y1=\"62\" x2=\"25\" y2=\"62\" />\n",
       "  <line x1=\"0\" y1=\"63\" x2=\"25\" y2=\"63\" />\n",
       "  <line x1=\"0\" y1=\"63\" x2=\"25\" y2=\"63\" />\n",
       "  <line x1=\"0\" y1=\"64\" x2=\"25\" y2=\"64\" />\n",
       "  <line x1=\"0\" y1=\"65\" x2=\"25\" y2=\"65\" />\n",
       "  <line x1=\"0\" y1=\"66\" x2=\"25\" y2=\"66\" />\n",
       "  <line x1=\"0\" y1=\"66\" x2=\"25\" y2=\"66\" />\n",
       "  <line x1=\"0\" y1=\"67\" x2=\"25\" y2=\"67\" />\n",
       "  <line x1=\"0\" y1=\"68\" x2=\"25\" y2=\"68\" />\n",
       "  <line x1=\"0\" y1=\"69\" x2=\"25\" y2=\"69\" />\n",
       "  <line x1=\"0\" y1=\"69\" x2=\"25\" y2=\"69\" />\n",
       "  <line x1=\"0\" y1=\"70\" x2=\"25\" y2=\"70\" />\n",
       "  <line x1=\"0\" y1=\"71\" x2=\"25\" y2=\"71\" />\n",
       "  <line x1=\"0\" y1=\"72\" x2=\"25\" y2=\"72\" />\n",
       "  <line x1=\"0\" y1=\"72\" x2=\"25\" y2=\"72\" />\n",
       "  <line x1=\"0\" y1=\"73\" x2=\"25\" y2=\"73\" />\n",
       "  <line x1=\"0\" y1=\"74\" x2=\"25\" y2=\"74\" />\n",
       "  <line x1=\"0\" y1=\"75\" x2=\"25\" y2=\"75\" />\n",
       "  <line x1=\"0\" y1=\"75\" x2=\"25\" y2=\"75\" />\n",
       "  <line x1=\"0\" y1=\"76\" x2=\"25\" y2=\"76\" />\n",
       "  <line x1=\"0\" y1=\"77\" x2=\"25\" y2=\"77\" />\n",
       "  <line x1=\"0\" y1=\"78\" x2=\"25\" y2=\"78\" />\n",
       "  <line x1=\"0\" y1=\"78\" x2=\"25\" y2=\"78\" />\n",
       "  <line x1=\"0\" y1=\"79\" x2=\"25\" y2=\"79\" />\n",
       "  <line x1=\"0\" y1=\"80\" x2=\"25\" y2=\"80\" />\n",
       "  <line x1=\"0\" y1=\"81\" x2=\"25\" y2=\"81\" />\n",
       "  <line x1=\"0\" y1=\"81\" x2=\"25\" y2=\"81\" />\n",
       "  <line x1=\"0\" y1=\"82\" x2=\"25\" y2=\"82\" />\n",
       "  <line x1=\"0\" y1=\"83\" x2=\"25\" y2=\"83\" />\n",
       "  <line x1=\"0\" y1=\"84\" x2=\"25\" y2=\"84\" />\n",
       "  <line x1=\"0\" y1=\"84\" x2=\"25\" y2=\"84\" />\n",
       "  <line x1=\"0\" y1=\"85\" x2=\"25\" y2=\"85\" />\n",
       "  <line x1=\"0\" y1=\"86\" x2=\"25\" y2=\"86\" />\n",
       "  <line x1=\"0\" y1=\"87\" x2=\"25\" y2=\"87\" />\n",
       "  <line x1=\"0\" y1=\"87\" x2=\"25\" y2=\"87\" />\n",
       "  <line x1=\"0\" y1=\"88\" x2=\"25\" y2=\"88\" />\n",
       "  <line x1=\"0\" y1=\"89\" x2=\"25\" y2=\"89\" />\n",
       "  <line x1=\"0\" y1=\"90\" x2=\"25\" y2=\"90\" />\n",
       "  <line x1=\"0\" y1=\"90\" x2=\"25\" y2=\"90\" />\n",
       "  <line x1=\"0\" y1=\"91\" x2=\"25\" y2=\"91\" />\n",
       "  <line x1=\"0\" y1=\"92\" x2=\"25\" y2=\"92\" />\n",
       "  <line x1=\"0\" y1=\"93\" x2=\"25\" y2=\"93\" />\n",
       "  <line x1=\"0\" y1=\"93\" x2=\"25\" y2=\"93\" />\n",
       "  <line x1=\"0\" y1=\"94\" x2=\"25\" y2=\"94\" />\n",
       "  <line x1=\"0\" y1=\"95\" x2=\"25\" y2=\"95\" />\n",
       "  <line x1=\"0\" y1=\"96\" x2=\"25\" y2=\"96\" />\n",
       "  <line x1=\"0\" y1=\"96\" x2=\"25\" y2=\"96\" />\n",
       "  <line x1=\"0\" y1=\"97\" x2=\"25\" y2=\"97\" />\n",
       "  <line x1=\"0\" y1=\"98\" x2=\"25\" y2=\"98\" />\n",
       "  <line x1=\"0\" y1=\"99\" x2=\"25\" y2=\"99\" />\n",
       "  <line x1=\"0\" y1=\"99\" x2=\"25\" y2=\"99\" />\n",
       "  <line x1=\"0\" y1=\"100\" x2=\"25\" y2=\"100\" />\n",
       "  <line x1=\"0\" y1=\"101\" x2=\"25\" y2=\"101\" />\n",
       "  <line x1=\"0\" y1=\"102\" x2=\"25\" y2=\"102\" />\n",
       "  <line x1=\"0\" y1=\"102\" x2=\"25\" y2=\"102\" />\n",
       "  <line x1=\"0\" y1=\"103\" x2=\"25\" y2=\"103\" />\n",
       "  <line x1=\"0\" y1=\"104\" x2=\"25\" y2=\"104\" />\n",
       "  <line x1=\"0\" y1=\"105\" x2=\"25\" y2=\"105\" />\n",
       "  <line x1=\"0\" y1=\"105\" x2=\"25\" y2=\"105\" />\n",
       "  <line x1=\"0\" y1=\"106\" x2=\"25\" y2=\"106\" />\n",
       "  <line x1=\"0\" y1=\"107\" x2=\"25\" y2=\"107\" />\n",
       "  <line x1=\"0\" y1=\"108\" x2=\"25\" y2=\"108\" />\n",
       "  <line x1=\"0\" y1=\"108\" x2=\"25\" y2=\"108\" />\n",
       "  <line x1=\"0\" y1=\"109\" x2=\"25\" y2=\"109\" />\n",
       "  <line x1=\"0\" y1=\"110\" x2=\"25\" y2=\"110\" />\n",
       "  <line x1=\"0\" y1=\"111\" x2=\"25\" y2=\"111\" />\n",
       "  <line x1=\"0\" y1=\"111\" x2=\"25\" y2=\"111\" />\n",
       "  <line x1=\"0\" y1=\"112\" x2=\"25\" y2=\"112\" />\n",
       "  <line x1=\"0\" y1=\"113\" x2=\"25\" y2=\"113\" />\n",
       "  <line x1=\"0\" y1=\"114\" x2=\"25\" y2=\"114\" />\n",
       "  <line x1=\"0\" y1=\"114\" x2=\"25\" y2=\"114\" />\n",
       "  <line x1=\"0\" y1=\"115\" x2=\"25\" y2=\"115\" />\n",
       "  <line x1=\"0\" y1=\"116\" x2=\"25\" y2=\"116\" />\n",
       "  <line x1=\"0\" y1=\"117\" x2=\"25\" y2=\"117\" />\n",
       "  <line x1=\"0\" y1=\"117\" x2=\"25\" y2=\"117\" />\n",
       "  <line x1=\"0\" y1=\"118\" x2=\"25\" y2=\"118\" />\n",
       "  <line x1=\"0\" y1=\"119\" x2=\"25\" y2=\"119\" />\n",
       "  <line x1=\"0\" y1=\"120\" x2=\"25\" y2=\"120\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Vertical lines -->\n",
       "  <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
       "  <line x1=\"25\" y1=\"0\" x2=\"25\" y2=\"120\" style=\"stroke-width:2\" />\n",
       "\n",
       "  <!-- Colored Rectangle -->\n",
       "  <polygon points=\"0.000000,0.000000 25.412617,0.000000 25.412617,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
       "\n",
       "  <!-- Text -->\n",
       "  <text x=\"12.706308\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >3</text>\n",
       "  <text x=\"45.412617\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,45.412617,60.000000)\">100000000</text>\n",
       "</svg>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "dask.array<values, shape=(100000000, 3), dtype=float64, chunksize=(625000, 3)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = df.to_dask_array(lengths=True)\n",
    "array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                 intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "                 n_jobs=1, penalty='l2', random_state=None, solver='admm',\n",
       "                 solver_kwargs=None, tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(array[:, 1:2], array[:, 2:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.99999902])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.9999992658668226"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.intercept_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Conda Dask Cluster",
   "language": "python",
   "name": "conda_dask_cluster"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}