notebooks/demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\t\t\t<script type=\"text/javascript\">\n",
       "\t\t\t<!--\n",
       "\t\t\t\t\n",
       "\t\t\t{\n",
       "\t\t\t\tvar element = document.getElementById('NetworKit_script');\n",
       "\t\t\t\tif (element) {\n",
       "\t\t\t\t\telement.parentNode.removeChild(element);\n",
       "\t\t\t\t}\n",
       "\t\t\t\telement = document.createElement('script');\n",
       "\t\t\t\telement.type = 'text/javascript';\n",
       "\t\t\t\telement.innerHTML = 'function NetworKit_pageEmbed(id) { var i, j; var elements; elements = document.getElementById(id).getElementsByClassName(\"Plot\"); for (i=0; i<elements.length; i++) { elements[i].id = id + \"_Plot_\" + i; var data = elements[i].getAttribute(\"data-image\").split(\"|\"); elements[i].removeAttribute(\"data-image\"); var content = \"<div class=\\\\\"Image\\\\\" id=\\\\\"\" + elements[i].id + \"_Image\\\\\" />\"; elements[i].innerHTML = content; elements[i].setAttribute(\"data-image-index\", 0); elements[i].setAttribute(\"data-image-length\", data.length); for (j=0; j<data.length; j++) { elements[i].setAttribute(\"data-image-\" + j, data[j]); } NetworKit_plotUpdate(elements[i]); elements[i].onclick = function (e) { NetworKit_overlayShow((e.target) ? e.target : e.srcElement); } } elements = document.getElementById(id).getElementsByClassName(\"HeatCell\"); for (i=0; i<elements.length; i++) { var data = parseFloat(elements[i].getAttribute(\"data-heat\")); var color = \"#00FF00\"; if (data <= 1 && data > 0) { color = \"hsla(0, 100%, 75%, \" + (data) + \")\"; } else if (data <= 0 && data >= -1) { color = \"hsla(240, 100%, 75%, \" + (-data) + \")\"; } elements[i].style.backgroundColor = color; } elements = document.getElementById(id).getElementsByClassName(\"Details\"); for (i=0; i<elements.length; i++) { elements[i].setAttribute(\"data-title\", \"-\"); NetworKit_toggleDetails(elements[i]); elements[i].onclick = function (e) { NetworKit_toggleDetails((e.target) ? e.target : e.srcElement); } } elements = document.getElementById(id).getElementsByClassName(\"MathValue\"); for (i=elements.length-1; i>=0; i--) { value = elements[i].innerHTML.trim(); if (value === \"nan\") { elements[i].parentNode.innerHTML = \"\" } } elements = document.getElementById(id).getElementsByClassName(\"SubCategory\"); for (i=elements.length-1; i>=0; i--) { value = elements[i].innerHTML.trim(); if (value === \"\") { elements[i].parentNode.removeChild(elements[i]) } } elements = document.getElementById(id).getElementsByClassName(\"Category\"); for (i=elements.length-1; i>=0; i--) { value = elements[i].innerHTML.trim(); if (value === \"\") { elements[i].parentNode.removeChild(elements[i]) } } var isFirefox = false; try { isFirefox = typeof InstallTrigger !== \"undefined\"; } catch (e) {} if (!isFirefox) { alert(\"Currently the function\\'s output is only fully supported by Firefox.\"); } } function NetworKit_plotUpdate(source) { var index = source.getAttribute(\"data-image-index\"); var data = source.getAttribute(\"data-image-\" + index); var image = document.getElementById(source.id + \"_Image\"); image.style.backgroundImage = \"url(\" + data + \")\"; } function NetworKit_showElement(id, show) { var element = document.getElementById(id); element.style.display = (show) ? \"block\" : \"none\"; } function NetworKit_overlayShow(source) { NetworKit_overlayUpdate(source); NetworKit_showElement(\"NetworKit_Overlay\", true); } function NetworKit_overlayUpdate(source) { document.getElementById(\"NetworKit_Overlay_Title\").innerHTML = source.title; var index = source.getAttribute(\"data-image-index\"); var data = source.getAttribute(\"data-image-\" + index); var image = document.getElementById(\"NetworKit_Overlay_Image\"); image.setAttribute(\"data-id\", source.id); image.style.backgroundImage = \"url(\" + data + \")\"; var link = document.getElementById(\"NetworKit_Overlay_Toolbar_Bottom_Save\"); link.href = data; link.download = source.title + \".svg\"; } function NetworKit_overlayImageShift(delta) { var image = document.getElementById(\"NetworKit_Overlay_Image\"); var source = document.getElementById(image.getAttribute(\"data-id\")); var index = parseInt(source.getAttribute(\"data-image-index\")); var length = parseInt(source.getAttribute(\"data-image-length\")); var index = (index+delta) % length; if (index < 0) { index = length + index; } source.setAttribute(\"data-image-index\", index); NetworKit_overlayUpdate(source); } function NetworKit_toggleDetails(source) { var childs = source.children; var show = false; if (source.getAttribute(\"data-title\") == \"-\") { source.setAttribute(\"data-title\", \"+\"); show = false; } else { source.setAttribute(\"data-title\", \"-\"); show = true; } for (i=0; i<childs.length; i++) { if (show) { childs[i].style.display = \"block\"; } else { childs[i].style.display = \"none\"; } } }';\n",
       "\t\t\t\telement.setAttribute('id', 'NetworKit_script');\n",
       "\t\t\t\tdocument.head.appendChild(element);\n",
       "\t\t\t}\n",
       "\t\t\n",
       "\t\t\t\t\n",
       "\t\t\t{\n",
       "\t\t\t\tvar element = document.getElementById('NetworKit_style');\n",
       "\t\t\t\tif (element) {\n",
       "\t\t\t\t\telement.parentNode.removeChild(element);\n",
       "\t\t\t\t}\n",
       "\t\t\t\telement = document.createElement('style');\n",
       "\t\t\t\telement.type = 'text/css';\n",
       "\t\t\t\telement.innerHTML = '.NetworKit_Page { font-family: Arial, Helvetica, sans-serif; font-size: 14px; } .NetworKit_Page .Value:before { font-family: Arial, Helvetica, sans-serif; font-size: 1.05em; content: attr(data-title) \":\"; margin-left: -2.5em; padding-right: 0.5em; } .NetworKit_Page .Details .Value:before { display: block; } .NetworKit_Page .Value { font-family: monospace; white-space: pre; padding-left: 2.5em; white-space: -moz-pre-wrap !important; white-space: -pre-wrap; white-space: -o-pre-wrap; white-space: pre-wrap; word-wrap: break-word; tab-size: 4; -moz-tab-size: 4; } .NetworKit_Page .Category { clear: both; padding-left: 1em; margin-bottom: 1.5em; } .NetworKit_Page .Category:before { content: attr(data-title); font-size: 1.75em; display: block; margin-left: -0.8em; margin-bottom: 0.5em; } .NetworKit_Page .SubCategory { margin-bottom: 1.5em; padding-left: 1em; } .NetworKit_Page .SubCategory:before { font-size: 1.6em; display: block; margin-left: -0.8em; margin-bottom: 0.5em; } .NetworKit_Page .SubCategory[data-title]:before { content: attr(data-title); } .NetworKit_Page .Block { display: block; } .NetworKit_Page .Block:after { content: \".\"; visibility: hidden; display: block; height: 0; clear: both; } .NetworKit_Page .Block .Thumbnail_Overview, .NetworKit_Page .Block .Thumbnail_ScatterPlot { width: 260px; float: left; } .NetworKit_Page .Block .Thumbnail_Overview img, .NetworKit_Page .Block .Thumbnail_ScatterPlot img { width: 260px; } .NetworKit_Page .Block .Thumbnail_Overview:before, .NetworKit_Page .Block .Thumbnail_ScatterPlot:before { display: block; text-align: center; font-weight: bold; } .NetworKit_Page .Block .Thumbnail_Overview:before { content: attr(data-title); } .NetworKit_Page .HeatCell { font-family: \"Courier New\", Courier, monospace; cursor: pointer; } .NetworKit_Page .HeatCell, .NetworKit_Page .HeatCellName { display: inline; padding: 0.1em; margin-right: 2px; background-color: #FFFFFF } .NetworKit_Page .HeatCellName { margin-left: 0.25em; } .NetworKit_Page .HeatCell:before { content: attr(data-heat); display: inline-block; color: #000000; width: 4em; text-align: center; } .NetworKit_Page .Measure { clear: both; } .NetworKit_Page .Measure .Details { cursor: pointer; } .NetworKit_Page .Measure .Details:before { content: \"[\" attr(data-title) \"]\"; display: block; } .NetworKit_Page .Measure .Details .Value { border-left: 1px dotted black; margin-left: 0.4em; padding-left: 3.5em; pointer-events: none; } .NetworKit_Page .Measure .Details .Spacer:before { content: \".\"; opacity: 0.0; pointer-events: none; } .NetworKit_Page .Measure .Plot { width: 440px; height: 440px; cursor: pointer; float: left; margin-left: -0.9em; margin-right: 20px; } .NetworKit_Page .Measure .Plot .Image { background-repeat: no-repeat; background-position: center center; background-size: contain; height: 100%; pointer-events: none; } .NetworKit_Page .Measure .Stat { width: 500px; float: left; } .NetworKit_Page .Measure .Stat .Group { padding-left: 1.25em; margin-bottom: 0.75em; } .NetworKit_Page .Measure .Stat .Group .Title { font-size: 1.1em; display: block; margin-bottom: 0.3em; margin-left: -0.75em; border-right-style: dotted; border-right-width: 1px; border-bottom-style: dotted; border-bottom-width: 1px; background-color: #D0D0D0; padding-left: 0.2em; } .NetworKit_Page .Measure .Stat .Group .List { -webkit-column-count: 3; -moz-column-count: 3; column-count: 3; } .NetworKit_Page .Measure .Stat .Group .List .Entry { position: relative; line-height: 1.75em; } .NetworKit_Page .Measure .Stat .Group .List .Entry[data-tooltip]:before { position: absolute; left: 0; top: -40px; background-color: #808080; color: #ffffff; height: 30px; line-height: 30px; border-radius: 5px; padding: 0 15px; content: attr(data-tooltip); white-space: nowrap; display: none; } .NetworKit_Page .Measure .Stat .Group .List .Entry[data-tooltip]:after { position: absolute; left: 15px; top: -10px; border-top: 7px solid #808080; border-left: 7px solid transparent; border-right: 7px solid transparent; content: \"\"; display: none; } .NetworKit_Page .Measure .Stat .Group .List .Entry[data-tooltip]:hover:after, .NetworKit_Page .Measure .Stat .Group .List .Entry[data-tooltip]:hover:before { display: block; } .NetworKit_Page .Measure .Stat .Group .List .Entry .MathValue { font-family: \"Courier New\", Courier, monospace; } .NetworKit_Page .Measure:after { content: \".\"; visibility: hidden; display: block; height: 0; clear: both; } .NetworKit_Page .PartitionPie { clear: both; } .NetworKit_Page .PartitionPie img { width: 600px; } #NetworKit_Overlay { left: 0px; top: 0px; display: none; position: absolute; width: 100%; height: 100%; background-color: rgba(0,0,0,0.6); z-index: 1000; } #NetworKit_Overlay_Title { position: absolute; color: white; transform: rotate(-90deg); width: 32em; height: 32em; padding-right: 0.5em; padding-top: 0.5em; text-align: right; font-size: 40px; } #NetworKit_Overlay .button { background: white; cursor: pointer; } #NetworKit_Overlay .button:before { size: 13px; display: inline-block; text-align: center; margin-top: 0.5em; margin-bottom: 0.5em; width: 1.5em; height: 1.5em; } #NetworKit_Overlay .icon-close:before { content: \"X\"; } #NetworKit_Overlay .icon-previous:before { content: \"P\"; } #NetworKit_Overlay .icon-next:before { content: \"N\"; } #NetworKit_Overlay .icon-save:before { content: \"S\"; } #NetworKit_Overlay_Toolbar_Top, #NetworKit_Overlay_Toolbar_Bottom { position: absolute; width: 40px; right: 13px; text-align: right; z-index: 1100; } #NetworKit_Overlay_Toolbar_Top { top: 0.5em; } #NetworKit_Overlay_Toolbar_Bottom { Bottom: 0.5em; } #NetworKit_Overlay_ImageContainer { position: absolute; top: 5%; left: 5%; height: 90%; width: 90%; background-repeat: no-repeat; background-position: center center; background-size: contain; } #NetworKit_Overlay_Image { height: 100%; width: 100%; background-repeat: no-repeat; background-position: center center; background-size: contain; }';\n",
       "\t\t\t\telement.setAttribute('id', 'NetworKit_style');\n",
       "\t\t\t\tdocument.head.appendChild(element);\n",
       "\t\t\t}\n",
       "\t\t\n",
       "\t\t\t\t\n",
       "\t\t\t{\n",
       "\t\t\t\tvar element = document.getElementById('NetworKit_Overlay');\n",
       "\t\t\t\tif (element) {\n",
       "\t\t\t\t\telement.parentNode.removeChild(element);\n",
       "\t\t\t\t}\n",
       "\t\t\t\telement = document.createElement('div');\n",
       "\t\t\t\telement.innerHTML = '<div id=\"NetworKit_Overlay_Toolbar_Top\"><div class=\"button icon-close\" id=\"NetworKit_Overlay_Close\" /></div><div id=\"NetworKit_Overlay_Title\" /> <div id=\"NetworKit_Overlay_ImageContainer\"> <div id=\"NetworKit_Overlay_Image\" /> </div> <div id=\"NetworKit_Overlay_Toolbar_Bottom\"> <div class=\"button icon-previous\" onclick=\"NetworKit_overlayImageShift(-1)\" /> <div class=\"button icon-next\" onclick=\"NetworKit_overlayImageShift(1)\" /> <a id=\"NetworKit_Overlay_Toolbar_Bottom_Save\"><div class=\"button icon-save\" /></a> </div>';\n",
       "\t\t\t\telement.setAttribute('id', 'NetworKit_Overlay');\n",
       "\t\t\t\tdocument.body.appendChild(element);\n",
       "\t\t\t\tdocument.getElementById('NetworKit_Overlay_Close').onclick = function (e) {\n",
       "\t\t\t\t\tdocument.getElementById('NetworKit_Overlay').style.display = 'none';\n",
       "\t\t\t\t}\n",
       "\t\t\t}\n",
       "\t\t\n",
       "\t\t\t-->\n",
       "\t\t\t</script>\n",
       "\t\t"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import skylink\n",
    "import numpy as np\n",
    "from astropy.table import Table\n",
    "import scipy.stats as stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's make a mock-up data to experiment with:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tnormal(mu=None, sigma=None, n=None, lower=-0.5, upper=0.5):\n",
    "    return np.clip(np.random.normal(np.repeat(mu, n), sigma), lower, upper)\n",
    "\n",
    "n = 1_000_000\n",
    "np.random.seed(2)\n",
    "ra = np.random.uniform(4, 6, n)\n",
    "dec = np.random.uniform(-1, 1, n)\n",
    "\n",
    "cat_a = Table({'ra': ra, 'dec': dec})\n",
    "cat_b = Table({'ra': np.append(ra+tnormal(0, 0.0004, n), ra+tnormal(0, 0.0001, n)),\n",
    "               'dec': np.append(dec+tnormal(0, 0.0002, n), dec+tnormal(0, 0.0002, n))})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above cell generates *n=1M* galaxies for `cat_a` and *2M* galaxies for `cat_b`. Now that you have the two catalogs as dictionaries, we can pass them to `skylink` and do an internal (i.e. `friends of friends`) match for the two catalogs combined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Running 8 parallel jobs\n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Mosaicking data    \n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Growing the first KD-Tree    \n",
      "\b\u001b[38;5;2m\u001b[1m✔\u001b[0m Used the same KD-Tree for the second set of coordinates since this is an internal match.\n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Stored the KD-Tree in the cache.\n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Finding all pairs of points whose distance is at most 0.75 arcsec    \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Creating matching lists', max=481553.0, style=ProgressSty…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Created matching lists from KD-Trees\n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Building the representative graph/network    \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Finding connected components of thegraphs and shared comp…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Assigned group ids for each chunk by using connected components of the graphs\n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Concatenating the results from different processes    \n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Stitch fragmented group ids from different mosaic sets    \n",
      "\u001b[38;5;2m\u001b[1m✔\u001b[0m Rearrange indices to be the same as original input    \n",
      "\b\u001b[1m\u001b[38;5;2m✔ Success! Took 0:00:09 to execute.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "results = skylink.match({'a': cat_a, 'b':cat_b},  # catalogs to match in a dictionary\n",
    "                        linking_lengths = 0.75,   # the linking length in arcsecond (can specify multiple values)\n",
    "                        graph_lib = 'networkit',  # the graph library to use (other options: 'igraph', 'networkx')\n",
    "                        nprocs = 8,               # number of processors for parallelization (multiprocessing by default)\n",
    "                        overlap = 1.0,            # in the units of `linking_length`\n",
    "                        use_linked_mask = True,   # create a mask for objects in overlapping regions to save time stitching fragmented group ids\n",
    "                        sort = False,             # sort/group the results at the end in terms of `group_id` and `row_index`\n",
    "                        verbose = True,           # display extended information wile processing (does not include progressbars and busy indicators)\n",
    "                        leave = False,            # leave the progressbars on the screen after they are done\n",
    "                        return_pandas = False     # return a panda's datafreame instead of an astropy Table\n",
    "                                                  # and some more options ...\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After the run is finished, we can see the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=3000000</i>\n",
       "<table id=\"table140231094597520\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>row_index</th><th>catalog_key</th><th>group_id</th></tr></thead>\n",
       "<thead><tr><th>int64</th><th>str1</th><th>int64</th></tr></thead>\n",
       "<tr><td>0</td><td>a</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>a</td><td>1</td></tr>\n",
       "<tr><td>2</td><td>a</td><td>2</td></tr>\n",
       "<tr><td>3</td><td>a</td><td>3</td></tr>\n",
       "<tr><td>4</td><td>a</td><td>4</td></tr>\n",
       "<tr><td>5</td><td>a</td><td>5</td></tr>\n",
       "<tr><td>6</td><td>a</td><td>6</td></tr>\n",
       "<tr><td>7</td><td>a</td><td>7</td></tr>\n",
       "<tr><td>8</td><td>a</td><td>8</td></tr>\n",
       "<tr><td>9</td><td>a</td><td>9</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>1999990</td><td>b</td><td>970368</td></tr>\n",
       "<tr><td>1999991</td><td>b</td><td>970369</td></tr>\n",
       "<tr><td>1999992</td><td>b</td><td>1980066</td></tr>\n",
       "<tr><td>1999993</td><td>b</td><td>1980067</td></tr>\n",
       "<tr><td>1999994</td><td>b</td><td>970371</td></tr>\n",
       "<tr><td>1999995</td><td>b</td><td>1980068</td></tr>\n",
       "<tr><td>1999996</td><td>b</td><td>1980069</td></tr>\n",
       "<tr><td>1999997</td><td>b</td><td>1980070</td></tr>\n",
       "<tr><td>1999998</td><td>b</td><td>970375</td></tr>\n",
       "<tr><td>1999999</td><td>b</td><td>1980071</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=3000000>\n",
       "row_index catalog_key group_id\n",
       "  int64       str1     int64  \n",
       "--------- ----------- --------\n",
       "        0           a        0\n",
       "        1           a        1\n",
       "        2           a        2\n",
       "        3           a        3\n",
       "        4           a        4\n",
       "        5           a        5\n",
       "        6           a        6\n",
       "        7           a        7\n",
       "        8           a        8\n",
       "        9           a        9\n",
       "      ...         ...      ...\n",
       "  1999990           b   970368\n",
       "  1999991           b   970369\n",
       "  1999992           b  1980066\n",
       "  1999993           b  1980067\n",
       "  1999994           b   970371\n",
       "  1999995           b  1980068\n",
       "  1999996           b  1980069\n",
       "  1999997           b  1980070\n",
       "  1999998           b   970375\n",
       "  1999999           b  1980071"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In case you forgot to set `sort=True`, you can always do so afterwards:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=3000000</i>\n",
       "<table id=\"table140230552037584\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>row_index</th><th>catalog_key</th><th>group_id</th></tr></thead>\n",
       "<thead><tr><th>int64</th><th>str1</th><th>int64</th></tr></thead>\n",
       "<tr><td>0</td><td>a</td><td>0</td></tr>\n",
       "<tr><td>1000000</td><td>b</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>a</td><td>1</td></tr>\n",
       "<tr><td>1000001</td><td>b</td><td>1</td></tr>\n",
       "<tr><td>2</td><td>a</td><td>2</td></tr>\n",
       "<tr><td>856688</td><td>a</td><td>2</td></tr>\n",
       "<tr><td>1000002</td><td>b</td><td>2</td></tr>\n",
       "<tr><td>1856688</td><td>b</td><td>2</td></tr>\n",
       "<tr><td>3</td><td>a</td><td>3</td></tr>\n",
       "<tr><td>3</td><td>b</td><td>3</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>1999982</td><td>b</td><td>1980062</td></tr>\n",
       "<tr><td>1999983</td><td>b</td><td>1980063</td></tr>\n",
       "<tr><td>1999984</td><td>b</td><td>1980064</td></tr>\n",
       "<tr><td>1999986</td><td>b</td><td>1980065</td></tr>\n",
       "<tr><td>1999992</td><td>b</td><td>1980066</td></tr>\n",
       "<tr><td>1999993</td><td>b</td><td>1980067</td></tr>\n",
       "<tr><td>1999995</td><td>b</td><td>1980068</td></tr>\n",
       "<tr><td>1999996</td><td>b</td><td>1980069</td></tr>\n",
       "<tr><td>1999997</td><td>b</td><td>1980070</td></tr>\n",
       "<tr><td>1999999</td><td>b</td><td>1980071</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=3000000>\n",
       "row_index catalog_key group_id\n",
       "  int64       str1     int64  \n",
       "--------- ----------- --------\n",
       "        0           a        0\n",
       "  1000000           b        0\n",
       "        1           a        1\n",
       "  1000001           b        1\n",
       "        2           a        2\n",
       "   856688           a        2\n",
       "  1000002           b        2\n",
       "  1856688           b        2\n",
       "        3           a        3\n",
       "        3           b        3\n",
       "      ...         ...      ...\n",
       "  1999982           b  1980062\n",
       "  1999983           b  1980063\n",
       "  1999984           b  1980064\n",
       "  1999986           b  1980065\n",
       "  1999992           b  1980066\n",
       "  1999993           b  1980067\n",
       "  1999995           b  1980068\n",
       "  1999996           b  1980069\n",
       "  1999997           b  1980070\n",
       "  1999999           b  1980071"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.group_by('group_id')  # or group_by(['group_id','row_index'] if needed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also, if forgot to set `return_pandas=True`, you can convert your table to a pandas dataframe by:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_index</th>\n",
       "      <th>catalog_key</th>\n",
       "      <th>group_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>a</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>a</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>a</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>a</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999995</th>\n",
       "      <td>1999995</td>\n",
       "      <td>b</td>\n",
       "      <td>1980068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999996</th>\n",
       "      <td>1999996</td>\n",
       "      <td>b</td>\n",
       "      <td>1980069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999997</th>\n",
       "      <td>1999997</td>\n",
       "      <td>b</td>\n",
       "      <td>1980070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999998</th>\n",
       "      <td>1999998</td>\n",
       "      <td>b</td>\n",
       "      <td>970375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999999</th>\n",
       "      <td>1999999</td>\n",
       "      <td>b</td>\n",
       "      <td>1980071</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3000000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         row_index catalog_key  group_id\n",
       "0                0           a         0\n",
       "1                1           a         1\n",
       "2                2           a         2\n",
       "3                3           a         3\n",
       "4                4           a         4\n",
       "...            ...         ...       ...\n",
       "2999995    1999995           b   1980068\n",
       "2999996    1999996           b   1980069\n",
       "2999997    1999997           b   1980070\n",
       "2999998    1999998           b    970375\n",
       "2999999    1999999           b   1980071\n",
       "\n",
       "[3000000 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = results.to_pandas()\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to check if the outputs of the two matchings are equal (including outputs from the benchmark `FoFCatalogMatching` package) you will need to sort them in a unique way first. This can automatically be done by using:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m\u001b[1m✔ The Tables are equal!\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "skylink.testing.assert_equal(results, results) # they, of course, give you equality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m\u001b[1m✔ The dataframes are equal!\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "skylink.testing.assert_equal(results_df, results_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Passing a path string (path to the pandas dataframe) is also allowed for one or both objects.\n",
    "\n",
    "*Note: you can pickle pandas dataframes simply by doing `df.to_pickle(\"df.pkl\")`*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Loading pandas pickle for the second object ...\n",
      "\u001b[38;5;1m\u001b[1m✘ The dataframes are not equal!\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "skylink.testing.assert_equal(results_df, 'data/df.pkl') # they shouldn't be equal since I used a different linking length for the pickled one!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}