From 43916eff0daf5813160a9a333648f010d71863cd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 3 Apr 2018 14:38:25 -0400 Subject: [PATCH 01/12] Renamed astrocat attributes and git add_sources() working. --- astrodbkit/astrocat.py | 71 +++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index 700918c..1a752ce 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -33,7 +33,7 @@ def __init__(self, name='Test'): The name of the database """ self.name = name - self.catalog = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) + self.sources = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) self.n_sources = 0 self.history = "{}: Database created".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.catalogs = {} @@ -49,7 +49,7 @@ def info(self): """ print(self.history) - def add_source(self, ra, dec, flag='', radius=10*q.arcsec): + def add_source(self, ra, dec, flag='', radius=10*q.arcsec, catalogs={}): """ Add a source to the catalog manually and find data in existing catalogs @@ -63,9 +63,12 @@ def add_source(self, ra, dec, flag='', radius=10*q.arcsec): A flag for the source radius: float The cross match radius for the list of catalogs + catalogs: dict + Additional catalogs to search, e.g. + catalogs={'TMASS':{'cat_loc':'II/246/out', 'id_col':'id', 'ra_col':'RAJ2000', 'dec_col':'DEJ2000'}} """ - # Get the id - id = int(len(self.catalog)+1) + # Set the id + self.n_sources += 1 # Check the coordinates ra = ra.to(q.deg) @@ -74,10 +77,16 @@ def add_source(self, ra, dec, flag='', radius=10*q.arcsec): # Search the catalogs for this source for cat_name,params in self.catalogs.items(): - self.Vizier_query(params['cat_loc'], cat_name, ra, dec, radius, ra_col=params['ra_col'], dec_col=params['dec_col'], append=True, group=False) + self.Vizier_query(params['cat_loc'], cat_name, ra, dec, radius, ra_col=params['ra_col'], dec_col=params['dec_col'], append=True, force_id=self.n_sources, group=False) + + # Search additional catalogs + for cat_name,params in catalogs.items(): + if cat_name not in self.catalogs: + self.Vizier_query(params['cat_loc'], cat_name, ra, dec, radius, ra_col=params['ra_col'], dec_col=params['dec_col'], force_id=self.n_sources, group=False) # Add the source to the catalog - self.catalog = self.catalog.append([id, ra.value, dec.value, flag, datasets], ignore_index=True) + new_cat = pd.DataFrame([[self.n_sources, ra.value, dec.value, flag, datasets]], columns=self.sources.columns) + self.sources = self.sources.append(new_cat, ignore_index=True) def delete_source(self, id): """ @@ -89,10 +98,10 @@ def delete_source(self, id): The id of the source in the catalog """ # Set the index - self.catalog.set_index('id') + self.sources.set_index('id') # Exclude the unwanted source - self.catalog = self.catalog[self.catalog.id!=id] + self.sources = self.sources[self.sources.id!=id] # Remove the records from the catalogs for cat_name in self.catalogs: @@ -100,7 +109,7 @@ def delete_source(self, id): print('{} records removed from {} catalog'.format(int(len(getattr(self, cat_name))-len(new_cat)), cat_name)) setattr(self, cat_name, new_cat) - def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ2000', cat_loc='', append=False, count=-1): + def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ2000', cat_loc='', append=False, force_id='', count=-1): """ Ingest a data file and regroup sources @@ -120,6 +129,8 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 The location of the original catalog data append: bool Append the catalog rather than replace + force_id: int + Assigns a specific id in the catalog count: int The number of table rows to add (This is mainly for testing purposes) @@ -167,7 +178,7 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 data.insert(0,'catID', ['{}_{}'.format(cat_name,n+1) for n in range(last,last+len(data))]) data.insert(0,'dec_corr', data['dec']) data.insert(0,'ra_corr', data['ra']) - data.insert(0,'source_id', np.nan) + data.insert(0,'source_id', force_id or np.nan) print('Ingesting {} rows from {} catalog...'.format(len(data),cat_name)) @@ -205,7 +216,7 @@ def inventory(self, source_id): else: print('Source:') - print(at.Table.from_pandas(self.catalog[self.catalog['id']==source_id]).pprint()) + print(at.Table.from_pandas(self.sources[self.sources['id']==source_id]).pprint()) for cat_name in self.catalogs: cat = getattr(self, cat_name) rows = cat[cat['source_id']==source_id] @@ -262,7 +273,7 @@ def SDSS_spectra_query(self, cat_name, ra, dec, radius, group=True, **kwargs): if self._catalog_check(cat_name): # Prep the current catalog as an astropy.QTable - tab = at.Table.from_pandas(self.catalog) + tab = at.Table.from_pandas(self.sources) # Cone search Vizier print("Searching SDSS for sources within {} of ({}, {}). Please be patient...".format(viz_cat, radius, ra, dec)) @@ -280,7 +291,7 @@ def SDSS_spectra_query(self, cat_name, ra, dec, radius, group=True, **kwargs): if len(self.catalogs)>1 and group: self.group_sources(self.xmatch_radius) - def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec_col='DEJ2000', columns=["**"], append=False, group=True, **kwargs): + def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec_col='DEJ2000', columns=["**"], append=False, force_id='', group=True, **kwargs): """ Use astroquery to search a catalog for sources within a search cone @@ -304,6 +315,8 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec The list of columns to pass to astroquery append: bool Append the catalog rather than replace + force_id: int + Assigns a specific id in the catalog """ # Verify the cat_name if self._catalog_check(cat_name, append=append): @@ -321,7 +334,7 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec return # Ingest the data - self.ingest_data(data, cat_name, 'id', ra_col=ra_col, dec_col=dec_col, cat_loc=viz_cat, append=append) + self.ingest_data(data, cat_name, 'id', ra_col=ra_col, dec_col=dec_col, cat_loc=viz_cat, append=append, force_id=force_id) # Regroup if len(self.catalogs)>1 and group: @@ -329,7 +342,7 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000', radius='', group=True): """ - Use astroquery to pull in and cross match a catalog with sources in self.catalog + Use astroquery to pull in and cross match a catalog with sources in self.sources Parameters ---------- @@ -341,7 +354,7 @@ def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000' The matching radius """ # Make sure sources have been grouped - if self.catalog.empty: + if self.sources.empty: print('Please run group_sources() before cross matching.') return @@ -351,7 +364,7 @@ def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000' viz_cat = "vizier:{}".format(viz_cat) # Prep the current catalog as an astropy.QTable - tab = at.Table.from_pandas(self.catalog) + tab = at.Table.from_pandas(self.sources) # Crossmatch with Vizier print("Cross matching {} sources with {} catalog. Please be patient...".format(len(tab), viz_cat)) @@ -413,12 +426,12 @@ def group_sources(self, radius='', plot=False): unique_coords = np.asarray([np.mean(coords[source_ids==id], axis=0) for id in list(set(source_ids))]) # Generate a source catalog - self.catalog = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) - self.catalog['id'] = unique_source_ids - self.catalog[['ra','dec']] = unique_coords - self.catalog['flag'] = [None]*len(unique_source_ids) - # self.catalog['flag'] = ['d{}'.format(i) if i>1 else '' for i in Counter(source_ids).values()] - self.catalog['datasets'] = Counter(source_ids).values() + self.sources = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) + self.sources['id'] = unique_source_ids + self.sources[['ra','dec']] = unique_coords + self.sources['flag'] = [None]*len(unique_source_ids) + # self.sources['flag'] = ['d{}'.format(i) if i>1 else '' for i in Counter(source_ids).values()] + self.sources['datasets'] = Counter(source_ids).values() # Update history self.history += "\n{}: Catalog grouped with radius {} arcsec.".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.xmatch_radius) @@ -495,7 +508,7 @@ def load(self, path): DB = joblib.load(path) # Load the attributes - self.catalog = DB.catalog + self.sources = DB.catalog self.n_sources = DB.n_sources self.name = DB.name self.history = DB.history @@ -535,11 +548,11 @@ def correct_offsets(self, cat_name, truth='ACS'): else: # First, remove any previous catalog correction - self.catalog.loc[self.catalog['cat_name']==cat_name, 'ra_corr'] = self.catalog.loc[self.catalog['cat_name']==cat_name, '_RAJ2000'] - self.catalog.loc[self.catalog['cat_name']==cat_name, 'dec_corr'] = self.catalog.loc[self.catalog['cat_name']==cat_name, '_DEJ2000'] + self.sources.loc[self.sources['cat_name']==cat_name, 'ra_corr'] = self.sources.loc[self.sources['cat_name']==cat_name, '_RAJ2000'] + self.sources.loc[self.sources['cat_name']==cat_name, 'dec_corr'] = self.sources.loc[self.sources['cat_name']==cat_name, '_DEJ2000'] # Copy the catalog - onc_gr = self.catalog.copy() + onc_gr = self.sources.copy() # restrict to one-to-one matches, sort by oncID so that matches are paired o2o_new = onc_gr.loc[(onc_gr['oncflag'].str.contains('o')) & (onc_gr['cat_name'] == cat_name) ,:].sort_values('oncID') @@ -582,8 +595,8 @@ def correct_offsets(self, cat_name, truth='ACS'): # Update the coordinates of the appropriate sources print('Shifting {} sources by {}" in RA and {}" in Dec...'.format(cat_name,mu_ra,mu_dec)) - self.catalog.loc[self.catalog['cat_name']==cat_name, 'ra_corr'] += mu_ra - self.catalog.loc[self.catalog['cat_name']==cat_name, 'dec_corr'] += mu_dec + self.sources.loc[self.sources['cat_name']==cat_name, 'ra_corr'] += mu_ra + self.sources.loc[self.sources['cat_name']==cat_name, 'dec_corr'] += mu_dec # Update history now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") From c323209bea99d40dc6bce4db2761afe2d373c010 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 3 Apr 2018 14:38:54 -0400 Subject: [PATCH 02/12] Added sklearn to requirements. --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c120058..0509981 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ numpy astropy matplotlib pandas -requests \ No newline at end of file +requests +sklearn \ No newline at end of file From 77c9bf1707fd35d7cf48765e9a8f725966c9f49f Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 3 Apr 2018 14:40:29 -0400 Subject: [PATCH 03/12] Added sklearn to the setup.py file. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4954b9..7d2d9b9 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas'], + install_requires=['numpy','astropy','matplotlib','requests','pandas','sklearn'], ) \ No newline at end of file From 62cce01393db826be80f6b50457378993fea5f2b Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 3 Apr 2018 14:50:42 -0400 Subject: [PATCH 04/12] Updated sklearn requirement to scikit-learn --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0509981..2467f30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ astropy matplotlib pandas requests -sklearn \ No newline at end of file +scikit-learn \ No newline at end of file diff --git a/setup.py b/setup.py index 7d2d9b9..760315f 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas','sklearn'], + install_requires=['numpy','astropy','matplotlib','requests','pandas','scikit-learn'], ) \ No newline at end of file From 1d470d1c6c25d9ecee44adc0f46a1c30d17d11d0 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Thu, 19 Apr 2018 12:41:45 -0400 Subject: [PATCH 05/12] Removed sklearn dependency and added tyr except statement for import to get around Travis fail. --- astrodbkit/astrocat.py | 7 +++++-- requirements.txt | 3 +-- setup.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index 1a752ce..73272d1 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -11,13 +11,16 @@ import astropy.table as at import astropy.coordinates as coord import datetime -from sklearn.cluster import DBSCAN from collections import Counter from scipy.stats import norm from astroquery.vizier import Vizier from astroquery.xmatch import XMatch -from sklearn.externals import joblib from astropy.coordinates import SkyCoord +try: + from sklearn.cluster import DBSCAN + from sklearn.externals import joblib +except ImportError: + pass Vizier.ROW_LIMIT = -1 diff --git a/requirements.txt b/requirements.txt index 2467f30..c120058 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,4 @@ numpy astropy matplotlib pandas -requests -scikit-learn \ No newline at end of file +requests \ No newline at end of file diff --git a/setup.py b/setup.py index 760315f..e4954b9 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas','scikit-learn'], + install_requires=['numpy','astropy','matplotlib','requests','pandas'], ) \ No newline at end of file From 273bce82a464cec7d7b7f9e119877b32cbb13496 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Thu, 19 Apr 2018 14:25:53 -0400 Subject: [PATCH 06/12] Added astrocat notebook. --- astrodbkit/astrocat.py | 63 +- .../using_astrocat-checkpoint.ipynb | 1136 +++++++++++++++++ astrodbkit/notebooks/using_astrocat.ipynb | 1136 +++++++++++++++++ 3 files changed, 2329 insertions(+), 6 deletions(-) create mode 100644 astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb create mode 100644 astrodbkit/notebooks/using_astrocat.ipynb diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index 73272d1..90efc6b 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -16,6 +16,9 @@ from astroquery.vizier import Vizier from astroquery.xmatch import XMatch from astropy.coordinates import SkyCoord +import pandas as pd +from bokeh.plotting import ColumnDataSource, figure, output_file, show +from bokeh.io import output_notebook, show try: from sklearn.cluster import DBSCAN from sklearn.externals import joblib @@ -52,6 +55,39 @@ def info(self): """ print(self.history) + def plot(self, cat_name, x, y, **kwargs): + """ + Plot the named columns of the given attribute if the value is a pandas.DataFrame + + Parameters + ---------- + cat_name: str + The attribute name + x: str + The name of the column to plot on the x-axis + y: str + The name of the column to plot on the y-axis + """ + # Get the attribute + if isinstance(cat_name, str) and hasattr(self, cat_name): + attr = getattr(self, cat_name) + else: + print('No attribute named',cat_name) + return + + # Make sure the attribute is a DataFrame + if isinstance(attr, pd.core.frame.DataFrame): + ds = ColumnDataSource(attr) + myPlot = figure() + myPlot.xaxis.axis_label = x + myPlot.yaxis.axis_label = y + myPlot.circle(x, y, source=ds) + plt = show(myPlot, notebook_handle=True) + + else: + print(cat_name,'is not a Pandas DataFrame!') + return + def add_source(self, ra, dec, flag='', radius=10*q.arcsec, catalogs={}): """ Add a source to the catalog manually and find data in existing catalogs @@ -199,7 +235,7 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 except AttributeError: print("No catalog named '{}'. Set 'append=False' to create it.".format(cat_name)) - def inventory(self, source_id): + def inventory(self, source_id, return_inventory=False): """ Look at the inventory for a given source @@ -217,15 +253,30 @@ def inventory(self, source_id): print('Please enter an integer between 1 and',self.n_sources) else: - - print('Source:') - print(at.Table.from_pandas(self.sources[self.sources['id']==source_id]).pprint()) + + # Empty inventory + inv = {} + + # Add the record from the source table + inv['source'] = at.Table.from_pandas(self.sources[self.sources['id']==source_id]) + for cat_name in self.catalogs: cat = getattr(self, cat_name) rows = cat[cat['source_id']==source_id] if not rows.empty: - print('\n{}:'.format(cat_name)) - at.Table.from_pandas(rows).pprint() + inv[cat_name] = at.Table.from_pandas(rows) + + if return_inventory: + + # Return the data + return inv + + else: + + # Print out the data in each catalog + for cat_name, data in inv.items(): + print('\n',cat_name,':') + data.pprint() def _catalog_check(self, cat_name, append=False): """ diff --git a/astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb b/astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb new file mode 100644 index 0000000..e05042d --- /dev/null +++ b/astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb @@ -0,0 +1,1136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The `astrocat` module\n", + "This notebook will take you through a simple use case for the `astrocat` module. In our example, we will construct a source catalog of targets in the Orion Nebula Cluster from Vizier data." + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "(function(global) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n", + " window._bokeh_onload_callbacks = [];\n", + " window._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "\n", + " \n", + " if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n", + " window._bokeh_timeout = Date.now() + 5000;\n", + " window._bokeh_failed_load = false;\n", + " }\n", + "\n", + " var NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " if (window.Bokeh !== undefined) {\n", + " var el = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n", + " } else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", + " }\n", + " finally {\n", + " delete window._bokeh_onload_callbacks\n", + " }\n", + " console.info(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(js_urls, callback) {\n", + " window._bokeh_onload_callbacks.push(callback);\n", + " if (window._bokeh_is_loading > 0) {\n", + " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " window._bokeh_is_loading = js_urls.length;\n", + " for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " var s = document.createElement('script');\n", + " s.src = url;\n", + " s.async = false;\n", + " s.onreadystatechange = s.onload = function() {\n", + " window._bokeh_is_loading--;\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", + " run_callbacks()\n", + " }\n", + " };\n", + " s.onerror = function() {\n", + " console.warn(\"failed to load library \" + url);\n", + " };\n", + " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", + " }\n", + " };var element = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " if (element == null) {\n", + " console.log(\"Bokeh: ERROR: autoload.js configured with elementid '8265b4a8-0ce0-42a8-88d3-4337a027ee84' but no matching script tag was found. \")\n", + " return false;\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.js\"];\n", + "\n", + " var inline_js = [\n", + " function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\").textContent = \"BokehJS is loading...\";\n", + " },\n", + " function(Bokeh) {\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " \n", + " if ((window.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i](window.Bokeh);\n", + " }if (force === true) {\n", + " display_loaded();\n", + " }} else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!window._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " window._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " var cell = $(document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\")).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + "\n", + " }\n", + "\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(js_urls, function() {\n", + " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(this));" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from astrodbkit import astrocat\n", + "import astropy.coordinates as coord\n", + "import astropy.units as q\n", + "from bokeh.io import output_notebook\n", + "output_notebook()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's initialize the catalog and define our area of interest." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Create empty catalog\n", + "cat = astrocat.Catalog()\n", + "\n", + "# Region of interest (Keep it small for demonstration purposes)\n", + "ra = 83.81775*q.deg\n", + "dec = -5.38788889*q.deg\n", + "cone_radius = 0.05*q.deg" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's grab all the 2MASS sources in this region" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 915 rows from TMASS catalog...\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/246/out', 'TMASS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And take a look at the results, which are stored in a `pandas.DataFrame` as an attribute of the `Catalog` object" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
source_idra_corrdec_corrcatID_rradecerrMajerrMinerrPA...DoptPAoptBmagRmagNoptextKeyscanKeycoaddKeycoaddOpt
0NaN83.807105-5.427688TMASS_10.04118283.807105-5.4276880.100.100...--------0--643191479335244Opt
1NaN83.798154-5.413375TMASS_20.03209583.798154-5.4133750.080.0817...--------0--643191479335244Opt
2NaN83.803910-5.428761TMASS_30.04312983.803910-5.4287610.220.20101...--------0--643191479335244Opt
3NaN83.807116-5.429630TMASS_40.04305983.807116-5.4296300.120.12145...--------0--643191479335244Opt
4NaN83.799794-5.417830TMASS_50.03487083.799794-5.4178300.150.147...--------0--643191479335244Opt
5NaN83.804124-5.400455TMASS_60.01849283.804124-5.4004550.380.3744...--------0--643191479335244Opt
6NaN83.799942-5.413466TMASS_70.03112083.799942-5.4134660.260.26138...--------0--643191479335244Opt
7NaN83.804354-5.422725TMASS_80.03729983.804354-5.4227250.190.19138...--------0--643191479335244Opt
8NaN83.772534-5.404180TMASS_90.04787783.772534-5.4041800.060.060...--------0--643191479335244Opt
9NaN83.772389-5.402932TMASS_100.04760483.772389-5.4029320.060.060...--------0--643191479335244Opt
\n", + "

10 rows × 66 columns

\n", + "
" + ], + "text/plain": [ + " source_id ra_corr dec_corr catID _r ra dec \\\n", + "0 NaN 83.807105 -5.427688 TMASS_1 0.041182 83.807105 -5.427688 \n", + "1 NaN 83.798154 -5.413375 TMASS_2 0.032095 83.798154 -5.413375 \n", + "2 NaN 83.803910 -5.428761 TMASS_3 0.043129 83.803910 -5.428761 \n", + "3 NaN 83.807116 -5.429630 TMASS_4 0.043059 83.807116 -5.429630 \n", + "4 NaN 83.799794 -5.417830 TMASS_5 0.034870 83.799794 -5.417830 \n", + "5 NaN 83.804124 -5.400455 TMASS_6 0.018492 83.804124 -5.400455 \n", + "6 NaN 83.799942 -5.413466 TMASS_7 0.031120 83.799942 -5.413466 \n", + "7 NaN 83.804354 -5.422725 TMASS_8 0.037299 83.804354 -5.422725 \n", + "8 NaN 83.772534 -5.404180 TMASS_9 0.047877 83.772534 -5.404180 \n", + "9 NaN 83.772389 -5.402932 TMASS_10 0.047604 83.772389 -5.402932 \n", + "\n", + " errMaj errMin errPA ... Dopt PAopt Bmag Rmag Nopt extKey scanKey \\\n", + "0 0.10 0.10 0 ... -- -- -- -- 0 -- 64319 \n", + "1 0.08 0.08 17 ... -- -- -- -- 0 -- 64319 \n", + "2 0.22 0.20 101 ... -- -- -- -- 0 -- 64319 \n", + "3 0.12 0.12 145 ... -- -- -- -- 0 -- 64319 \n", + "4 0.15 0.14 7 ... -- -- -- -- 0 -- 64319 \n", + "5 0.38 0.37 44 ... -- -- -- -- 0 -- 64319 \n", + "6 0.26 0.26 138 ... -- -- -- -- 0 -- 64319 \n", + "7 0.19 0.19 138 ... -- -- -- -- 0 -- 64319 \n", + "8 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "9 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "\n", + " coaddKey coadd Opt \n", + "0 1479335 244 Opt \n", + "1 1479335 244 Opt \n", + "2 1479335 244 Opt \n", + "3 1479335 244 Opt \n", + "4 1479335 244 Opt \n", + "5 1479335 244 Opt \n", + "6 1479335 244 Opt \n", + "7 1479335 244 Opt \n", + "8 1479335 244 Opt \n", + "9 1479335 244 Opt \n", + "\n", + "[10 rows x 66 columns]" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.TMASS[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot some stuff for funzies..." + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cat.plot('TMASS', 'Jmag', 'Hmag')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets's get some ALLWISE and HST/ACS data as well." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/328/allwise for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 13 rows from ALLWISE catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ALLWISE']\n", + "Searching J/ApJS/207/10/table5 for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 2097 rows from ACS catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ACS', 'ALLWISE']\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/328/allwise', 'ALLWISE', ra, dec, cone_radius)\n", + "cat.Vizier_query('J/ApJS/207/10/table5', 'ACS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's look at the source catalog, which was generated from the proximity of the data points in ingested catalogs" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idradecflagdatasets
0183.807105-5.427688None1
1283.798143-5.413356None3
2383.803910-5.428761None1
3483.807116-5.429630None1
4583.799794-5.417830None1
5683.804124-5.400455None1
6783.799942-5.413466None1
7883.804354-5.422725None1
8983.772525-5.404179None3
91083.772377-5.402924None3
\n", + "
" + ], + "text/plain": [ + " id ra dec flag datasets\n", + "0 1 83.807105 -5.427688 None 1\n", + "1 2 83.798143 -5.413356 None 3\n", + "2 3 83.803910 -5.428761 None 1\n", + "3 4 83.807116 -5.429630 None 1\n", + "4 5 83.799794 -5.417830 None 1\n", + "5 6 83.804124 -5.400455 None 1\n", + "6 7 83.799942 -5.413466 None 1\n", + "7 8 83.804354 -5.422725 None 1\n", + "8 9 83.772525 -5.404179 None 3\n", + "9 10 83.772377 -5.402924 None 3" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.sources[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And check out the inventory of one of these objects" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Source:\n", + " id ra dec flag datasets\n", + "--- ------------- -------------- ---- --------\n", + " 10 83.7723768889 -5.40292362963 -- 3\n", + "None\n", + "\n", + "TMASS:\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- --------- --------- -------- -------- ... ------- -------- ----- ---\n", + " 10 83.772389 -5.402932 TMASS_10 0.047604 ... 64319 1479335 244 Opt\n", + "\n", + "ACS:\n", + "source_id ra_corr dec_corr ... sky850s e_sky850s Obs \n", + "--------- ------------- -------------- ... ------- --------- -------------------\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-05T06:41:24\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-06T04:26:25\n" + ] + } + ], + "source": [ + "cat.inventory(source_id=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we can see that this source was detected in the 2MASS and ACS catalogs but not in the ALLWISE catalog.\n", + "\n", + "We can also manually add sources and have them cross-matched in the given catalogs. Let's do Vega just for this demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from TMASS catalog...\n", + "Searching J/ApJS/207/10/table5 for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "No data found in J/ApJS/207/10/table5 within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg).\n", + "Searching II/328/allwise for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from ALLWISE catalog...\n" + ] + } + ], + "source": [ + "# RA and Dec of new source\n", + "new_ra = 279.23473479*q.deg\n", + "new_dec = 38.78368896*q.deg\n", + "cat.add_source(new_ra, new_dec)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Take a look!" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " TMASS :\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- ---------- --------- --------- ----- ... ------- -------- ----- ---\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt\n", + "\n", + " source :\n", + " id ra dec flag datasets\n", + "---- ------------ ----------- ---- --------\n", + "1307 279.23473479 38.78368896 0\n", + "\n", + " ALLWISE :\n", + "source_id ra_corr dec_corr catID _r ... covW3 covW4 _2Mkey d2M _2M\n", + "--------- --------- ---------- ---------- ----- ... ----- ------ ------ --- ---\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 0.889 34.399 -- -- 2M\n" + ] + } + ], + "source": [ + "# The new source has the highest source_id\n", + "cat.inventory(source_id=cat.n_sources)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'TMASS': \n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt \n", + " int64 float64 float64 str9 float64 ... int64 int64 int64 str3\n", + "--------- ---------- --------- --------- ------- ... ------- -------- ----- ----\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt, 'source':
\n", + " id ra dec flag datasets\n", + "int64 float64 float64 str1 int64 \n", + "----- ------------ ----------- ---- --------\n", + " 1307 279.23473479 38.78368896 0, 'ALLWISE':
\n", + "source_id ra_corr dec_corr catID _r ... covW4 _2Mkey d2M _2M \n", + " int64 float64 float64 str10 float64 ... object object object str2\n", + "--------- --------- ---------- ---------- ------- ... ------ ------ ------ ----\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 34.399 -- -- 2M}\n" + ] + } + ], + "source": [ + "# Or return the data as a dictionary\n", + "inventory = cat.inventory(source_id=cat.n_sources, return_inventory=True)\n", + "print(inventory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Astroconda", + "language": "python", + "name": "astroconda_kernel" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/astrodbkit/notebooks/using_astrocat.ipynb b/astrodbkit/notebooks/using_astrocat.ipynb new file mode 100644 index 0000000..45272df --- /dev/null +++ b/astrodbkit/notebooks/using_astrocat.ipynb @@ -0,0 +1,1136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The `astrocat` module\n", + "This notebook will take you through a simple use case for the `astrocat` module. In our example, we will construct a source catalog of targets in the Orion Nebula Cluster from Vizier data." + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "(function(global) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n", + " window._bokeh_onload_callbacks = [];\n", + " window._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "\n", + " \n", + " if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n", + " window._bokeh_timeout = Date.now() + 5000;\n", + " window._bokeh_failed_load = false;\n", + " }\n", + "\n", + " var NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " if (window.Bokeh !== undefined) {\n", + " var el = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n", + " } else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", + " }\n", + " finally {\n", + " delete window._bokeh_onload_callbacks\n", + " }\n", + " console.info(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(js_urls, callback) {\n", + " window._bokeh_onload_callbacks.push(callback);\n", + " if (window._bokeh_is_loading > 0) {\n", + " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " window._bokeh_is_loading = js_urls.length;\n", + " for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " var s = document.createElement('script');\n", + " s.src = url;\n", + " s.async = false;\n", + " s.onreadystatechange = s.onload = function() {\n", + " window._bokeh_is_loading--;\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", + " run_callbacks()\n", + " }\n", + " };\n", + " s.onerror = function() {\n", + " console.warn(\"failed to load library \" + url);\n", + " };\n", + " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", + " }\n", + " };var element = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " if (element == null) {\n", + " console.log(\"Bokeh: ERROR: autoload.js configured with elementid '8265b4a8-0ce0-42a8-88d3-4337a027ee84' but no matching script tag was found. \")\n", + " return false;\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.js\"];\n", + "\n", + " var inline_js = [\n", + " function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\").textContent = \"BokehJS is loading...\";\n", + " },\n", + " function(Bokeh) {\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " \n", + " if ((window.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i](window.Bokeh);\n", + " }if (force === true) {\n", + " display_loaded();\n", + " }} else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!window._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " window._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " var cell = $(document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\")).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + "\n", + " }\n", + "\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(js_urls, function() {\n", + " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(this));" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from astrodbkit import astrocat\n", + "import astropy.coordinates as coord\n", + "import astropy.units as q\n", + "from bokeh.io import output_notebook\n", + "output_notebook()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's initialize the catalog and define our area of interest." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Create empty catalog\n", + "cat = astrocat.Catalog()\n", + "\n", + "# Region of interest (Keep it small for demonstration purposes)\n", + "ra = 83.81775*q.deg\n", + "dec = -5.38788889*q.deg\n", + "cone_radius = 0.05*q.deg" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's grab all the 2MASS sources in this region" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 915 rows from TMASS catalog...\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/246/out', 'TMASS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And take a look at the results, which are stored in a `pandas.DataFrame` as an attribute of the `Catalog` object" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
source_idra_corrdec_corrcatID_rradecerrMajerrMinerrPA...DoptPAoptBmagRmagNoptextKeyscanKeycoaddKeycoaddOpt
0NaN83.807105-5.427688TMASS_10.04118283.807105-5.4276880.100.100...--------0--643191479335244Opt
1NaN83.798154-5.413375TMASS_20.03209583.798154-5.4133750.080.0817...--------0--643191479335244Opt
2NaN83.803910-5.428761TMASS_30.04312983.803910-5.4287610.220.20101...--------0--643191479335244Opt
3NaN83.807116-5.429630TMASS_40.04305983.807116-5.4296300.120.12145...--------0--643191479335244Opt
4NaN83.799794-5.417830TMASS_50.03487083.799794-5.4178300.150.147...--------0--643191479335244Opt
5NaN83.804124-5.400455TMASS_60.01849283.804124-5.4004550.380.3744...--------0--643191479335244Opt
6NaN83.799942-5.413466TMASS_70.03112083.799942-5.4134660.260.26138...--------0--643191479335244Opt
7NaN83.804354-5.422725TMASS_80.03729983.804354-5.4227250.190.19138...--------0--643191479335244Opt
8NaN83.772534-5.404180TMASS_90.04787783.772534-5.4041800.060.060...--------0--643191479335244Opt
9NaN83.772389-5.402932TMASS_100.04760483.772389-5.4029320.060.060...--------0--643191479335244Opt
\n", + "

10 rows × 66 columns

\n", + "" + ], + "text/plain": [ + " source_id ra_corr dec_corr catID _r ra dec \\\n", + "0 NaN 83.807105 -5.427688 TMASS_1 0.041182 83.807105 -5.427688 \n", + "1 NaN 83.798154 -5.413375 TMASS_2 0.032095 83.798154 -5.413375 \n", + "2 NaN 83.803910 -5.428761 TMASS_3 0.043129 83.803910 -5.428761 \n", + "3 NaN 83.807116 -5.429630 TMASS_4 0.043059 83.807116 -5.429630 \n", + "4 NaN 83.799794 -5.417830 TMASS_5 0.034870 83.799794 -5.417830 \n", + "5 NaN 83.804124 -5.400455 TMASS_6 0.018492 83.804124 -5.400455 \n", + "6 NaN 83.799942 -5.413466 TMASS_7 0.031120 83.799942 -5.413466 \n", + "7 NaN 83.804354 -5.422725 TMASS_8 0.037299 83.804354 -5.422725 \n", + "8 NaN 83.772534 -5.404180 TMASS_9 0.047877 83.772534 -5.404180 \n", + "9 NaN 83.772389 -5.402932 TMASS_10 0.047604 83.772389 -5.402932 \n", + "\n", + " errMaj errMin errPA ... Dopt PAopt Bmag Rmag Nopt extKey scanKey \\\n", + "0 0.10 0.10 0 ... -- -- -- -- 0 -- 64319 \n", + "1 0.08 0.08 17 ... -- -- -- -- 0 -- 64319 \n", + "2 0.22 0.20 101 ... -- -- -- -- 0 -- 64319 \n", + "3 0.12 0.12 145 ... -- -- -- -- 0 -- 64319 \n", + "4 0.15 0.14 7 ... -- -- -- -- 0 -- 64319 \n", + "5 0.38 0.37 44 ... -- -- -- -- 0 -- 64319 \n", + "6 0.26 0.26 138 ... -- -- -- -- 0 -- 64319 \n", + "7 0.19 0.19 138 ... -- -- -- -- 0 -- 64319 \n", + "8 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "9 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "\n", + " coaddKey coadd Opt \n", + "0 1479335 244 Opt \n", + "1 1479335 244 Opt \n", + "2 1479335 244 Opt \n", + "3 1479335 244 Opt \n", + "4 1479335 244 Opt \n", + "5 1479335 244 Opt \n", + "6 1479335 244 Opt \n", + "7 1479335 244 Opt \n", + "8 1479335 244 Opt \n", + "9 1479335 244 Opt \n", + "\n", + "[10 rows x 66 columns]" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.TMASS[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot some stuff for funzies..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cat.plot('TMASS', 'Jmag', 'Hmag')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets's get some ALLWISE and HST/ACS data as well." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/328/allwise for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 13 rows from ALLWISE catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ALLWISE']\n", + "Searching J/ApJS/207/10/table5 for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 2097 rows from ACS catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ACS', 'ALLWISE']\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/328/allwise', 'ALLWISE', ra, dec, cone_radius)\n", + "cat.Vizier_query('J/ApJS/207/10/table5', 'ACS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's look at the source catalog, which was generated from the proximity of the data points in ingested catalogs" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idradecflagdatasets
0183.807105-5.427688None1
1283.798143-5.413356None3
2383.803910-5.428761None1
3483.807116-5.429630None1
4583.799794-5.417830None1
5683.804124-5.400455None1
6783.799942-5.413466None1
7883.804354-5.422725None1
8983.772525-5.404179None3
91083.772377-5.402924None3
\n", + "
" + ], + "text/plain": [ + " id ra dec flag datasets\n", + "0 1 83.807105 -5.427688 None 1\n", + "1 2 83.798143 -5.413356 None 3\n", + "2 3 83.803910 -5.428761 None 1\n", + "3 4 83.807116 -5.429630 None 1\n", + "4 5 83.799794 -5.417830 None 1\n", + "5 6 83.804124 -5.400455 None 1\n", + "6 7 83.799942 -5.413466 None 1\n", + "7 8 83.804354 -5.422725 None 1\n", + "8 9 83.772525 -5.404179 None 3\n", + "9 10 83.772377 -5.402924 None 3" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.sources[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And check out the inventory of one of these objects" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Source:\n", + " id ra dec flag datasets\n", + "--- ------------- -------------- ---- --------\n", + " 10 83.7723768889 -5.40292362963 -- 3\n", + "None\n", + "\n", + "TMASS:\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- --------- --------- -------- -------- ... ------- -------- ----- ---\n", + " 10 83.772389 -5.402932 TMASS_10 0.047604 ... 64319 1479335 244 Opt\n", + "\n", + "ACS:\n", + "source_id ra_corr dec_corr ... sky850s e_sky850s Obs \n", + "--------- ------------- -------------- ... ------- --------- -------------------\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-05T06:41:24\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-06T04:26:25\n" + ] + } + ], + "source": [ + "cat.inventory(source_id=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we can see that this source was detected in the 2MASS and ACS catalogs but not in the ALLWISE catalog.\n", + "\n", + "We can also manually add sources and have them cross-matched in the given catalogs. Let's do Vega just for this demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from TMASS catalog...\n", + "Searching J/ApJS/207/10/table5 for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "No data found in J/ApJS/207/10/table5 within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg).\n", + "Searching II/328/allwise for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from ALLWISE catalog...\n" + ] + } + ], + "source": [ + "# RA and Dec of new source\n", + "new_ra = 279.23473479*q.deg\n", + "new_dec = 38.78368896*q.deg\n", + "cat.add_source(new_ra, new_dec)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Take a look!" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " TMASS :\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- ---------- --------- --------- ----- ... ------- -------- ----- ---\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt\n", + "\n", + " source :\n", + " id ra dec flag datasets\n", + "---- ------------ ----------- ---- --------\n", + "1307 279.23473479 38.78368896 0\n", + "\n", + " ALLWISE :\n", + "source_id ra_corr dec_corr catID _r ... covW3 covW4 _2Mkey d2M _2M\n", + "--------- --------- ---------- ---------- ----- ... ----- ------ ------ --- ---\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 0.889 34.399 -- -- 2M\n" + ] + } + ], + "source": [ + "# The new source has the highest source_id\n", + "cat.inventory(source_id=cat.n_sources)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'TMASS': \n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt \n", + " int64 float64 float64 str9 float64 ... int64 int64 int64 str3\n", + "--------- ---------- --------- --------- ------- ... ------- -------- ----- ----\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt, 'source':
\n", + " id ra dec flag datasets\n", + "int64 float64 float64 str1 int64 \n", + "----- ------------ ----------- ---- --------\n", + " 1307 279.23473479 38.78368896 0, 'ALLWISE':
\n", + "source_id ra_corr dec_corr catID _r ... covW4 _2Mkey d2M _2M \n", + " int64 float64 float64 str10 float64 ... object object object str2\n", + "--------- --------- ---------- ---------- ------- ... ------ ------ ------ ----\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 34.399 -- -- 2M}\n" + ] + } + ], + "source": [ + "# Or return the data as a dictionary\n", + "inventory = cat.inventory(source_id=cat.n_sources, return_inventory=True)\n", + "print(inventory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Astroconda", + "language": "python", + "name": "astroconda_kernel" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} From f5551b430fbd4df70331562987fde3b7db5772b8 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Thu, 19 Apr 2018 14:48:07 -0400 Subject: [PATCH 07/12] Added scipy requirement for Travis. --- astrodbkit/notebooks/using_astrocat.ipynb | 2 +- requirements.txt | 3 ++- setup.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/astrodbkit/notebooks/using_astrocat.ipynb b/astrodbkit/notebooks/using_astrocat.ipynb index 45272df..b0646d4 100644 --- a/astrodbkit/notebooks/using_astrocat.ipynb +++ b/astrodbkit/notebooks/using_astrocat.ipynb @@ -601,7 +601,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 85, "metadata": { "collapsed": false }, diff --git a/requirements.txt b/requirements.txt index c120058..e2167b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ numpy astropy matplotlib pandas -requests \ No newline at end of file +requests +scipy \ No newline at end of file diff --git a/setup.py b/setup.py index e4954b9..7be4922 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas'], + install_requires=['numpy','astropy','matplotlib','requests','pandas','scipy'], ) \ No newline at end of file From b1cbe9f99953988b42cac89efd9aecbeebfa32ee Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Thu, 19 Apr 2018 15:03:23 -0400 Subject: [PATCH 08/12] Removed scipy dependency. --- astrodbkit/astrocat.py | 2 +- requirements.txt | 3 +-- setup.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index 90efc6b..e72ea43 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -12,7 +12,7 @@ import astropy.coordinates as coord import datetime from collections import Counter -from scipy.stats import norm +# from scipy.stats import norm from astroquery.vizier import Vizier from astroquery.xmatch import XMatch from astropy.coordinates import SkyCoord diff --git a/requirements.txt b/requirements.txt index e2167b3..c120058 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,4 @@ numpy astropy matplotlib pandas -requests -scipy \ No newline at end of file +requests \ No newline at end of file diff --git a/setup.py b/setup.py index 7be4922..e4954b9 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas','scipy'], + install_requires=['numpy','astropy','matplotlib','requests','pandas'], ) \ No newline at end of file From 1955c73b21de41eae55a8c81350b367fcc29c121 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Thu, 19 Apr 2018 15:16:41 -0400 Subject: [PATCH 09/12] Added astroquery dependency. --- requirements.txt | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c120058..e1e3afe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ numpy astropy matplotlib pandas -requests \ No newline at end of file +requests +astroquery \ No newline at end of file diff --git a/setup.py b/setup.py index e4954b9..e0b6aa7 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas'], + install_requires=['numpy','astropy','matplotlib','requests','pandas','astroquery'], ) \ No newline at end of file From 3771f3ea55a62d09596d00bd778f5242b13fc419 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 6 Jun 2018 11:51:53 -0400 Subject: [PATCH 10/12] Added link to original record in astrocat Vizier query. --- astrodbkit/astrocat.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index e72ea43..f1bc869 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -383,10 +383,14 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec try: data = V.query_region(crds, radius=radius, catalog=viz_cat)[0] + + # Add the link to original record + data['record'] = ['http://vizier.u-strasbg.fr/viz-bin/VizieR-5?-ref=VIZ5b17f9660734&-out.add=.&-source={}&recno={}'.format(viz_cat,n+1) for n in range(len(data))] + except: print("No data found in {} within {} of ({}, {}).".format(viz_cat, radius, ra, dec)) return - + # Ingest the data self.ingest_data(data, cat_name, 'id', ra_col=ra_col, dec_col=dec_col, cat_loc=viz_cat, append=append, force_id=force_id) @@ -692,7 +696,7 @@ def default_rename_columns(cat_name): defaults = {'2MASS':{'JD':'epoch', 'Qflg':'flags', 'Jmag':'2MASS.J', 'Hmag':'2MASS.H', 'Kmag':'2MASS.Ks', 'e_Jmag':'2MASS.J_unc', 'e_Hmag':'2MASS.H_unc', 'e_Kmag':'2MASS.Ks_unc'}, 'WISE':{'qph':'flags', 'W1mag':'WISE.W1', 'W2mag':'WISE.W2', 'W3mag':'WISE.W3', 'W4mag':'WISE.W4', 'e_W1mag':'WISE.W1_unc', 'e_W2mag':'WISE.W2_unc', 'e_W3mag':'WISE.W3_unc', 'e_W4mag':'WISE.W4_unc'}, 'SDSS':{'ObsDate':'epoch', 'flags':'oflags', 'Q':'flags', 'umag':'SDSS.u', 'gmag':'SDSS.g', 'rmag':'SDSS.r', 'imag':'SDSS.i', 'zmag':'SDSS.z', 'e_umag':'SDSS.u_unc', 'e_gmag':'SDSS.g_unc', 'e_rmag':'SDSS.r_unc', 'e_imag':'SDSS.i_unc', 'e_zmag':'SDSS.z_unc'}, - 'TGAS':{'Epoch':'epoch', 'Plx':'parallax', 'e_Plx':'parallax_unc'}} + 'GAIA':{'Epoch':'epoch', 'Plx':'parallax', 'e_Plx':'parallax_unc', 'Gmag':'Gaia.G', 'e_Gmag':'Gaia.G_unc', 'BPmag':'Gaia.BP', 'e_BPmag':'Gaia.BP_unc', 'RPmag':'Gaia.RP', 'e_RPmag':'Gaia.RP_unc'}} return defaults[cat_name] @@ -713,7 +717,7 @@ def default_column_fill(cat_name): defaults = {'2MASS':{'publication_shortname':'Cutr03', 'telescope_id':2, 'instrument_id':5, 'system_id':2}, 'WISE':{'publication_shortname':'Cutr13', 'telescope_id':3, 'instrument_id':6, 'system_id':2}, 'SDSS':{'publication_shortname':'Alam15', 'telescope_id':6, 'instrument_id':9, 'system_id':2}, - 'TGAS':{'publication_shortname':'Gaia16', 'telescope_id':4, 'instrument_id':7, 'system_id':1}} + 'GAIA':{'publication_shortname':'Gaia18', 'telescope_id':4, 'instrument_id':7, 'system_id':1}} return defaults[cat_name] \ No newline at end of file From 3c57948c60dd4ed64810ad7b550b697a222a631b Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 6 Jun 2018 12:05:19 -0400 Subject: [PATCH 11/12] Added bokeh to dependencies. --- requirements.txt | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e1e3afe..4aadcd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ astropy matplotlib pandas requests -astroquery \ No newline at end of file +astroquery +bokeh \ No newline at end of file diff --git a/setup.py b/setup.py index e0b6aa7..cf3f219 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas','astroquery'], + install_requires=['numpy','astropy','matplotlib','requests','pandas','astroquery','bokeh'], ) \ No newline at end of file From 0ef4ba119d51628edf32fd2214a1baa08753d046 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 25 Mar 2020 09:08:18 -0400 Subject: [PATCH 12/12] Catch for column rename --- astrodbkit/astrocat.py | 17 +++++++---------- astrodbkit/astrodb.py | 12 ++++++++++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index f1bc869..0b7d237 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -19,11 +19,8 @@ import pandas as pd from bokeh.plotting import ColumnDataSource, figure, output_file, show from bokeh.io import output_notebook, show -try: - from sklearn.cluster import DBSCAN - from sklearn.externals import joblib -except ImportError: - pass +from sklearn.cluster import DBSCAN +from sklearn.externals import joblib Vizier.ROW_LIMIT = -1 @@ -39,7 +36,7 @@ def __init__(self, name='Test'): The name of the database """ self.name = name - self.sources = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) + self.sources = pd.DataFrame(columns=('id', 'ra', 'dec', 'flag', 'datasets')) self.n_sources = 0 self.history = "{}: Database created".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.catalogs = {} @@ -148,7 +145,7 @@ def delete_source(self, id): print('{} records removed from {} catalog'.format(int(len(getattr(self, cat_name))-len(new_cat)), cat_name)) setattr(self, cat_name, new_cat) - def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ2000', cat_loc='', append=False, force_id='', count=-1): + def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ2000', cat_loc='', append=False, delimiter='\t', force_id='', count=-1): """ Ingest a data file and regroup sources @@ -183,7 +180,7 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 if isinstance(data, str): cat_loc = cat_loc or data - data = pd.read_csv(data, sep='\t', comment='#', engine='python')[:count] + data = pd.read_csv(data, sep=delimiter, comment='#', engine='python')[:count] elif isinstance(data, pd.core.frame.DataFrame): cat_loc = cat_loc or type(data) @@ -345,7 +342,7 @@ def SDSS_spectra_query(self, cat_name, ra, dec, radius, group=True, **kwargs): if len(self.catalogs)>1 and group: self.group_sources(self.xmatch_radius) - def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec_col='DEJ2000', columns=["**"], append=False, force_id='', group=True, **kwargs): + def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec_col='DEJ2000', columns=["**", "+_r"], append=False, force_id='', group=True, nrows=-1, **kwargs): """ Use astroquery to search a catalog for sources within a search cone @@ -379,7 +376,7 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec print("Searching {} for sources within {} of ({}, {}). Please be patient...".format(viz_cat, radius, ra, dec)) crds = coord.SkyCoord(ra=ra, dec=dec, frame='icrs') V = Vizier(columns=columns, **kwargs) - V.ROW_LIMIT = -1 + V.ROW_LIMIT = nrows try: data = V.query_region(crds, radius=radius, catalog=viz_cat)[0] diff --git a/astrodbkit/astrodb.py b/astrodbkit/astrodb.py index d1ce360..9a7a15c 100755 --- a/astrodbkit/astrodb.py +++ b/astrodbkit/astrodb.py @@ -53,7 +53,7 @@ def create_database(dbpath, schema='', overwrite=True): # Load the schema if given if schema: - os.system("cat {} | sqlite3 {}".format(schema,dbpath)) + os.system("cat {} | sqlite3 {}".format(schema, dbpath)) # Otherwise just make an empty SOURCES table else: @@ -326,8 +326,16 @@ def add_data(self, data, table, delimiter='|', bands='', clean_up=True, rename_c # Rename columns if isinstance(rename_columns,str): rename_columns = astrocat.default_rename_columns(rename_columns) + + try_again = [] for input_name,new_name in rename_columns.items(): - data.rename_column(input_name,new_name) + try: + data.rename_column(input_name,new_name) + except KeyError: + try_again.append(input_name) + + for input_name in try_again: + data.rename_column(input_name,rename_columns[input_name]) # Add column fills if isinstance(column_fill,str):