diff --git a/astrodbkit/astrocat.py b/astrodbkit/astrocat.py index 700918c..0b7d237 100644 --- a/astrodbkit/astrocat.py +++ b/astrodbkit/astrocat.py @@ -11,13 +11,16 @@ import astropy.table as at import astropy.coordinates as coord import datetime -from sklearn.cluster import DBSCAN from collections import Counter -from scipy.stats import norm +# from scipy.stats import norm from astroquery.vizier import Vizier from astroquery.xmatch import XMatch -from sklearn.externals import joblib from astropy.coordinates import SkyCoord +import pandas as pd +from bokeh.plotting import ColumnDataSource, figure, output_file, show +from bokeh.io import output_notebook, show +from sklearn.cluster import DBSCAN +from sklearn.externals import joblib Vizier.ROW_LIMIT = -1 @@ -33,7 +36,7 @@ def __init__(self, name='Test'): The name of the database """ self.name = name - self.catalog = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) + self.sources = pd.DataFrame(columns=('id', 'ra', 'dec', 'flag', 'datasets')) self.n_sources = 0 self.history = "{}: Database created".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.catalogs = {} @@ -49,7 +52,40 @@ def info(self): """ print(self.history) - def add_source(self, ra, dec, flag='', radius=10*q.arcsec): + def plot(self, cat_name, x, y, **kwargs): + """ + Plot the named columns of the given attribute if the value is a pandas.DataFrame + + Parameters + ---------- + cat_name: str + The attribute name + x: str + The name of the column to plot on the x-axis + y: str + The name of the column to plot on the y-axis + """ + # Get the attribute + if isinstance(cat_name, str) and hasattr(self, cat_name): + attr = getattr(self, cat_name) + else: + print('No attribute named',cat_name) + return + + # Make sure the attribute is a DataFrame + if isinstance(attr, pd.core.frame.DataFrame): + ds = ColumnDataSource(attr) + myPlot = figure() + myPlot.xaxis.axis_label = x + myPlot.yaxis.axis_label = y + myPlot.circle(x, y, source=ds) + plt = show(myPlot, notebook_handle=True) + + else: + print(cat_name,'is not a Pandas DataFrame!') + return + + def add_source(self, ra, dec, flag='', radius=10*q.arcsec, catalogs={}): """ Add a source to the catalog manually and find data in existing catalogs @@ -63,9 +99,12 @@ def add_source(self, ra, dec, flag='', radius=10*q.arcsec): A flag for the source radius: float The cross match radius for the list of catalogs + catalogs: dict + Additional catalogs to search, e.g. + catalogs={'TMASS':{'cat_loc':'II/246/out', 'id_col':'id', 'ra_col':'RAJ2000', 'dec_col':'DEJ2000'}} """ - # Get the id - id = int(len(self.catalog)+1) + # Set the id + self.n_sources += 1 # Check the coordinates ra = ra.to(q.deg) @@ -74,10 +113,16 @@ def add_source(self, ra, dec, flag='', radius=10*q.arcsec): # Search the catalogs for this source for cat_name,params in self.catalogs.items(): - self.Vizier_query(params['cat_loc'], cat_name, ra, dec, radius, ra_col=params['ra_col'], dec_col=params['dec_col'], append=True, group=False) + self.Vizier_query(params['cat_loc'], cat_name, ra, dec, radius, ra_col=params['ra_col'], dec_col=params['dec_col'], append=True, force_id=self.n_sources, group=False) + + # Search additional catalogs + for cat_name,params in catalogs.items(): + if cat_name not in self.catalogs: + self.Vizier_query(params['cat_loc'], cat_name, ra, dec, radius, ra_col=params['ra_col'], dec_col=params['dec_col'], force_id=self.n_sources, group=False) # Add the source to the catalog - self.catalog = self.catalog.append([id, ra.value, dec.value, flag, datasets], ignore_index=True) + new_cat = pd.DataFrame([[self.n_sources, ra.value, dec.value, flag, datasets]], columns=self.sources.columns) + self.sources = self.sources.append(new_cat, ignore_index=True) def delete_source(self, id): """ @@ -89,10 +134,10 @@ def delete_source(self, id): The id of the source in the catalog """ # Set the index - self.catalog.set_index('id') + self.sources.set_index('id') # Exclude the unwanted source - self.catalog = self.catalog[self.catalog.id!=id] + self.sources = self.sources[self.sources.id!=id] # Remove the records from the catalogs for cat_name in self.catalogs: @@ -100,7 +145,7 @@ def delete_source(self, id): print('{} records removed from {} catalog'.format(int(len(getattr(self, cat_name))-len(new_cat)), cat_name)) setattr(self, cat_name, new_cat) - def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ2000', cat_loc='', append=False, count=-1): + def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ2000', cat_loc='', append=False, delimiter='\t', force_id='', count=-1): """ Ingest a data file and regroup sources @@ -120,6 +165,8 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 The location of the original catalog data append: bool Append the catalog rather than replace + force_id: int + Assigns a specific id in the catalog count: int The number of table rows to add (This is mainly for testing purposes) @@ -133,7 +180,7 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 if isinstance(data, str): cat_loc = cat_loc or data - data = pd.read_csv(data, sep='\t', comment='#', engine='python')[:count] + data = pd.read_csv(data, sep=delimiter, comment='#', engine='python')[:count] elif isinstance(data, pd.core.frame.DataFrame): cat_loc = cat_loc or type(data) @@ -167,7 +214,7 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 data.insert(0,'catID', ['{}_{}'.format(cat_name,n+1) for n in range(last,last+len(data))]) data.insert(0,'dec_corr', data['dec']) data.insert(0,'ra_corr', data['ra']) - data.insert(0,'source_id', np.nan) + data.insert(0,'source_id', force_id or np.nan) print('Ingesting {} rows from {} catalog...'.format(len(data),cat_name)) @@ -185,7 +232,7 @@ def ingest_data(self, data, cat_name, id_col, ra_col='_RAJ2000', dec_col='_DEJ20 except AttributeError: print("No catalog named '{}'. Set 'append=False' to create it.".format(cat_name)) - def inventory(self, source_id): + def inventory(self, source_id, return_inventory=False): """ Look at the inventory for a given source @@ -203,15 +250,30 @@ def inventory(self, source_id): print('Please enter an integer between 1 and',self.n_sources) else: - - print('Source:') - print(at.Table.from_pandas(self.catalog[self.catalog['id']==source_id]).pprint()) + + # Empty inventory + inv = {} + + # Add the record from the source table + inv['source'] = at.Table.from_pandas(self.sources[self.sources['id']==source_id]) + for cat_name in self.catalogs: cat = getattr(self, cat_name) rows = cat[cat['source_id']==source_id] if not rows.empty: - print('\n{}:'.format(cat_name)) - at.Table.from_pandas(rows).pprint() + inv[cat_name] = at.Table.from_pandas(rows) + + if return_inventory: + + # Return the data + return inv + + else: + + # Print out the data in each catalog + for cat_name, data in inv.items(): + print('\n',cat_name,':') + data.pprint() def _catalog_check(self, cat_name, append=False): """ @@ -262,7 +324,7 @@ def SDSS_spectra_query(self, cat_name, ra, dec, radius, group=True, **kwargs): if self._catalog_check(cat_name): # Prep the current catalog as an astropy.QTable - tab = at.Table.from_pandas(self.catalog) + tab = at.Table.from_pandas(self.sources) # Cone search Vizier print("Searching SDSS for sources within {} of ({}, {}). Please be patient...".format(viz_cat, radius, ra, dec)) @@ -280,7 +342,7 @@ def SDSS_spectra_query(self, cat_name, ra, dec, radius, group=True, **kwargs): if len(self.catalogs)>1 and group: self.group_sources(self.xmatch_radius) - def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec_col='DEJ2000', columns=["**"], append=False, group=True, **kwargs): + def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec_col='DEJ2000', columns=["**", "+_r"], append=False, force_id='', group=True, nrows=-1, **kwargs): """ Use astroquery to search a catalog for sources within a search cone @@ -304,6 +366,8 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec The list of columns to pass to astroquery append: bool Append the catalog rather than replace + force_id: int + Assigns a specific id in the catalog """ # Verify the cat_name if self._catalog_check(cat_name, append=append): @@ -312,16 +376,20 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec print("Searching {} for sources within {} of ({}, {}). Please be patient...".format(viz_cat, radius, ra, dec)) crds = coord.SkyCoord(ra=ra, dec=dec, frame='icrs') V = Vizier(columns=columns, **kwargs) - V.ROW_LIMIT = -1 + V.ROW_LIMIT = nrows try: data = V.query_region(crds, radius=radius, catalog=viz_cat)[0] + + # Add the link to original record + data['record'] = ['http://vizier.u-strasbg.fr/viz-bin/VizieR-5?-ref=VIZ5b17f9660734&-out.add=.&-source={}&recno={}'.format(viz_cat,n+1) for n in range(len(data))] + except: print("No data found in {} within {} of ({}, {}).".format(viz_cat, radius, ra, dec)) return - + # Ingest the data - self.ingest_data(data, cat_name, 'id', ra_col=ra_col, dec_col=dec_col, cat_loc=viz_cat, append=append) + self.ingest_data(data, cat_name, 'id', ra_col=ra_col, dec_col=dec_col, cat_loc=viz_cat, append=append, force_id=force_id) # Regroup if len(self.catalogs)>1 and group: @@ -329,7 +397,7 @@ def Vizier_query(self, viz_cat, cat_name, ra, dec, radius, ra_col='RAJ2000', dec def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000', radius='', group=True): """ - Use astroquery to pull in and cross match a catalog with sources in self.catalog + Use astroquery to pull in and cross match a catalog with sources in self.sources Parameters ---------- @@ -341,7 +409,7 @@ def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000' The matching radius """ # Make sure sources have been grouped - if self.catalog.empty: + if self.sources.empty: print('Please run group_sources() before cross matching.') return @@ -351,7 +419,7 @@ def Vizier_xmatch(self, viz_cat, cat_name, ra_col='_RAJ2000', dec_col='_DEJ2000' viz_cat = "vizier:{}".format(viz_cat) # Prep the current catalog as an astropy.QTable - tab = at.Table.from_pandas(self.catalog) + tab = at.Table.from_pandas(self.sources) # Crossmatch with Vizier print("Cross matching {} sources with {} catalog. Please be patient...".format(len(tab), viz_cat)) @@ -413,12 +481,12 @@ def group_sources(self, radius='', plot=False): unique_coords = np.asarray([np.mean(coords[source_ids==id], axis=0) for id in list(set(source_ids))]) # Generate a source catalog - self.catalog = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) - self.catalog['id'] = unique_source_ids - self.catalog[['ra','dec']] = unique_coords - self.catalog['flag'] = [None]*len(unique_source_ids) - # self.catalog['flag'] = ['d{}'.format(i) if i>1 else '' for i in Counter(source_ids).values()] - self.catalog['datasets'] = Counter(source_ids).values() + self.sources = pd.DataFrame(columns=('id','ra','dec','flag','datasets')) + self.sources['id'] = unique_source_ids + self.sources[['ra','dec']] = unique_coords + self.sources['flag'] = [None]*len(unique_source_ids) + # self.sources['flag'] = ['d{}'.format(i) if i>1 else '' for i in Counter(source_ids).values()] + self.sources['datasets'] = Counter(source_ids).values() # Update history self.history += "\n{}: Catalog grouped with radius {} arcsec.".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.xmatch_radius) @@ -495,7 +563,7 @@ def load(self, path): DB = joblib.load(path) # Load the attributes - self.catalog = DB.catalog + self.sources = DB.catalog self.n_sources = DB.n_sources self.name = DB.name self.history = DB.history @@ -535,11 +603,11 @@ def correct_offsets(self, cat_name, truth='ACS'): else: # First, remove any previous catalog correction - self.catalog.loc[self.catalog['cat_name']==cat_name, 'ra_corr'] = self.catalog.loc[self.catalog['cat_name']==cat_name, '_RAJ2000'] - self.catalog.loc[self.catalog['cat_name']==cat_name, 'dec_corr'] = self.catalog.loc[self.catalog['cat_name']==cat_name, '_DEJ2000'] + self.sources.loc[self.sources['cat_name']==cat_name, 'ra_corr'] = self.sources.loc[self.sources['cat_name']==cat_name, '_RAJ2000'] + self.sources.loc[self.sources['cat_name']==cat_name, 'dec_corr'] = self.sources.loc[self.sources['cat_name']==cat_name, '_DEJ2000'] # Copy the catalog - onc_gr = self.catalog.copy() + onc_gr = self.sources.copy() # restrict to one-to-one matches, sort by oncID so that matches are paired o2o_new = onc_gr.loc[(onc_gr['oncflag'].str.contains('o')) & (onc_gr['cat_name'] == cat_name) ,:].sort_values('oncID') @@ -582,8 +650,8 @@ def correct_offsets(self, cat_name, truth='ACS'): # Update the coordinates of the appropriate sources print('Shifting {} sources by {}" in RA and {}" in Dec...'.format(cat_name,mu_ra,mu_dec)) - self.catalog.loc[self.catalog['cat_name']==cat_name, 'ra_corr'] += mu_ra - self.catalog.loc[self.catalog['cat_name']==cat_name, 'dec_corr'] += mu_dec + self.sources.loc[self.sources['cat_name']==cat_name, 'ra_corr'] += mu_ra + self.sources.loc[self.sources['cat_name']==cat_name, 'dec_corr'] += mu_dec # Update history now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -625,7 +693,7 @@ def default_rename_columns(cat_name): defaults = {'2MASS':{'JD':'epoch', 'Qflg':'flags', 'Jmag':'2MASS.J', 'Hmag':'2MASS.H', 'Kmag':'2MASS.Ks', 'e_Jmag':'2MASS.J_unc', 'e_Hmag':'2MASS.H_unc', 'e_Kmag':'2MASS.Ks_unc'}, 'WISE':{'qph':'flags', 'W1mag':'WISE.W1', 'W2mag':'WISE.W2', 'W3mag':'WISE.W3', 'W4mag':'WISE.W4', 'e_W1mag':'WISE.W1_unc', 'e_W2mag':'WISE.W2_unc', 'e_W3mag':'WISE.W3_unc', 'e_W4mag':'WISE.W4_unc'}, 'SDSS':{'ObsDate':'epoch', 'flags':'oflags', 'Q':'flags', 'umag':'SDSS.u', 'gmag':'SDSS.g', 'rmag':'SDSS.r', 'imag':'SDSS.i', 'zmag':'SDSS.z', 'e_umag':'SDSS.u_unc', 'e_gmag':'SDSS.g_unc', 'e_rmag':'SDSS.r_unc', 'e_imag':'SDSS.i_unc', 'e_zmag':'SDSS.z_unc'}, - 'TGAS':{'Epoch':'epoch', 'Plx':'parallax', 'e_Plx':'parallax_unc'}} + 'GAIA':{'Epoch':'epoch', 'Plx':'parallax', 'e_Plx':'parallax_unc', 'Gmag':'Gaia.G', 'e_Gmag':'Gaia.G_unc', 'BPmag':'Gaia.BP', 'e_BPmag':'Gaia.BP_unc', 'RPmag':'Gaia.RP', 'e_RPmag':'Gaia.RP_unc'}} return defaults[cat_name] @@ -646,7 +714,7 @@ def default_column_fill(cat_name): defaults = {'2MASS':{'publication_shortname':'Cutr03', 'telescope_id':2, 'instrument_id':5, 'system_id':2}, 'WISE':{'publication_shortname':'Cutr13', 'telescope_id':3, 'instrument_id':6, 'system_id':2}, 'SDSS':{'publication_shortname':'Alam15', 'telescope_id':6, 'instrument_id':9, 'system_id':2}, - 'TGAS':{'publication_shortname':'Gaia16', 'telescope_id':4, 'instrument_id':7, 'system_id':1}} + 'GAIA':{'publication_shortname':'Gaia18', 'telescope_id':4, 'instrument_id':7, 'system_id':1}} return defaults[cat_name] \ No newline at end of file diff --git a/astrodbkit/astrodb.py b/astrodbkit/astrodb.py index d1ce360..9a7a15c 100755 --- a/astrodbkit/astrodb.py +++ b/astrodbkit/astrodb.py @@ -53,7 +53,7 @@ def create_database(dbpath, schema='', overwrite=True): # Load the schema if given if schema: - os.system("cat {} | sqlite3 {}".format(schema,dbpath)) + os.system("cat {} | sqlite3 {}".format(schema, dbpath)) # Otherwise just make an empty SOURCES table else: @@ -326,8 +326,16 @@ def add_data(self, data, table, delimiter='|', bands='', clean_up=True, rename_c # Rename columns if isinstance(rename_columns,str): rename_columns = astrocat.default_rename_columns(rename_columns) + + try_again = [] for input_name,new_name in rename_columns.items(): - data.rename_column(input_name,new_name) + try: + data.rename_column(input_name,new_name) + except KeyError: + try_again.append(input_name) + + for input_name in try_again: + data.rename_column(input_name,rename_columns[input_name]) # Add column fills if isinstance(column_fill,str): diff --git a/astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb b/astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb new file mode 100644 index 0000000..e05042d --- /dev/null +++ b/astrodbkit/notebooks/.ipynb_checkpoints/using_astrocat-checkpoint.ipynb @@ -0,0 +1,1136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The `astrocat` module\n", + "This notebook will take you through a simple use case for the `astrocat` module. In our example, we will construct a source catalog of targets in the Orion Nebula Cluster from Vizier data." + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "(function(global) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n", + " window._bokeh_onload_callbacks = [];\n", + " window._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "\n", + " \n", + " if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n", + " window._bokeh_timeout = Date.now() + 5000;\n", + " window._bokeh_failed_load = false;\n", + " }\n", + "\n", + " var NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " if (window.Bokeh !== undefined) {\n", + " var el = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n", + " } else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", + " }\n", + " finally {\n", + " delete window._bokeh_onload_callbacks\n", + " }\n", + " console.info(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(js_urls, callback) {\n", + " window._bokeh_onload_callbacks.push(callback);\n", + " if (window._bokeh_is_loading > 0) {\n", + " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " window._bokeh_is_loading = js_urls.length;\n", + " for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " var s = document.createElement('script');\n", + " s.src = url;\n", + " s.async = false;\n", + " s.onreadystatechange = s.onload = function() {\n", + " window._bokeh_is_loading--;\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", + " run_callbacks()\n", + " }\n", + " };\n", + " s.onerror = function() {\n", + " console.warn(\"failed to load library \" + url);\n", + " };\n", + " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", + " }\n", + " };var element = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " if (element == null) {\n", + " console.log(\"Bokeh: ERROR: autoload.js configured with elementid '8265b4a8-0ce0-42a8-88d3-4337a027ee84' but no matching script tag was found. \")\n", + " return false;\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.js\"];\n", + "\n", + " var inline_js = [\n", + " function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\").textContent = \"BokehJS is loading...\";\n", + " },\n", + " function(Bokeh) {\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " \n", + " if ((window.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i](window.Bokeh);\n", + " }if (force === true) {\n", + " display_loaded();\n", + " }} else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!window._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " window._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " var cell = $(document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\")).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + "\n", + " }\n", + "\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(js_urls, function() {\n", + " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(this));" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from astrodbkit import astrocat\n", + "import astropy.coordinates as coord\n", + "import astropy.units as q\n", + "from bokeh.io import output_notebook\n", + "output_notebook()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's initialize the catalog and define our area of interest." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Create empty catalog\n", + "cat = astrocat.Catalog()\n", + "\n", + "# Region of interest (Keep it small for demonstration purposes)\n", + "ra = 83.81775*q.deg\n", + "dec = -5.38788889*q.deg\n", + "cone_radius = 0.05*q.deg" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's grab all the 2MASS sources in this region" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 915 rows from TMASS catalog...\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/246/out', 'TMASS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And take a look at the results, which are stored in a `pandas.DataFrame` as an attribute of the `Catalog` object" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
source_idra_corrdec_corrcatID_rradecerrMajerrMinerrPA...DoptPAoptBmagRmagNoptextKeyscanKeycoaddKeycoaddOpt
0NaN83.807105-5.427688TMASS_10.04118283.807105-5.4276880.100.100...--------0--643191479335244Opt
1NaN83.798154-5.413375TMASS_20.03209583.798154-5.4133750.080.0817...--------0--643191479335244Opt
2NaN83.803910-5.428761TMASS_30.04312983.803910-5.4287610.220.20101...--------0--643191479335244Opt
3NaN83.807116-5.429630TMASS_40.04305983.807116-5.4296300.120.12145...--------0--643191479335244Opt
4NaN83.799794-5.417830TMASS_50.03487083.799794-5.4178300.150.147...--------0--643191479335244Opt
5NaN83.804124-5.400455TMASS_60.01849283.804124-5.4004550.380.3744...--------0--643191479335244Opt
6NaN83.799942-5.413466TMASS_70.03112083.799942-5.4134660.260.26138...--------0--643191479335244Opt
7NaN83.804354-5.422725TMASS_80.03729983.804354-5.4227250.190.19138...--------0--643191479335244Opt
8NaN83.772534-5.404180TMASS_90.04787783.772534-5.4041800.060.060...--------0--643191479335244Opt
9NaN83.772389-5.402932TMASS_100.04760483.772389-5.4029320.060.060...--------0--643191479335244Opt
\n", + "

10 rows × 66 columns

\n", + "
" + ], + "text/plain": [ + " source_id ra_corr dec_corr catID _r ra dec \\\n", + "0 NaN 83.807105 -5.427688 TMASS_1 0.041182 83.807105 -5.427688 \n", + "1 NaN 83.798154 -5.413375 TMASS_2 0.032095 83.798154 -5.413375 \n", + "2 NaN 83.803910 -5.428761 TMASS_3 0.043129 83.803910 -5.428761 \n", + "3 NaN 83.807116 -5.429630 TMASS_4 0.043059 83.807116 -5.429630 \n", + "4 NaN 83.799794 -5.417830 TMASS_5 0.034870 83.799794 -5.417830 \n", + "5 NaN 83.804124 -5.400455 TMASS_6 0.018492 83.804124 -5.400455 \n", + "6 NaN 83.799942 -5.413466 TMASS_7 0.031120 83.799942 -5.413466 \n", + "7 NaN 83.804354 -5.422725 TMASS_8 0.037299 83.804354 -5.422725 \n", + "8 NaN 83.772534 -5.404180 TMASS_9 0.047877 83.772534 -5.404180 \n", + "9 NaN 83.772389 -5.402932 TMASS_10 0.047604 83.772389 -5.402932 \n", + "\n", + " errMaj errMin errPA ... Dopt PAopt Bmag Rmag Nopt extKey scanKey \\\n", + "0 0.10 0.10 0 ... -- -- -- -- 0 -- 64319 \n", + "1 0.08 0.08 17 ... -- -- -- -- 0 -- 64319 \n", + "2 0.22 0.20 101 ... -- -- -- -- 0 -- 64319 \n", + "3 0.12 0.12 145 ... -- -- -- -- 0 -- 64319 \n", + "4 0.15 0.14 7 ... -- -- -- -- 0 -- 64319 \n", + "5 0.38 0.37 44 ... -- -- -- -- 0 -- 64319 \n", + "6 0.26 0.26 138 ... -- -- -- -- 0 -- 64319 \n", + "7 0.19 0.19 138 ... -- -- -- -- 0 -- 64319 \n", + "8 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "9 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "\n", + " coaddKey coadd Opt \n", + "0 1479335 244 Opt \n", + "1 1479335 244 Opt \n", + "2 1479335 244 Opt \n", + "3 1479335 244 Opt \n", + "4 1479335 244 Opt \n", + "5 1479335 244 Opt \n", + "6 1479335 244 Opt \n", + "7 1479335 244 Opt \n", + "8 1479335 244 Opt \n", + "9 1479335 244 Opt \n", + "\n", + "[10 rows x 66 columns]" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.TMASS[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot some stuff for funzies..." + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cat.plot('TMASS', 'Jmag', 'Hmag')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets's get some ALLWISE and HST/ACS data as well." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/328/allwise for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 13 rows from ALLWISE catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ALLWISE']\n", + "Searching J/ApJS/207/10/table5 for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 2097 rows from ACS catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ACS', 'ALLWISE']\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/328/allwise', 'ALLWISE', ra, dec, cone_radius)\n", + "cat.Vizier_query('J/ApJS/207/10/table5', 'ACS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's look at the source catalog, which was generated from the proximity of the data points in ingested catalogs" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idradecflagdatasets
0183.807105-5.427688None1
1283.798143-5.413356None3
2383.803910-5.428761None1
3483.807116-5.429630None1
4583.799794-5.417830None1
5683.804124-5.400455None1
6783.799942-5.413466None1
7883.804354-5.422725None1
8983.772525-5.404179None3
91083.772377-5.402924None3
\n", + "
" + ], + "text/plain": [ + " id ra dec flag datasets\n", + "0 1 83.807105 -5.427688 None 1\n", + "1 2 83.798143 -5.413356 None 3\n", + "2 3 83.803910 -5.428761 None 1\n", + "3 4 83.807116 -5.429630 None 1\n", + "4 5 83.799794 -5.417830 None 1\n", + "5 6 83.804124 -5.400455 None 1\n", + "6 7 83.799942 -5.413466 None 1\n", + "7 8 83.804354 -5.422725 None 1\n", + "8 9 83.772525 -5.404179 None 3\n", + "9 10 83.772377 -5.402924 None 3" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.sources[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And check out the inventory of one of these objects" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Source:\n", + " id ra dec flag datasets\n", + "--- ------------- -------------- ---- --------\n", + " 10 83.7723768889 -5.40292362963 -- 3\n", + "None\n", + "\n", + "TMASS:\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- --------- --------- -------- -------- ... ------- -------- ----- ---\n", + " 10 83.772389 -5.402932 TMASS_10 0.047604 ... 64319 1479335 244 Opt\n", + "\n", + "ACS:\n", + "source_id ra_corr dec_corr ... sky850s e_sky850s Obs \n", + "--------- ------------- -------------- ... ------- --------- -------------------\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-05T06:41:24\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-06T04:26:25\n" + ] + } + ], + "source": [ + "cat.inventory(source_id=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we can see that this source was detected in the 2MASS and ACS catalogs but not in the ALLWISE catalog.\n", + "\n", + "We can also manually add sources and have them cross-matched in the given catalogs. Let's do Vega just for this demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from TMASS catalog...\n", + "Searching J/ApJS/207/10/table5 for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "No data found in J/ApJS/207/10/table5 within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg).\n", + "Searching II/328/allwise for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from ALLWISE catalog...\n" + ] + } + ], + "source": [ + "# RA and Dec of new source\n", + "new_ra = 279.23473479*q.deg\n", + "new_dec = 38.78368896*q.deg\n", + "cat.add_source(new_ra, new_dec)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Take a look!" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " TMASS :\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- ---------- --------- --------- ----- ... ------- -------- ----- ---\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt\n", + "\n", + " source :\n", + " id ra dec flag datasets\n", + "---- ------------ ----------- ---- --------\n", + "1307 279.23473479 38.78368896 0\n", + "\n", + " ALLWISE :\n", + "source_id ra_corr dec_corr catID _r ... covW3 covW4 _2Mkey d2M _2M\n", + "--------- --------- ---------- ---------- ----- ... ----- ------ ------ --- ---\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 0.889 34.399 -- -- 2M\n" + ] + } + ], + "source": [ + "# The new source has the highest source_id\n", + "cat.inventory(source_id=cat.n_sources)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'TMASS': \n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt \n", + " int64 float64 float64 str9 float64 ... int64 int64 int64 str3\n", + "--------- ---------- --------- --------- ------- ... ------- -------- ----- ----\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt, 'source':
\n", + " id ra dec flag datasets\n", + "int64 float64 float64 str1 int64 \n", + "----- ------------ ----------- ---- --------\n", + " 1307 279.23473479 38.78368896 0, 'ALLWISE':
\n", + "source_id ra_corr dec_corr catID _r ... covW4 _2Mkey d2M _2M \n", + " int64 float64 float64 str10 float64 ... object object object str2\n", + "--------- --------- ---------- ---------- ------- ... ------ ------ ------ ----\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 34.399 -- -- 2M}\n" + ] + } + ], + "source": [ + "# Or return the data as a dictionary\n", + "inventory = cat.inventory(source_id=cat.n_sources, return_inventory=True)\n", + "print(inventory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Astroconda", + "language": "python", + "name": "astroconda_kernel" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/astrodbkit/notebooks/using_astrocat.ipynb b/astrodbkit/notebooks/using_astrocat.ipynb new file mode 100644 index 0000000..b0646d4 --- /dev/null +++ b/astrodbkit/notebooks/using_astrocat.ipynb @@ -0,0 +1,1136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The `astrocat` module\n", + "This notebook will take you through a simple use case for the `astrocat` module. In our example, we will construct a source catalog of targets in the Orion Nebula Cluster from Vizier data." + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "(function(global) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n", + " window._bokeh_onload_callbacks = [];\n", + " window._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "\n", + " \n", + " if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n", + " window._bokeh_timeout = Date.now() + 5000;\n", + " window._bokeh_failed_load = false;\n", + " }\n", + "\n", + " var NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " if (window.Bokeh !== undefined) {\n", + " var el = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n", + " } else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", + " }\n", + " finally {\n", + " delete window._bokeh_onload_callbacks\n", + " }\n", + " console.info(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(js_urls, callback) {\n", + " window._bokeh_onload_callbacks.push(callback);\n", + " if (window._bokeh_is_loading > 0) {\n", + " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " window._bokeh_is_loading = js_urls.length;\n", + " for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " var s = document.createElement('script');\n", + " s.src = url;\n", + " s.async = false;\n", + " s.onreadystatechange = s.onload = function() {\n", + " window._bokeh_is_loading--;\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", + " run_callbacks()\n", + " }\n", + " };\n", + " s.onerror = function() {\n", + " console.warn(\"failed to load library \" + url);\n", + " };\n", + " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", + " }\n", + " };var element = document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\");\n", + " if (element == null) {\n", + " console.log(\"Bokeh: ERROR: autoload.js configured with elementid '8265b4a8-0ce0-42a8-88d3-4337a027ee84' but no matching script tag was found. \")\n", + " return false;\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.js\"];\n", + "\n", + " var inline_js = [\n", + " function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " },\n", + " \n", + " function(Bokeh) {\n", + " \n", + " document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\").textContent = \"BokehJS is loading...\";\n", + " },\n", + " function(Bokeh) {\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.6.min.css\");\n", + " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.6.min.css\");\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " \n", + " if ((window.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i](window.Bokeh);\n", + " }if (force === true) {\n", + " display_loaded();\n", + " }} else if (Date.now() < window._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!window._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " window._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " var cell = $(document.getElementById(\"8265b4a8-0ce0-42a8-88d3-4337a027ee84\")).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + "\n", + " }\n", + "\n", + " if (window._bokeh_is_loading === 0) {\n", + " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(js_urls, function() {\n", + " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(this));" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from astrodbkit import astrocat\n", + "import astropy.coordinates as coord\n", + "import astropy.units as q\n", + "from bokeh.io import output_notebook\n", + "output_notebook()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's initialize the catalog and define our area of interest." + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Create empty catalog\n", + "cat = astrocat.Catalog()\n", + "\n", + "# Region of interest (Keep it small for demonstration purposes)\n", + "ra = 83.81775*q.deg\n", + "dec = -5.38788889*q.deg\n", + "cone_radius = 0.05*q.deg" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's grab all the 2MASS sources in this region" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 915 rows from TMASS catalog...\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/246/out', 'TMASS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And take a look at the results, which are stored in a `pandas.DataFrame` as an attribute of the `Catalog` object" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
source_idra_corrdec_corrcatID_rradecerrMajerrMinerrPA...DoptPAoptBmagRmagNoptextKeyscanKeycoaddKeycoaddOpt
0NaN83.807105-5.427688TMASS_10.04118283.807105-5.4276880.100.100...--------0--643191479335244Opt
1NaN83.798154-5.413375TMASS_20.03209583.798154-5.4133750.080.0817...--------0--643191479335244Opt
2NaN83.803910-5.428761TMASS_30.04312983.803910-5.4287610.220.20101...--------0--643191479335244Opt
3NaN83.807116-5.429630TMASS_40.04305983.807116-5.4296300.120.12145...--------0--643191479335244Opt
4NaN83.799794-5.417830TMASS_50.03487083.799794-5.4178300.150.147...--------0--643191479335244Opt
5NaN83.804124-5.400455TMASS_60.01849283.804124-5.4004550.380.3744...--------0--643191479335244Opt
6NaN83.799942-5.413466TMASS_70.03112083.799942-5.4134660.260.26138...--------0--643191479335244Opt
7NaN83.804354-5.422725TMASS_80.03729983.804354-5.4227250.190.19138...--------0--643191479335244Opt
8NaN83.772534-5.404180TMASS_90.04787783.772534-5.4041800.060.060...--------0--643191479335244Opt
9NaN83.772389-5.402932TMASS_100.04760483.772389-5.4029320.060.060...--------0--643191479335244Opt
\n", + "

10 rows × 66 columns

\n", + "" + ], + "text/plain": [ + " source_id ra_corr dec_corr catID _r ra dec \\\n", + "0 NaN 83.807105 -5.427688 TMASS_1 0.041182 83.807105 -5.427688 \n", + "1 NaN 83.798154 -5.413375 TMASS_2 0.032095 83.798154 -5.413375 \n", + "2 NaN 83.803910 -5.428761 TMASS_3 0.043129 83.803910 -5.428761 \n", + "3 NaN 83.807116 -5.429630 TMASS_4 0.043059 83.807116 -5.429630 \n", + "4 NaN 83.799794 -5.417830 TMASS_5 0.034870 83.799794 -5.417830 \n", + "5 NaN 83.804124 -5.400455 TMASS_6 0.018492 83.804124 -5.400455 \n", + "6 NaN 83.799942 -5.413466 TMASS_7 0.031120 83.799942 -5.413466 \n", + "7 NaN 83.804354 -5.422725 TMASS_8 0.037299 83.804354 -5.422725 \n", + "8 NaN 83.772534 -5.404180 TMASS_9 0.047877 83.772534 -5.404180 \n", + "9 NaN 83.772389 -5.402932 TMASS_10 0.047604 83.772389 -5.402932 \n", + "\n", + " errMaj errMin errPA ... Dopt PAopt Bmag Rmag Nopt extKey scanKey \\\n", + "0 0.10 0.10 0 ... -- -- -- -- 0 -- 64319 \n", + "1 0.08 0.08 17 ... -- -- -- -- 0 -- 64319 \n", + "2 0.22 0.20 101 ... -- -- -- -- 0 -- 64319 \n", + "3 0.12 0.12 145 ... -- -- -- -- 0 -- 64319 \n", + "4 0.15 0.14 7 ... -- -- -- -- 0 -- 64319 \n", + "5 0.38 0.37 44 ... -- -- -- -- 0 -- 64319 \n", + "6 0.26 0.26 138 ... -- -- -- -- 0 -- 64319 \n", + "7 0.19 0.19 138 ... -- -- -- -- 0 -- 64319 \n", + "8 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "9 0.06 0.06 0 ... -- -- -- -- 0 -- 64319 \n", + "\n", + " coaddKey coadd Opt \n", + "0 1479335 244 Opt \n", + "1 1479335 244 Opt \n", + "2 1479335 244 Opt \n", + "3 1479335 244 Opt \n", + "4 1479335 244 Opt \n", + "5 1479335 244 Opt \n", + "6 1479335 244 Opt \n", + "7 1479335 244 Opt \n", + "8 1479335 244 Opt \n", + "9 1479335 244 Opt \n", + "\n", + "[10 rows x 66 columns]" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.TMASS[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot some stuff for funzies..." + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cat.plot('TMASS', 'Jmag', 'Hmag')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now lets's get some ALLWISE and HST/ACS data as well." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/328/allwise for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 13 rows from ALLWISE catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ALLWISE']\n", + "Searching J/ApJS/207/10/table5 for sources within 0.05 deg of (83.81775 deg, -5.38788889 deg). Please be patient...\n", + "Ingesting 2097 rows from ACS catalog...\n", + "Grouping sources from the following catalogs: ['TMASS', 'ACS', 'ALLWISE']\n" + ] + } + ], + "source": [ + "cat.Vizier_query('II/328/allwise', 'ALLWISE', ra, dec, cone_radius)\n", + "cat.Vizier_query('J/ApJS/207/10/table5', 'ACS', ra, dec, cone_radius)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's look at the source catalog, which was generated from the proximity of the data points in ingested catalogs" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idradecflagdatasets
0183.807105-5.427688None1
1283.798143-5.413356None3
2383.803910-5.428761None1
3483.807116-5.429630None1
4583.799794-5.417830None1
5683.804124-5.400455None1
6783.799942-5.413466None1
7883.804354-5.422725None1
8983.772525-5.404179None3
91083.772377-5.402924None3
\n", + "
" + ], + "text/plain": [ + " id ra dec flag datasets\n", + "0 1 83.807105 -5.427688 None 1\n", + "1 2 83.798143 -5.413356 None 3\n", + "2 3 83.803910 -5.428761 None 1\n", + "3 4 83.807116 -5.429630 None 1\n", + "4 5 83.799794 -5.417830 None 1\n", + "5 6 83.804124 -5.400455 None 1\n", + "6 7 83.799942 -5.413466 None 1\n", + "7 8 83.804354 -5.422725 None 1\n", + "8 9 83.772525 -5.404179 None 3\n", + "9 10 83.772377 -5.402924 None 3" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat.sources[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And check out the inventory of one of these objects" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Source:\n", + " id ra dec flag datasets\n", + "--- ------------- -------------- ---- --------\n", + " 10 83.7723768889 -5.40292362963 -- 3\n", + "None\n", + "\n", + "TMASS:\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- --------- --------- -------- -------- ... ------- -------- ----- ---\n", + " 10 83.772389 -5.402932 TMASS_10 0.047604 ... 64319 1479335 244 Opt\n", + "\n", + "ACS:\n", + "source_id ra_corr dec_corr ... sky850s e_sky850s Obs \n", + "--------- ------------- -------------- ... ------- --------- -------------------\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-05T06:41:24\n", + " 10 83.7723708333 -5.40291944444 ... -- -- 2005-04-06T04:26:25\n" + ] + } + ], + "source": [ + "cat.inventory(source_id=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we can see that this source was detected in the 2MASS and ACS catalogs but not in the ALLWISE catalog.\n", + "\n", + "We can also manually add sources and have them cross-matched in the given catalogs. Let's do Vega just for this demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Searching II/246/out for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from TMASS catalog...\n", + "Searching J/ApJS/207/10/table5 for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "No data found in J/ApJS/207/10/table5 within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg).\n", + "Searching II/328/allwise for sources within 10.0 arcsec of (279.23473479 deg, 38.78368896 deg). Please be patient...\n", + "Ingesting 1 rows from ALLWISE catalog...\n" + ] + } + ], + "source": [ + "# RA and Dec of new source\n", + "new_ra = 279.23473479*q.deg\n", + "new_dec = 38.78368896*q.deg\n", + "cat.add_source(new_ra, new_dec)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Take a look!" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " TMASS :\n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt\n", + "--------- ---------- --------- --------- ----- ... ------- -------- ----- ---\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt\n", + "\n", + " source :\n", + " id ra dec flag datasets\n", + "---- ------------ ----------- ---- --------\n", + "1307 279.23473479 38.78368896 0\n", + "\n", + " ALLWISE :\n", + "source_id ra_corr dec_corr catID _r ... covW3 covW4 _2Mkey d2M _2M\n", + "--------- --------- ---------- ---------- ----- ... ----- ------ ------ --- ---\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 0.889 34.399 -- -- 2M\n" + ] + } + ], + "source": [ + "# The new source has the highest source_id\n", + "cat.inventory(source_id=cat.n_sources)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'TMASS': \n", + "source_id ra_corr dec_corr catID _r ... scanKey coaddKey coadd Opt \n", + " int64 float64 float64 str9 float64 ... int64 int64 int64 str3\n", + "--------- ---------- --------- --------- ------- ... ------- -------- ----- ----\n", + " 1307 279.234723 38.783691 TMASS_916 0.049 ... 33663 774238 138 Opt, 'source':
\n", + " id ra dec flag datasets\n", + "int64 float64 float64 str1 int64 \n", + "----- ------------ ----------- ---- --------\n", + " 1307 279.23473479 38.78368896 0, 'ALLWISE':
\n", + "source_id ra_corr dec_corr catID _r ... covW4 _2Mkey d2M _2M \n", + " int64 float64 float64 str10 float64 ... object object object str2\n", + "--------- --------- ---------- ---------- ------- ... ------ ------ ------ ----\n", + " 1307 279.23549 38.7845586 ALLWISE_14 3.755 ... 34.399 -- -- 2M}\n" + ] + } + ], + "source": [ + "# Or return the data as a dictionary\n", + "inventory = cat.inventory(source_id=cat.n_sources, return_inventory=True)\n", + "print(inventory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Astroconda", + "language": "python", + "name": "astroconda_kernel" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/requirements.txt b/requirements.txt index f672288..7c20a7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ matplotlib pandas requests astroquery +bokeh diff --git a/setup.py b/setup.py index e0b6aa7..cf3f219 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,6 @@ ], keywords='astrophysics', packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - install_requires=['numpy','astropy','matplotlib','requests','pandas','astroquery'], + install_requires=['numpy','astropy','matplotlib','requests','pandas','astroquery','bokeh'], ) \ No newline at end of file