diff --git a/README.md b/README.md index fa333aa..147972a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ adler -s 8268570668335894776 ``` This will currently print some phase curve information to the terminal. +One can also read from a local database, for example: + +``` +adler -s 8268570668335894776 -i tests/data/testing_database.db +``` + Notes: 1) The single quotes around `'[dev]'` may not be required for your operating system. 2) `pre-commit install` will initialize pre-commit for this local repository, so diff --git a/notebooks/adler_demo/adler_demo.ipynb b/notebooks/adler_demo/adler_demo.ipynb new file mode 100644 index 0000000..5694d63 --- /dev/null +++ b/notebooks/adler_demo/adler_demo.ipynb @@ -0,0 +1,590 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b7d20301-a262-4820-bb3a-7fb43e2556d6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:14.129116Z", + "iopub.status.busy": "2024-05-16T16:38:14.128827Z", + "iopub.status.idle": "2024-05-16T16:38:15.583700Z", + "shell.execute_reply": "2024-05-16T16:38:15.582986Z", + "shell.execute_reply.started": "2024-05-16T16:38:14.129097Z" + } + }, + "outputs": [], + "source": [ + "from adler.dataclasses.AdlerPlanetoid import AdlerPlanetoid\n", + "from adler.science.PhaseCurve import PhaseCurve\n", + "from adler.utilities.plotting_utilities import plot_errorbar\n", + "import adler.utilities.science_utilities as sci_utils\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.gridspec as gridspec\n", + "import astropy.units as u" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2de26df9-d4e8-465e-9698-8db0219ebcdd", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:15.585255Z", + "iopub.status.busy": "2024-05-16T16:38:15.584619Z", + "iopub.status.idle": "2024-05-16T16:38:15.588002Z", + "shell.execute_reply": "2024-05-16T16:38:15.587337Z", + "shell.execute_reply.started": "2024-05-16T16:38:15.585234Z" + } + }, + "outputs": [], + "source": [ + "# notebook to show adler searching for outlying photometry\n", + "# we could simply use the adler api to do this, but let's demo the CLI that would be running on a server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a82c036c-e922-4316-9cd6-74119bfe2357", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:15.589009Z", + "iopub.status.busy": "2024-05-16T16:38:15.588798Z", + "iopub.status.idle": "2024-05-16T16:38:15.603718Z", + "shell.execute_reply": "2024-05-16T16:38:15.602882Z", + "shell.execute_reply.started": "2024-05-16T16:38:15.588995Z" + } + }, + "outputs": [], + "source": [ + "# ssObjectId of object to analyse\n", + "ssoid1 = \"6098332225018\" # good test object\n", + "ssoid2 = \"6098332225018000\" # fake outburst object\n", + "filt = \"r\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3715f85a-1939-4ebb-894b-b262ea84865c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:15.604948Z", + "iopub.status.busy": "2024-05-16T16:38:15.604643Z", + "iopub.status.idle": "2024-05-16T16:38:15.765721Z", + "shell.execute_reply": "2024-05-16T16:38:15.765102Z", + "shell.execute_reply.started": "2024-05-16T16:38:15.604930Z" + } + }, + "outputs": [], + "source": [ + "# here we use an offline SQL database which contains the observations of the sso\n", + "fname = \"/home/jrob/lsst-adler/notebooks/gen_test_data/adler_demo_testing_database.db\"\n", + "planetoid1 = AdlerPlanetoid.construct_from_SQL(ssoid1, sql_filename=fname)\n", + "planetoid2 = AdlerPlanetoid.construct_from_SQL(ssoid2, sql_filename=fname)\n", + "\n", + "# or query DP0.3 directly when on RSP\n", + "# planetoid1 = AdlerPlanetoid.construct_from_RSP(ssoid1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43d4f906-c055-4dd6-abe5-b559b3992023", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:15.768171Z", + "iopub.status.busy": "2024-05-16T16:38:15.767880Z", + "iopub.status.idle": "2024-05-16T16:38:15.772145Z", + "shell.execute_reply": "2024-05-16T16:38:15.771604Z", + "shell.execute_reply.started": "2024-05-16T16:38:15.768154Z" + } + }, + "outputs": [], + "source": [ + "planetoid1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c5be63b-295a-4ec0-959d-bc27e2e50ca1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:15.773095Z", + "iopub.status.busy": "2024-05-16T16:38:15.772893Z", + "iopub.status.idle": "2024-05-16T16:38:15.808371Z", + "shell.execute_reply": "2024-05-16T16:38:15.807616Z", + "shell.execute_reply.started": "2024-05-16T16:38:15.773080Z" + } + }, + "outputs": [], + "source": [ + "planetoid1.__dict__" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8350af69-b369-4f96-b46d-f00865504505", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:15.809631Z", + "iopub.status.busy": "2024-05-16T16:38:15.809299Z", + "iopub.status.idle": "2024-05-16T16:38:16.363874Z", + "shell.execute_reply": "2024-05-16T16:38:16.363226Z", + "shell.execute_reply.started": "2024-05-16T16:38:15.809611Z" + } + }, + "outputs": [], + "source": [ + "fig = plot_errorbar(planetoid1, filt_list=[\"r\"], x_plot=\"midPointMjdTai\", y_plot=\"reduced_mag\")\n", + "fig = plot_errorbar(planetoid1, filt_list=[\"r\"], x_plot=\"phaseAngle\", y_plot=\"reduced_mag\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b334d23a-792c-42a3-a396-533987b08bee", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.365028Z", + "iopub.status.busy": "2024-05-16T16:38:16.364799Z", + "iopub.status.idle": "2024-05-16T16:38:16.830194Z", + "shell.execute_reply": "2024-05-16T16:38:16.829613Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.365012Z" + } + }, + "outputs": [], + "source": [ + "fig = plot_errorbar(planetoid2, filt_list=[\"r\"], x_plot=\"midPointMjdTai\", y_plot=\"reduced_mag\")\n", + "fig = plot_errorbar(planetoid2, filt_list=[\"r\"], x_plot=\"phaseAngle\", y_plot=\"reduced_mag\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "598c6098-0ddf-470f-acc8-e1962d9abbb7", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.831659Z", + "iopub.status.busy": "2024-05-16T16:38:16.831084Z", + "iopub.status.idle": "2024-05-16T16:38:16.834249Z", + "shell.execute_reply": "2024-05-16T16:38:16.833691Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.831640Z" + } + }, + "outputs": [], + "source": [ + "# inspect observations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c31d37ac-e020-4e1b-a47d-842d4d2f7e53", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.835361Z", + "iopub.status.busy": "2024-05-16T16:38:16.835001Z", + "iopub.status.idle": "2024-05-16T16:38:16.849517Z", + "shell.execute_reply": "2024-05-16T16:38:16.848946Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.835345Z" + } + }, + "outputs": [], + "source": [ + "obs = planetoid1.observations_in_filter(filt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08860ab8-5b1b-49b5-9390-f379b5429b51", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.850788Z", + "iopub.status.busy": "2024-05-16T16:38:16.850311Z", + "iopub.status.idle": "2024-05-16T16:38:16.864473Z", + "shell.execute_reply": "2024-05-16T16:38:16.863848Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.850770Z" + } + }, + "outputs": [], + "source": [ + "df_obs = pd.DataFrame(obs.__dict__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46852c0e-d4c4-4e2c-9c64-f8497898c3a4", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.865419Z", + "iopub.status.busy": "2024-05-16T16:38:16.865219Z", + "iopub.status.idle": "2024-05-16T16:38:16.894396Z", + "shell.execute_reply": "2024-05-16T16:38:16.893776Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.865405Z" + } + }, + "outputs": [], + "source": [ + "df_obs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1a8c2c3-0639-4a05-80ff-ae4231bdaf4a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.895808Z", + "iopub.status.busy": "2024-05-16T16:38:16.895268Z", + "iopub.status.idle": "2024-05-16T16:38:16.900887Z", + "shell.execute_reply": "2024-05-16T16:38:16.900322Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.895787Z" + } + }, + "outputs": [], + "source": [ + "tmin = np.amin(np.floor(df_obs[\"midPointMjdTai\"])) # mjd\n", + "tmax = np.amax(np.floor(df_obs[\"midPointMjdTai\"])) + 1 # mjd\n", + "tmin, tmax" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a50e7a2e-a16f-4d08-b096-d891a720bb2f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:16.902113Z", + "iopub.status.busy": "2024-05-16T16:38:16.901692Z", + "iopub.status.idle": "2024-05-16T16:38:17.097053Z", + "shell.execute_reply": "2024-05-16T16:38:17.096408Z", + "shell.execute_reply.started": "2024-05-16T16:38:16.902095Z" + } + }, + "outputs": [], + "source": [ + "# cumulative data in filter\n", + "x_plot = \"midPointMjdTai\"\n", + "df_plot = df_obs.sort_values(x_plot)\n", + "\n", + "fig = plt.figure()\n", + "gs = gridspec.GridSpec(1, 1)\n", + "ax1 = plt.subplot(gs[0, 0])\n", + "\n", + "bins = np.arange(tmin, tmax + 1)\n", + "\n", + "values, base = np.histogram(df_plot[x_plot], bins=bins)\n", + "cumulative = np.cumsum(values)\n", + "ax1.plot(base[:-1] - base[0], cumulative, label=filt)\n", + "\n", + "data_mask = np.diff(cumulative) > 0\n", + "data_nights = base[1:-1][data_mask]\n", + "N_data = cumulative[1:][data_mask]\n", + "\n", + "ax1.scatter(data_nights - data_nights[0], N_data)\n", + "\n", + "ax1.set_xlabel(x_plot)\n", + "ax1.set_ylabel(\"number\")\n", + "ax1.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c2bcbd8-702a-468c-a96c-b1d2ee91e9ea", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:17.098165Z", + "iopub.status.busy": "2024-05-16T16:38:17.097942Z", + "iopub.status.idle": "2024-05-16T16:38:17.102814Z", + "shell.execute_reply": "2024-05-16T16:38:17.102289Z", + "shell.execute_reply.started": "2024-05-16T16:38:17.098148Z" + } + }, + "outputs": [], + "source": [ + "# number of data points per night of new data\n", + "np.diff(N_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c32de0b-71b4-459c-8a64-d181df4392cb", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:17.103777Z", + "iopub.status.busy": "2024-05-16T16:38:17.103554Z", + "iopub.status.idle": "2024-05-16T16:38:17.119164Z", + "shell.execute_reply": "2024-05-16T16:38:17.118612Z", + "shell.execute_reply.started": "2024-05-16T16:38:17.103761Z" + } + }, + "outputs": [], + "source": [ + "# nights when new data arrives\n", + "data_nights" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80108e40-0417-4b82-b73f-4cdf872b4a4c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:17.120143Z", + "iopub.status.busy": "2024-05-16T16:38:17.119942Z", + "iopub.status.idle": "2024-05-16T16:38:17.132937Z", + "shell.execute_reply": "2024-05-16T16:38:17.132315Z", + "shell.execute_reply.started": "2024-05-16T16:38:17.120128Z" + } + }, + "outputs": [], + "source": [ + "# write commands to simulate adler cli searching for phase curve outliers on incoming data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78e209fa-7cd3-4b29-a14c-28ddafd1f813", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T16:38:17.134002Z", + "iopub.status.busy": "2024-05-16T16:38:17.133785Z", + "iopub.status.idle": "2024-05-16T16:38:17.147051Z", + "shell.execute_reply": "2024-05-16T16:38:17.146431Z", + "shell.execute_reply.started": "2024-05-16T16:38:17.133987Z" + } + }, + "outputs": [], + "source": [ + "# cmd_list = []\n", + "# outpath = \"~/lsst-adler/logging\"\n", + "# cmd = \"adler -s {} -f {} -o {}\".format(ssoid2,filt,outpath)\n", + "\n", + "# for t0 in data_nights:\n", + "# t1 = t0+1\n", + "\n", + "# mask = (df_obs[\"midPointMjdTai\"] t0) &\n", + "# (diatable[\"midPointMjdTai\"] < t1))\n", + "\n", + "# diatable.loc[mask,\"mag\"] += mag_shift" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0433626a-057a-45c2-8cd1-68f3de4022b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.380986Z", + "iopub.status.busy": "2024-05-16T15:51:17.380732Z", + "iopub.status.idle": "2024-05-16T15:51:17.410063Z", + "shell.execute_reply": "2024-05-16T15:51:17.409487Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.380971Z" + } + }, + "outputs": [], + "source": [ + "diatable" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4a41ba6-b8bc-4a24-9e6d-e69d0dc19866", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.411036Z", + "iopub.status.busy": "2024-05-16T15:51:17.410846Z", + "iopub.status.idle": "2024-05-16T15:51:17.496166Z", + "shell.execute_reply": "2024-05-16T15:51:17.495588Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.411022Z" + } + }, + "outputs": [], + "source": [ + "# diatable.to_sql(\"diaSource\", con=cnx, if_exists=\"append\", index=False)\n", + "diatable.to_sql(\"diaSource\", con=cnx, if_exists=\"replace\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aaab53fe-5391-42f4-9458-c80f8201626f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.497155Z", + "iopub.status.busy": "2024-05-16T15:51:17.496952Z", + "iopub.status.idle": "2024-05-16T15:51:17.566910Z", + "shell.execute_reply": "2024-05-16T15:51:17.566241Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.497140Z" + } + }, + "outputs": [], + "source": [ + "# sssource_table.to_sql(\"ssSource\", con=cnx, if_exists=\"append\", index=False)\n", + "sssource_table.to_sql(\"ssSource\", con=cnx, if_exists=\"replace\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00e59db2-d3d5-47ba-b166-e9baa38a18ca", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.567972Z", + "iopub.status.busy": "2024-05-16T15:51:17.567727Z", + "iopub.status.idle": "2024-05-16T15:51:17.620969Z", + "shell.execute_reply": "2024-05-16T15:51:17.620385Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.567954Z" + } + }, + "outputs": [], + "source": [ + "# ssobject_table.to_sql(\"ssObject\", con=cnx, if_exists=\"append\", index=False)\n", + "ssobject_table.to_sql(\"ssObject\", con=cnx, if_exists=\"replace\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ef861b6-a2f7-48d3-9494-775525288635", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.621931Z", + "iopub.status.busy": "2024-05-16T15:51:17.621730Z", + "iopub.status.idle": "2024-05-16T15:51:17.670994Z", + "shell.execute_reply": "2024-05-16T15:51:17.670462Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.621916Z" + } + }, + "outputs": [], + "source": [ + "# mpcorb_table.to_sql(\"MPCORB\", con=cnx, if_exists=\"append\", index=False)\n", + "mpcorb_table.to_sql(\"MPCORB\", con=cnx, if_exists=\"replace\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbef0ea3-614e-46f1-9ef6-dd8373fe51c6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.671900Z", + "iopub.status.busy": "2024-05-16T15:51:17.671714Z", + "iopub.status.idle": "2024-05-16T15:51:17.703052Z", + "shell.execute_reply": "2024-05-16T15:51:17.702557Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.671885Z" + } + }, + "outputs": [], + "source": [ + "_diatable = diatable.copy()[diatable[\"ssObjectId\"] == test_id]\n", + "_diatable[\"ssObjectId\"] = fake_id\n", + "_diatable[\"diaSourceId\"] = _diatable[\"diaSourceId\"] * 1000\n", + "\n", + "# add an outburst\n", + "t0 = 63100\n", + "t1 = 63600\n", + "mag_shift = -1.5\n", + "\n", + "mask = (\n", + " (_diatable[\"ssObjectId\"] == fake_id)\n", + " & (_diatable[\"midPointMjdTai\"] > t0)\n", + " & (_diatable[\"midPointMjdTai\"] < t1)\n", + ")\n", + "\n", + "_diatable.loc[mask, \"mag\"] += mag_shift\n", + "\n", + "_diatable.to_sql(\"diaSource\", con=cnx, if_exists=\"append\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3bb4edd-844f-436c-acec-cf75075a216e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.703989Z", + "iopub.status.busy": "2024-05-16T15:51:17.703796Z", + "iopub.status.idle": "2024-05-16T15:51:17.724554Z", + "shell.execute_reply": "2024-05-16T15:51:17.724027Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.703975Z" + } + }, + "outputs": [], + "source": [ + "_diatable[diatable[\"band\"] == \"r\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53f27516-1c24-4567-b1f6-11b93cde07e8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.725408Z", + "iopub.status.busy": "2024-05-16T15:51:17.725225Z", + "iopub.status.idle": "2024-05-16T15:51:17.788111Z", + "shell.execute_reply": "2024-05-16T15:51:17.787591Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.725393Z" + } + }, + "outputs": [], + "source": [ + "_sssource_table = sssource_table.copy()[sssource_table[\"ssObjectId\"] == test_id]\n", + "_sssource_table[\"ssObjectId\"] = fake_id\n", + "_sssource_table[\"diaSourceId\"] = _sssource_table[\"diaSourceId\"] * 1000\n", + "_sssource_table.to_sql(\"ssSource\", con=cnx, if_exists=\"append\", index=False)\n", + "\n", + "_ssobject_table = ssobject_table.copy()[ssobject_table[\"ssObjectId\"] == test_id]\n", + "_ssobject_table[\"ssObjectId\"] = fake_id\n", + "_ssobject_table.to_sql(\"ssObject\", con=cnx, if_exists=\"append\", index=False)\n", + "\n", + "_mpcorb_table = mpcorb_table.copy()[mpcorb_table[\"ssObjectId\"] == test_id]\n", + "_mpcorb_table[\"ssObjectId\"] = fake_id\n", + "_mpcorb_table.to_sql(\"MPCORB\", con=cnx, if_exists=\"append\", index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5935a89f-ddae-4587-9546-0dce6f91789c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.788992Z", + "iopub.status.busy": "2024-05-16T15:51:17.788810Z", + "iopub.status.idle": "2024-05-16T15:51:17.792091Z", + "shell.execute_reply": "2024-05-16T15:51:17.791606Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.788978Z" + } + }, + "outputs": [], + "source": [ + "cnx.close()" + ] + }, + { + "cell_type": "markdown", + "id": "b62d7ab4-3ee7-46b2-9d30-f0b740c88068", + "metadata": {}, + "source": [ + "Testing everything went correctly..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a53c503-3a1a-44f3-b8c2-0aa5dde7a397", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.792894Z", + "iopub.status.busy": "2024-05-16T15:51:17.792718Z", + "iopub.status.idle": "2024-05-16T15:51:17.804834Z", + "shell.execute_reply": "2024-05-16T15:51:17.804254Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.792880Z" + } + }, + "outputs": [], + "source": [ + "cnx = sqlite3.connect(db_fname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06b38c09-3fe4-4eb9-811b-37a97052b0c8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.805587Z", + "iopub.status.busy": "2024-05-16T15:51:17.805408Z", + "iopub.status.idle": "2024-05-16T15:51:17.816695Z", + "shell.execute_reply": "2024-05-16T15:51:17.816120Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.805560Z" + } + }, + "outputs": [], + "source": [ + "# example_query = f\"\"\"\n", + "# SELECT\n", + "# ssObject.ssObjectId, mag, magErr, band, midpointMjdTai, ra, dec, phaseAngle,\n", + "# topocentricDist, heliocentricDist\n", + "# FROM\n", + "# ssObject\n", + "# JOIN diaSource ON ssObject.ssObjectId = diaSource.ssObjectId\n", + "# JOIN ssSource ON diaSource.diaSourceId = ssSource.diaSourceId\n", + "# WHERE\n", + "# ssObject.ssObjectId = {ssoid}\n", + "# \"\"\"\n", + "# example_query = f\"\"\"\n", + "# SELECT\n", + "# ssObject.ssObjectId, mag, magErr, band, midpointMjdTai, ra, dec, phaseAngle,\n", + "# topocentricDist, heliocentricDist\n", + "# FROM\n", + "# ssObject\n", + "# JOIN diaSource ON ssObject.ssObjectId = diaSource.ssObjectId\n", + "# JOIN ssSource ON diaSource.diaSourceId = ssSource.diaSourceId\n", + "# WHERE\n", + "# ssObject.ssObjectId in {ssoid_list}\n", + "# \"\"\"\n", + "\n", + "example_query = f\"\"\"\n", + " SELECT\n", + " ssObject.ssObjectId, ssSource.diaSourceId, mag, magErr, band, midpointMjdTai, ra, dec, phaseAngle,\n", + " topocentricDist, heliocentricDist\n", + " FROM\n", + " ssObject\n", + " JOIN diaSource ON ssObject.ssObjectId = diaSource.ssObjectId\n", + " JOIN ssSource ON diaSource.diaSourceId = ssSource.diaSourceId\n", + " WHERE\n", + " ssObject.ssObjectId = {fake_id} and band = 'r'\n", + " \"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53b44a8b-0aed-4d90-8397-aed7c6ba4514", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.817506Z", + "iopub.status.busy": "2024-05-16T15:51:17.817322Z", + "iopub.status.idle": "2024-05-16T15:51:17.849053Z", + "shell.execute_reply": "2024-05-16T15:51:17.848475Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.817493Z" + } + }, + "outputs": [], + "source": [ + "pd.read_sql_query(example_query, cnx)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90d6c5fc-4848-4d4f-84c7-5f247f08de7d", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.849938Z", + "iopub.status.busy": "2024-05-16T15:51:17.849750Z", + "iopub.status.idle": "2024-05-16T15:51:17.852669Z", + "shell.execute_reply": "2024-05-16T15:51:17.852185Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.849924Z" + } + }, + "outputs": [], + "source": [ + "cur = cnx.cursor()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8d59efa-1e98-4cd2-9e1a-3609df99934b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.853526Z", + "iopub.status.busy": "2024-05-16T15:51:17.853344Z", + "iopub.status.idle": "2024-05-16T15:51:17.867448Z", + "shell.execute_reply": "2024-05-16T15:51:17.866908Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.853512Z" + } + }, + "outputs": [], + "source": [ + "res = cur.execute(\"SELECT * FROM sqlite_schema\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c087e1b-8bfe-4572-accf-49585a6a08a1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-05-16T15:51:17.868644Z", + "iopub.status.busy": "2024-05-16T15:51:17.868421Z", + "iopub.status.idle": "2024-05-16T15:51:17.880840Z", + "shell.execute_reply": "2024-05-16T15:51:17.880300Z", + "shell.execute_reply.started": "2024-05-16T15:51:17.868629Z" + } + }, + "outputs": [], + "source": [ + "res.fetchall()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6da55b0-dc55-4762-8735-c5531fdcef9c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LSST", + "language": "python", + "name": "lsst" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/adler/adler.py b/src/adler/adler.py index b167466..2987cc6 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -1,12 +1,17 @@ import logging import argparse import astropy.units as u +import numpy as np +import pandas as pd +import os from adler.dataclasses.AdlerPlanetoid import AdlerPlanetoid from adler.science.PhaseCurve import PhaseCurve from adler.utilities.AdlerCLIArguments import AdlerCLIArguments from adler.utilities.adler_logging import setup_adler_logging from adler.utilities.readin_utilities import read_in_SSObjectID_file +from adler.utilities.plotting_utilities import plot_errorbar +import adler.utilities.science_utilities as sci_utils logger = logging.getLogger(__name__) @@ -14,6 +19,9 @@ def runAdler(cli_args): logger.info("Beginning Adler.") + N_pc_fit = 10 # minimum number of data points to fit phase curve + diff_cut = 1.0 + if cli_args.ssObjectId_list: ssObjectId_list = read_in_SSObjectID_file(cli_args.ssObjectId_list) else: @@ -23,34 +31,168 @@ def runAdler(cli_args): logger.info("Processing object {}/{}.".format(i + 1, len(ssObjectId_list))) logger.info("Ingesting all data for object {} from RSP...".format(cli_args.ssObjectId)) - planetoid = AdlerPlanetoid.construct_from_RSP(ssObjectId, cli_args.filter_list, cli_args.date_range) + # if cli_args.sql_filename != "None": + if cli_args.sql_filename: + msg = "query sql database {}".format(cli_args.sql_filename) + logger.info(msg) + print(msg) + planetoid = AdlerPlanetoid.construct_from_SQL( + ssObjectId, + filter_list=cli_args.filter_list, + date_range=cli_args.date_range, + sql_filename=cli_args.sql_filename, + ) + else: + msg = "query RSP" + logger.info(msg) + print(msg) + planetoid = AdlerPlanetoid.construct_from_RSP( + ssObjectId, cli_args.filter_list, cli_args.date_range + ) logger.info("Data successfully ingested.") logger.info("Calculating phase curves...") # now let's do some phase curves! - # get the r filter SSObject metadata - sso_r = planetoid.SSObject_in_filter("r") + # # operate on each filter in turn + for filt in planetoid.filter_list: + print("fit {} filter data".format(filt)) + + # get the filter SSObject metadata + sso = planetoid.SSObject_in_filter(filt) + + # get the observations + obs = planetoid.observations_in_filter(filt) + df_obs = pd.DataFrame(obs.__dict__) + df_obs["outlier"] = [False] * len(df_obs) + print(len(df_obs)) + + # load and merge the previous obs + save_file = "{}/df_outlier_{}.csv".format(cli_args.outpath, cli_args.ssObjectId) + if os.path.isfile(save_file): + print("load {}".format(save_file)) + _df_obs = pd.read_csv(save_file, index_col=0) + df_obs = df_obs.merge(_df_obs, on="midPointMjdTai", how="left") + df_obs = df_obs.rename({"outlier_y": "outlier"}, axis=1) + df_obs = df_obs.drop("outlier_x", axis=1) + else: + print("no previous obs to load") + + print(cli_args.date_range) + print(np.amax(df_obs["midPointMjdTai"])) + t1 = int(np.amax(df_obs["midPointMjdTai"])) + 1 + t0 = t1 - 1 + + # t_mask = (df_obs["midPointMjdTai"]= diff_cut] = True @@ -47,7 +47,7 @@ def outlier_std(new_res, data_res, std_cut=3.0): data_std = np.std(data_res) if not isinstance(new_res, np.ndarray): - new_res = np.array(new_res) + new_res = np.array(new_res, ndmin=1) outlier_flag = np.array([False] * len(new_res)) outlier_flag[np.abs(new_res) >= (data_std * std_cut)] = True diff --git a/tests/adler/utilities/test_AdlerCLIArguments.py b/tests/adler/utilities/test_AdlerCLIArguments.py index 372bc83..aed938e 100644 --- a/tests/adler/utilities/test_AdlerCLIArguments.py +++ b/tests/adler/utilities/test_AdlerCLIArguments.py @@ -6,13 +6,14 @@ # AdlerCLIArguments object takes an object as input, so we define a quick one here class args: - def __init__(self, ssObjectId, ssObjectId_list, filter_list, date_range, outpath, db_name): + def __init__(self, ssObjectId, ssObjectId_list, filter_list, date_range, outpath, db_name, sql_filename): self.ssObjectId = ssObjectId self.ssObjectId_list = ssObjectId_list self.filter_list = filter_list self.date_range = date_range self.outpath = outpath self.db_name = db_name + self.sql_filename = sql_filename def test_AdlerCLIArguments_population(): @@ -24,6 +25,7 @@ def test_AdlerCLIArguments_population(): "date_range": [60000.0, 67300.0], "outpath": "./", "db_name": "output", + "sql_filename": None, } good_arguments = args(**good_input_dict) good_arguments_object = AdlerCLIArguments(good_arguments) @@ -39,10 +41,17 @@ def test_AdlerCLIArguments_population(): good_arguments_object = AdlerCLIArguments(good_arguments) assert good_arguments_object.ssObjectId_list == get_test_data_filepath("test_SSOIDs.txt") + # also test setting sql_filename + + good_input_dict["sql_filename"] = get_test_data_filepath("testing_database.db") + good_arguments = args(**good_input_dict) + good_arguments_object = AdlerCLIArguments(good_arguments) + assert good_arguments_object.sql_filename == get_test_data_filepath("testing_database.db") + def test_AdlerCLIArguments_badSSOID(): # test that a bad ssObjectId triggers the right error - bad_ssoid_arguments = args("hello!", None, ["g", "r", "i"], [60000.0, 67300.0], "./", "output") + bad_ssoid_arguments = args("hello!", None, ["g", "r", "i"], [60000.0, 67300.0], "./", "output", "None") with pytest.raises(ValueError) as bad_ssoid_error: bad_ssoid_object = AdlerCLIArguments(bad_ssoid_arguments) @@ -55,7 +64,7 @@ def test_AdlerCLIArguments_badSSOID(): def test_AdlerCLIArguments_badfilters(): # test that non-LSST or unexpected filters trigger the right error - bad_filter_arguments = args("666", None, ["g", "r", "i", "m"], [60000.0, 67300.0], "./", "output") + bad_filter_arguments = args("666", None, ["g", "r", "i", "m"], [60000.0, 67300.0], "./", "output", None) with pytest.raises(ValueError) as bad_filter_error: bad_filter_object = AdlerCLIArguments(bad_filter_arguments) @@ -65,7 +74,7 @@ def test_AdlerCLIArguments_badfilters(): == "Unexpected filters found in --filter_list command-line argument. --filter_list must be a list of LSST filters." ) - bad_filter_arguments_2 = args("666", None, ["pony"], [60000.0, 67300.0], "./", "output") + bad_filter_arguments_2 = args("666", None, ["pony"], [60000.0, 67300.0], "./", "output", None) with pytest.raises(ValueError) as bad_filter_error_2: bad_filter_object = AdlerCLIArguments(bad_filter_arguments_2) @@ -78,7 +87,7 @@ def test_AdlerCLIArguments_badfilters(): def test_AdlerCLIArguments_baddates(): # test that overly-large dates trigger the right error - big_date_arguments = args("666", None, ["g", "r", "i"], [260000.0, 267300.0], "./", "output") + big_date_arguments = args("666", None, ["g", "r", "i"], [260000.0, 267300.0], "./", "output", None) with pytest.raises(ValueError) as big_date_error: big_date_object = AdlerCLIArguments(big_date_arguments) @@ -89,7 +98,7 @@ def test_AdlerCLIArguments_baddates(): ) # test that unexpected date values trigger the right error - bad_date_arguments = args("666", None, ["g", "r", "i"], [60000.0, "cheese"], "./", "output") + bad_date_arguments = args("666", None, ["g", "r", "i"], [60000.0, "cheese"], "./", "output", None) with pytest.raises(ValueError) as bad_date_error: bad_date_object = AdlerCLIArguments(bad_date_arguments) @@ -102,7 +111,7 @@ def test_AdlerCLIArguments_baddates(): def test_AdlerCLIArguments_badoutput(): bad_output_arguments = args( - "666", None, ["g", "r", "i"], [60000.0, 67300.0], "./definitely_fake_folder/", "output" + "666", None, ["g", "r", "i"], [60000.0, 67300.0], "./definitely_fake_folder/", "output", None ) with pytest.raises(ValueError) as bad_output_error: @@ -122,6 +131,7 @@ def test_AdlerCLIArguments_badlist(): [60000.0, 67300.0], "./definitely_fake_folder/", "output", + None, ) with pytest.raises(ValueError) as bad_list_error: diff --git a/tests/adler/utilities/test_science_utilities.py b/tests/adler/utilities/test_science_utilities.py index 3bee7c9..3a2d753 100644 --- a/tests/adler/utilities/test_science_utilities.py +++ b/tests/adler/utilities/test_science_utilities.py @@ -184,6 +184,30 @@ def test_outlier_diff(): assert_array_equal(outliers, outliers_y) +# check that a single value gets processed as a np.array +def test_outlier_diff_single_val(): + outliers = sci_utils.outlier_diff(y_res[0], diff_cut=1.0) + assert_array_equal(outliers, outliers_y[:1]) + + +# check that a list gets processed as a np.array +def test_outlier_diff_single_val(): + outliers = sci_utils.outlier_diff(list(y_res[:3]), diff_cut=1.0) + assert_array_equal(outliers, outliers_y[:3]) + + +# test if a single data point is outlying the std residuals of the previous data points +def test_outlier_std(): + outliers = sci_utils.outlier_std(y_res[-8], y_res[:-8]) + assert_array_equal(outliers, outliers_y[-8]) + + +# test the std residual of a list of data points +def test_outlier_std(): + outliers = sci_utils.outlier_std(y_res[-8:-5], y_res[:-8]) + assert_array_equal(outliers, outliers_y[-8:-5]) + + def test_zero_func(): assert sci_utils.zero_func(y_res) == 0