Decrease_cadence.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload #Use this to reload modules if they are changed on disk while the notebook is running\n",
    "from __future__ import division\n",
    "from snmachine import sndata, snfeatures, snclassifier, tsne_plot\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time, os, pywt,subprocess\n",
    "from sklearn.decomposition import PCA\n",
    "from astropy.table import Table,join,vstack,unique\n",
    "from astropy.io import fits\n",
    "import sklearn.metrics \n",
    "from functools import partial\n",
    "from multiprocessing import Pool\n",
    "import sncosmo\n",
    "%matplotlib nbagg\n",
    "\n",
    "from astropy.table import Column\n",
    "# WARNING...\n",
    "#Multinest uses a hardcoded character limit for the output file names. I believe it's a limit of 100 characters\n",
    "#so avoid making this file path to lengthy if using nested sampling or multinest output file names will be truncated\n",
    "\n",
    "#Change outdir to somewhere on your computer if you like\n",
    "dataset='spcc'\n",
    "outdir=os.path.join('output_%s_no_z' %dataset,'')\n",
    "out_features=os.path.join(outdir,'features') #Where we save the extracted features to\n",
    "out_class=os.path.join(outdir,'classifications') #Where we save the classification probabilities and ROC curves\n",
    "out_int=os.path.join(outdir,'int') #Any intermediate files (such as multinest chains or GP fits)\n",
    "\n",
    "subprocess.call(['mkdir',outdir])\n",
    "subprocess.call(['mkdir',out_features])\n",
    "subprocess.call(['mkdir',out_class])\n",
    "subprocess.call(['mkdir',out_int])\n",
    "\n",
    "read_from_file=False #True #We can use this flag to quickly rerun from saved features\n",
    "run_name=os.path.join(out_features,'%s_all' %dataset)\n",
    "rt=os.path.join('SPCC_SUBSET','')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_new_data(filename, table, output, sntype):\n",
    "    with open(filename) as f:\n",
    "        data = f.read().split(\"\\n\")\n",
    "    filters = data[5].split()\n",
    "    survey = data[0].split()\n",
    "    stuffRA = data[6].split()\n",
    "    stuffDec = data[7].split()\n",
    "    MWEBV = data[11].split()\n",
    "    if sntype==1:\n",
    "        typestring='SN Type = Ia , MODEL = mlcs2k2.SNchallenge'\n",
    "    elif sntype==2:\n",
    "        typestring='SN Type = II , MODEL = SDSS-017564'\n",
    "    elif sntype==3:\n",
    "        typestring='SN Type = Ic , MODEL = SDSS-014475'\n",
    "    else:\n",
    "        typestring='SOMETHING WENT HORRIBLY WRONG'\n",
    "    table.meta = {survey[0][:-1]: survey[1], stuffRA[0][:-1]: stuffRA[1], stuffDec[0][:-1]: stuffDec[1],filters[0][:-1]: filters[1],\n",
    "                 MWEBV[0][:-1]: MWEBV[1], 'SNTYPE': -9, 'SIM_COMMENT': typestring  }\n",
    "    #table.rename_column('mjd', 'MJD')\n",
    "    #table.rename_column('filter ', 'FLT')\n",
    "    #table.rename_column('flux', 'FLUXCAL')\n",
    "    #table.rename_column('flux_error', 'FLUXCALERR')\n",
    "    sncosmo.write_lc(table, 'new_mocks_decrease_cadence/%s'%output,pedantic=True, format = 'snana')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prototypes_Ia=[ 'DES_SN002542.DAT', 'DES_SN013866.DAT', 'DES_SN023940.DAT', 'DES_SN024734.DAT', 'DES_SN030701.DAT', 'DES_SN045040.DAT']\n",
    "prototypes_II=[ 'DES_SN002457.DAT', 'DES_SN005519.DAT', 'DES_SN006381.DAT', 'DES_SN008569.DAT', 'DES_SN013360.DAT', 'DES_SN013481.DAT']\n",
    "prototypes_Ibc=['DES_SN005399.DAT', 'DES_SN013863.DAT', 'DES_SN027266.DAT', 'DES_SN030183.DAT', 'DES_SN065493.DAT', 'DES_SN078241.DAT']\n",
    "len(prototypes_II)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading data...\n",
      "23 objects read into memory.\n"
     ]
    }
   ],
   "source": [
    "dat=sndata.Dataset(rt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "drop_lines(dat, {'desg': 1., 'desr': 0., 'desi': 1., 'desz': 0.})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DES_SN001695.DAT': <Table length=64>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       " 0.0160000000033   desr  -5.456      5.335    27.5    ab\n",
       " 0.0619999999981   desz   1.439      4.691    27.5    ab\n",
       "            2.14   desr  -2.837       3.03    27.5    ab\n",
       "           2.187   desz  -5.086      3.518    27.5    ab\n",
       "            10.0   desr  0.8236      1.251    27.5    ab\n",
       "          10.039   desz   3.812      2.176    27.5    ab\n",
       "          11.988   desr -0.5332      3.876    27.5    ab\n",
       "          12.031   desz -0.6927      4.692    27.5    ab\n",
       "          16.957   desr  -2.873       2.35    27.5    ab\n",
       "          23.008   desz   7.169      5.637    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "         119.875   desr   3.569      1.222    27.5    ab\n",
       "         119.918   desz   17.56      2.146    27.5    ab\n",
       "          127.93   desr    1.47      1.301    27.5    ab\n",
       "         128.871   desz   8.927      2.717    27.5    ab\n",
       "         135.883   desr   2.173      2.524    27.5    ab\n",
       "         139.894   desz   8.095      4.252    27.5    ab\n",
       "         143.871   desr  0.7718      1.365    27.5    ab\n",
       "         148.867   desz   5.982      2.609    27.5    ab\n",
       "         155.879   desr  0.3982      1.383    27.5    ab\n",
       "         157.871   desz   16.46      3.394    27.5    ab,\n",
       " 'DES_SN002457.DAT': <Table length=50>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       " 0.0160000000033   desr  -12.65      7.845    27.5    ab\n",
       "  0.977000000006   desz  -4.156      5.328    27.5    ab\n",
       "             2.0   desr  0.0281      2.425    27.5    ab\n",
       "           9.957   desz  -6.591      27.32    27.5    ab\n",
       "          10.121   desr  -1.047      4.152    27.5    ab\n",
       "          15.989   desr  -2.718        5.4    27.5    ab\n",
       "          15.993   desz   1.047      6.541    27.5    ab\n",
       "          29.942   desr   14.92      2.456    27.5    ab\n",
       "          29.957   desz   6.178      4.941    27.5    ab\n",
       "          37.942   desr    39.9      2.206    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "         104.977   desr   2.921      3.328    27.5    ab\n",
       "         104.989   desz   12.96      5.051    27.5    ab\n",
       "         109.981   desr   2.714      6.181    27.5    ab\n",
       "         110.891   desz   23.85      5.443    27.5    ab\n",
       "         113.868   desr   1.241      4.828    27.5    ab\n",
       "         113.965   desz   13.31      5.166    27.5    ab\n",
       "         125.821   desr   2.634      1.659    27.5    ab\n",
       "         125.852   desz   9.238       3.94    27.5    ab\n",
       "         133.832   desr   1.952      2.963    27.5    ab\n",
       "         133.879   desz   19.26      5.441    27.5    ab,\n",
       " 'DES_SN002542.DAT': <Table length=62>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00800000000163   desr   38.23      2.257    27.5    ab\n",
       "  0.0469999999987   desz   29.16      2.545    27.5    ab\n",
       "            3.004   desr   49.79      1.963    27.5    ab\n",
       "            3.075   desz   46.65      3.832    27.5    ab\n",
       "             4.09   desr    54.4      1.675    27.5    ab\n",
       "            4.137   desz   45.53      4.569    27.5    ab\n",
       "           11.985   desr    86.6      5.829    27.5    ab\n",
       "           12.047   desz   64.54      3.611    27.5    ab\n",
       "           13.079   desr   80.42      3.629    27.5    ab\n",
       "           13.121   desz   77.19      3.302    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           84.965   desz   13.36      3.098    27.5    ab\n",
       "           96.985   desr   2.461      1.217    27.5    ab\n",
       "           97.028   desz    9.73       2.26    27.5    ab\n",
       "          104.993   desr    3.36      1.277    27.5    ab\n",
       "          105.036   desz   12.99      2.305    27.5    ab\n",
       "          111.903   desr   5.661      2.117    27.5    ab\n",
       "           111.95   desz   7.943      2.589    27.5    ab\n",
       "          113.051   desr   4.146      2.703    27.5    ab\n",
       "          115.844   desz   7.881      2.068    27.5    ab\n",
       "          115.965   desr   3.344      1.738    27.5    ab,\n",
       " 'DES_SN005399.DAT': <Table length=68>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00800000000163   desr   28.55      2.205    27.5    ab\n",
       "  0.0469999999987   desz   18.93       2.51    27.5    ab\n",
       "            3.004   desr    26.2      1.789    27.5    ab\n",
       "            3.075   desz   30.67      3.774    27.5    ab\n",
       "             4.09   desr   26.12      1.405    27.5    ab\n",
       "            4.137   desz   21.21      4.505    27.5    ab\n",
       "           11.985   desr   28.95      5.606    27.5    ab\n",
       "           12.047   desz   30.72      3.449    27.5    ab\n",
       "           13.079   desr   22.03      3.306    27.5    ab\n",
       "           13.121   desz   25.39      3.005    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           111.95   desz   19.48      2.611    27.5    ab\n",
       "          113.051   desr   9.141      2.709    27.5    ab\n",
       "          115.844   desz   19.08      2.094    27.5    ab\n",
       "          115.965   desr   9.942       1.75    27.5    ab\n",
       "           126.86   desr   7.222     0.9163    27.5    ab\n",
       "          126.903   desz   16.12      2.231    27.5    ab\n",
       "          134.864   desr   5.358      1.503    27.5    ab\n",
       "          134.907   desz   15.99      2.454    27.5    ab\n",
       "           140.95   desr   4.483      1.864    27.5    ab\n",
       "           141.95   desz   16.54      2.992    27.5    ab,\n",
       " 'DES_SN005519.DAT': <Table length=49>\n",
       "      mjd       filter   flux  flux_error    zp   zpsys\n",
       "    float64      str4  float64  float64   float64  str2\n",
       " -------------- ------ ------- ---------- ------- -----\n",
       " 0.015999999996   desr  -5.537      11.64    27.5    ab\n",
       " 0.156999999999   desz  -1.319      2.842    27.5    ab\n",
       "          1.125   desr -0.5662      3.595    27.5    ab\n",
       "          1.168   desz -0.2056      2.925    27.5    ab\n",
       "          2.012   desr   8.636      3.867    27.5    ab\n",
       "  2.18399999999   desz   -2.94      2.922    27.5    ab\n",
       "          3.125   desr   3.304      2.654    27.5    ab\n",
       "           3.16   desz   1.673      3.017    27.5    ab\n",
       "          7.129   desr  0.6492      1.756    27.5    ab\n",
       "          7.164   desz -0.3965      2.936    27.5    ab\n",
       "            ...    ...     ...        ...     ...   ...\n",
       "         78.012   desz   29.24      2.767    27.5    ab\n",
       "         85.024   desr   11.37      2.536    27.5    ab\n",
       "         89.035   desz   24.65      4.275    27.5    ab\n",
       "         93.012   desr   10.79      1.385    27.5    ab\n",
       "         98.008   desz   25.58      2.652    27.5    ab\n",
       "         105.02   desr   4.999      1.389    27.5    ab\n",
       "        107.012   desz   22.43      3.408    27.5    ab\n",
       "        116.969   desr    3.45      1.863    27.5    ab\n",
       "        117.961   desz   14.28       3.71    27.5    ab\n",
       "        121.973   desr   4.328       2.06    27.5    ab,\n",
       " 'DES_SN006381.DAT': <Table length=56>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       " 0.0160000000033   desr  -3.907      5.335    27.5    ab\n",
       " 0.0619999999981   desz  0.4043      4.691    27.5    ab\n",
       "            2.14   desr  -1.686       3.03    27.5    ab\n",
       "           2.187   desz  -7.802       3.52    27.5    ab\n",
       "            10.0   desr   20.75      1.322    27.5    ab\n",
       "          10.039   desz   14.66      2.195    27.5    ab\n",
       "          11.988   desr   22.24      3.909    27.5    ab\n",
       "          12.031   desz   13.26        4.7    27.5    ab\n",
       "          16.957   desr   25.43      2.409    27.5    ab\n",
       "          23.008   desz   24.59      5.655    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "          84.058   desr   5.386      1.857    27.5    ab\n",
       "          90.008   desz   2.749       3.05    27.5    ab\n",
       "          97.004   desr    3.39      1.446    27.5    ab\n",
       "          99.019   desz    8.96      3.406    27.5    ab\n",
       "         105.012   desr  0.5847      2.178    27.5    ab\n",
       "         110.019   desz   9.166      3.898    27.5    ab\n",
       "         111.871   desr   -4.79      4.184    27.5    ab\n",
       "         112.004   desz   6.971      4.466    27.5    ab\n",
       "         119.875   desr   1.794       1.22    27.5    ab\n",
       "         119.918   desz   12.55      2.133    27.5    ab,\n",
       " 'DES_SN008569.DAT': <Table length=33>\n",
       "       mjd        filter   flux   flux_error    zp   zpsys\n",
       "     float64       str4  float64   float64   float64  str2\n",
       " ---------------- ------ -------- ---------- ------- -----\n",
       " 0.00800000000163   desr   0.2734      1.223    27.5    ab\n",
       "  0.0469999999987   desz   -1.338      2.416    27.5    ab\n",
       "   0.985000000001   desr    1.162      1.373    27.5    ab\n",
       "             1.02   desz  -0.1167       2.23    27.5    ab\n",
       "            1.996   desr  -0.7404      1.174    27.5    ab\n",
       "            2.043   desz  -0.3098      5.917    27.5    ab\n",
       "            3.043   desr   -3.868      3.168    27.5    ab\n",
       "            3.082   desz   0.6993      12.77    27.5    ab\n",
       "            5.012   desr   0.6669      1.325    27.5    ab\n",
       "            5.047   desz    1.615      3.029    27.5    ab\n",
       "              ...    ...      ...        ...     ...   ...\n",
       "            62.84   desz    10.13      2.718    27.5    ab\n",
       "           69.852   desr    8.253      2.531    27.5    ab\n",
       "           73.863   desz     7.41      4.252    27.5    ab\n",
       "            77.84   desr     7.11      1.375    27.5    ab\n",
       "           82.836   desz    7.831      2.611    27.5    ab\n",
       "           89.848   desr    5.233      1.389    27.5    ab\n",
       "            91.84   desz    4.592      3.382    27.5    ab\n",
       "          101.797   desr    1.767      1.863    27.5    ab\n",
       "          102.789   desz -0.08511      3.699    27.5    ab\n",
       "          106.801   desr     5.36      2.061    27.5    ab,\n",
       " 'DES_SN013360.DAT': <Table length=45>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00799999999435   desr   -2.33      3.864    27.5    ab\n",
       "   0.179999999993   desz   3.993      2.922    27.5    ab\n",
       "            1.121   desr  -2.353      2.654    27.5    ab\n",
       "            1.156   desz -0.3359      3.017    27.5    ab\n",
       "            5.125   desr   -1.85      1.757    27.5    ab\n",
       "             5.16   desz    1.02      2.936    27.5    ab\n",
       "            6.137   desr   1.754      1.875    27.5    ab\n",
       "            6.172   desz   1.364       3.19    27.5    ab\n",
       "            7.153   desr   3.316      2.253    27.5    ab\n",
       "    7.20299999999   desz  -1.946      3.867    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           76.008   desz   38.17      2.806    27.5    ab\n",
       "            83.02   desr   29.22       2.59    27.5    ab\n",
       "           87.031   desz   42.38      4.323    27.5    ab\n",
       "           91.008   desr   20.71      1.429    27.5    ab\n",
       "           96.004   desz   32.59       2.68    27.5    ab\n",
       "          103.016   desr   11.34      1.405    27.5    ab\n",
       "          105.008   desz    24.3      3.413    27.5    ab\n",
       "          114.965   desr   7.392      1.869    27.5    ab\n",
       "          115.957   desz   23.73      3.727    27.5    ab\n",
       "          119.969   desr   7.217      2.063    27.5    ab,\n",
       " 'DES_SN013481.DAT': <Table length=80>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00800000000163   desr    1.56      2.129    27.5    ab\n",
       "  0.0469999999987   desz   4.467      2.484    27.5    ab\n",
       "            3.004   desr  0.6139      1.709    27.5    ab\n",
       "            3.075   desz   7.738      3.732    27.5    ab\n",
       "             4.09   desr  -1.532      1.301    27.5    ab\n",
       "            4.137   desz  -1.758      4.487    27.5    ab\n",
       "           11.985   desr  -5.238       5.57    27.5    ab\n",
       "           12.047   desz   5.434      3.401    27.5    ab\n",
       "           13.079   desr   2.928      3.268    27.5    ab\n",
       "           13.121   desz   10.89      2.972    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           155.86   desr   7.992     0.9923    27.5    ab\n",
       "          155.903   desz   16.65      2.597    27.5    ab\n",
       "          163.864   desr   6.765      1.655    27.5    ab\n",
       "          163.907   desz   10.85      2.989    27.5    ab\n",
       "          171.879   desr   7.826      2.106    27.5    ab\n",
       "          172.836   desz   11.24      2.438    27.5    ab\n",
       "          173.864   desr   10.62      1.828    27.5    ab\n",
       "          173.907   desz   11.74      4.865    27.5    ab\n",
       "          174.864   desr    4.79      1.766    27.5    ab\n",
       "          174.907   desz   11.12      4.786    27.5    ab,\n",
       " 'DES_SN013863.DAT': <Table length=52>\n",
       "      mjd       filter   flux  flux_error    zp   zpsys\n",
       "    float64      str4  float64  float64   float64  str2\n",
       " -------------- ------ ------- ---------- ------- -----\n",
       " 0.038999999997   desz   -2.52      1.817    27.5    ab\n",
       "          3.922   desr  -2.976      3.581    27.5    ab\n",
       "          4.075   desz   1.156      4.013    27.5    ab\n",
       "           4.91   desr  0.7392      3.125    27.5    ab\n",
       "          5.059   desz  -5.451      4.836    27.5    ab\n",
       "         10.895   desr   9.617      11.64    27.5    ab\n",
       "         11.036   desz   2.448      2.843    27.5    ab\n",
       "         12.004   desr   5.424      3.596    27.5    ab\n",
       "         12.047   desz  -2.215      2.926    27.5    ab\n",
       "         12.891   desr -0.7612      3.863    27.5    ab\n",
       "            ...    ...     ...        ...     ...   ...\n",
       "         79.938   desz    20.0      2.154    27.5    ab\n",
       "          87.95   desr   5.564      1.309    27.5    ab\n",
       "         88.891   desz    13.6      2.724    27.5    ab\n",
       "         95.903   desr   7.143      2.529    27.5    ab\n",
       "         99.914   desz   18.27      4.263    27.5    ab\n",
       "        103.891   desr   2.627      1.368    27.5    ab\n",
       "        108.887   desz   8.733      2.612    27.5    ab\n",
       "        115.899   desr   2.282      1.385    27.5    ab\n",
       "        117.891   desz   5.766      3.382    27.5    ab\n",
       "        127.848   desr    1.33      1.862    27.5    ab,\n",
       " 'DES_SN013866.DAT': <Table length=47>\n",
       "      mjd      filter   flux  flux_error    zp   zpsys\n",
       "    float64     str4  float64  float64   float64  str2\n",
       " ------------- ------ ------- ---------- ------- -----\n",
       "           0.0   desz  -5.787      3.583    27.5    ab\n",
       "         2.871   desr   4.569      4.331    27.5    ab\n",
       " 2.90199999999   desz  -5.267      8.139    27.5    ab\n",
       "         3.887   desr   -6.84      4.494    27.5    ab\n",
       "         3.918   desz  -2.112      10.16    27.5    ab\n",
       "        11.856   desr  -3.603      4.369    27.5    ab\n",
       "        11.875   desz   3.793      6.472    27.5    ab\n",
       "        12.856   desr   4.818      4.331    27.5    ab\n",
       "        12.875   desz  -7.301      6.407    27.5    ab\n",
       "        17.856   desr  -3.111      3.584    27.5    ab\n",
       "           ...    ...     ...        ...     ...   ...\n",
       "       100.012   desr   10.06      2.903    27.5    ab\n",
       "       102.004   desz   39.96      4.356    27.5    ab\n",
       "       111.883   desr    9.31      5.408    27.5    ab\n",
       "       111.906   desz   21.25      6.749    27.5    ab\n",
       "       115.965   desr   10.43      3.882    27.5    ab\n",
       "       119.891   desz   17.42      4.477    27.5    ab\n",
       "       128.891   desr    8.43      2.189    27.5    ab\n",
       "        128.91   desz   18.98      3.684    27.5    ab\n",
       "       140.883   desr   9.693       6.19    27.5    ab\n",
       "       141.875   desz   23.28      13.46    27.5    ab,\n",
       " 'DES_SN016742.DAT': <Table length=48>\n",
       "      mjd       filter   flux  flux_error    zp   zpsys\n",
       "    float64      str4  float64  float64   float64  str2\n",
       " -------------- ------ ------- ---------- ------- -----\n",
       " 0.987999999998   desr   -9.45      17.59    27.5    ab\n",
       "          5.109   desz   1.961      3.581    27.5    ab\n",
       "           7.98   desr   0.478      4.331    27.5    ab\n",
       "          8.011   desz  -1.546      8.138    27.5    ab\n",
       "          8.996   desr   3.199      4.494    27.5    ab\n",
       "          9.027   desz  -16.88      10.17    27.5    ab\n",
       "         16.965   desr   44.71      4.469    27.5    ab\n",
       "         16.984   desz   23.37       6.49    27.5    ab\n",
       "         17.965   desr   43.61       4.43    27.5    ab\n",
       "         17.984   desz   44.69      6.464    27.5    ab\n",
       "            ...    ...     ...        ...     ...   ...\n",
       "        105.121   desr   4.038      2.888    27.5    ab\n",
       "        107.113   desz  -10.53       4.29    27.5    ab\n",
       "        116.992   desr   2.022      5.401    27.5    ab\n",
       "        117.015   desz   14.87      6.741    27.5    ab\n",
       "        121.074   desr  0.6109      3.868    27.5    ab\n",
       "          125.0   desz   6.388      4.463    27.5    ab\n",
       "          134.0   desr  -1.008      2.174    27.5    ab\n",
       "        134.019   desz   9.801      3.668    27.5    ab\n",
       "        145.992   desr  -1.686      6.184    27.5    ab\n",
       "        146.984   desz  -2.941      13.45    27.5    ab,\n",
       " 'DES_SN017270.DAT': <Table length=80>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00800000000163   desr -0.6359      2.129    27.5    ab\n",
       "  0.0469999999987   desz  0.9275      2.483    27.5    ab\n",
       "            3.004   desr   2.235       1.71    27.5    ab\n",
       "            3.075   desz -0.1346       3.73    27.5    ab\n",
       "             4.09   desr 0.06012      1.301    27.5    ab\n",
       "            4.137   desz   9.718       4.49    27.5    ab\n",
       "           11.985   desr   3.776      5.571    27.5    ab\n",
       "           12.047   desz   16.21      3.414    27.5    ab\n",
       "           13.079   desr   6.106      3.271    27.5    ab\n",
       "           13.121   desz   10.62      2.973    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           155.86   desr  0.4821     0.9739    27.5    ab\n",
       "          155.903   desz   6.192      2.579    27.5    ab\n",
       "          163.864   desr -0.1445      1.646    27.5    ab\n",
       "          163.907   desz  -5.762      2.982    27.5    ab\n",
       "          171.879   desr   2.309      2.099    27.5    ab\n",
       "          172.836   desz   4.977      2.428    27.5    ab\n",
       "          173.864   desr   3.101      1.815    27.5    ab\n",
       "          173.907   desz   7.702      4.861    27.5    ab\n",
       "          174.864   desr -0.2532       1.76    27.5    ab\n",
       "          174.907   desz  -1.172       4.78    27.5    ab,\n",
       " 'DES_SN023940.DAT': <Table length=47>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00699999999779   desr   12.26      7.184    27.5    ab\n",
       "   0.023000000001   desz   24.11      7.934    27.5    ab\n",
       "              2.0   desr   33.65      3.258    27.5    ab\n",
       "            2.015   desz   41.68       5.03    27.5    ab\n",
       "           10.101   desr   95.59      4.447    27.5    ab\n",
       "           10.124   desz    89.4      5.664    27.5    ab\n",
       "           15.968   desr   106.4      3.488    27.5    ab\n",
       "             16.0   desz   111.9      4.947    27.5    ab\n",
       "           29.007   desr   83.61      3.837    27.5    ab\n",
       "           29.031   desz   80.05      6.148    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           75.984   desz   19.79      4.037    27.5    ab\n",
       "           79.953   desr   12.57      5.064    27.5    ab\n",
       "           82.855   desz   22.13      4.637    27.5    ab\n",
       "            83.07   desr   11.14      3.931    27.5    ab\n",
       "           96.007   desr   8.152       2.74    27.5    ab\n",
       "           96.031   desz   9.708      5.016    27.5    ab\n",
       "          104.968   desr   4.223      3.135    27.5    ab\n",
       "          104.996   desz   10.37      4.111    27.5    ab\n",
       "          110.878   desr   6.264       5.05    27.5    ab\n",
       "          110.898   desz   4.496      5.624    27.5    ab,\n",
       " 'DES_SN024734.DAT': <Table length=62>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       "  0.015999999996   desr   6.454      3.269    27.5    ab\n",
       " 0.0579999999973   desz   4.278      2.965    27.5    ab\n",
       "           5.016   desr  -1.242      1.486    27.5    ab\n",
       "           5.051   desz  -5.387      2.386    27.5    ab\n",
       "          12.922   desr  -3.009      2.323    27.5    ab\n",
       "          12.961   desz  0.2645      2.711    27.5    ab\n",
       "          20.934   desr  -3.272      3.215    27.5    ab\n",
       "           20.98   desz -0.5636      3.845    27.5    ab\n",
       "          28.937   desr   28.54      1.115    27.5    ab\n",
       "          29.902   desz   38.96       2.07    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "         102.781   desz   82.93      2.585    27.5    ab\n",
       "         102.902   desr   58.24      2.079    27.5    ab\n",
       "         113.797   desr   49.56      1.319    27.5    ab\n",
       "          113.84   desz   48.54      2.393    27.5    ab\n",
       "         121.801   desr   46.32       1.75    27.5    ab\n",
       "         121.844   desz   53.46      2.636    27.5    ab\n",
       "         127.887   desr   45.44      2.062    27.5    ab\n",
       "         128.887   desz   52.93      3.139    27.5    ab\n",
       "         131.844   desr   49.96      3.063    27.5    ab\n",
       "         131.887   desz   53.05      3.124    27.5    ab,\n",
       " 'DES_SN027266.DAT': <Table length=80>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00800000000163   desr   2.453       2.13    27.5    ab\n",
       "  0.0469999999987   desz -0.7305      2.483    27.5    ab\n",
       "            3.004   desr   2.074       1.71    27.5    ab\n",
       "            3.075   desz  -3.522       3.73    27.5    ab\n",
       "             4.09   desr  0.7831        1.3    27.5    ab\n",
       "            4.137   desz  -3.744      4.487    27.5    ab\n",
       "           11.985   desr  -2.667      5.569    27.5    ab\n",
       "           12.047   desz  -4.466        3.4    27.5    ab\n",
       "           13.079   desr  0.2398      3.267    27.5    ab\n",
       "           13.121   desz   1.353      2.964    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           155.86   desr -0.6129     0.9744    27.5    ab\n",
       "          155.903   desz   5.966      2.579    27.5    ab\n",
       "          163.864   desr   1.097      1.647    27.5    ab\n",
       "          163.907   desz   6.447      2.983    27.5    ab\n",
       "          171.879   desr -0.9408      2.099    27.5    ab\n",
       "          172.836   desz   1.971      2.427    27.5    ab\n",
       "          173.864   desr   1.112      1.814    27.5    ab\n",
       "          173.907   desz   7.085      4.861    27.5    ab\n",
       "          174.864   desr  0.2658      1.761    27.5    ab\n",
       "          174.907   desz   8.507      4.783    27.5    ab,\n",
       " 'DES_SN030183.DAT': <Table length=53>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       " 0.0239999999976   desz   1.789      5.635    27.5    ab\n",
       "           7.016   desr  -0.246      1.603    27.5    ab\n",
       "   9.05899999999   desz   2.073        2.2    27.5    ab\n",
       "          15.024   desr   1.262      1.098    27.5    ab\n",
       "          17.035   desz   4.612      1.819    27.5    ab\n",
       "          20.918   desr   1.938      3.582    27.5    ab\n",
       "          21.071   desz  0.2011      4.014    27.5    ab\n",
       "          21.906   desr   5.305      3.128    27.5    ab\n",
       "          22.055   desz   3.852      4.836    27.5    ab\n",
       "          27.891   desr    11.8      11.64    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "          88.887   desr    14.6      4.195    27.5    ab\n",
       "           89.02   desz   26.48      4.493    27.5    ab\n",
       "          96.891   desr   13.27      1.251    27.5    ab\n",
       "          96.934   desz   26.37      2.179    27.5    ab\n",
       "         104.946   desr    8.89      1.316    27.5    ab\n",
       "         105.887   desz   17.26      2.731    27.5    ab\n",
       "         112.899   desr   5.703      2.528    27.5    ab\n",
       "          116.91   desz   10.73      4.254    27.5    ab\n",
       "         120.887   desr   5.143      1.371    27.5    ab\n",
       "         125.883   desz   4.124      2.608    27.5    ab,\n",
       " 'DES_SN030701.DAT': <Table length=53>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       "  0.015999999996   desr   2.941      2.429    27.5    ab\n",
       " 0.0309999999954   desz   1.599      4.938    27.5    ab\n",
       "           8.016   desr  0.5072      2.036    27.5    ab\n",
       "           8.031   desz  -3.979      4.045    27.5    ab\n",
       "            13.0   desr  -7.917      6.049    27.5    ab\n",
       "          13.016   desz   22.31       15.4    27.5    ab\n",
       "          14.164   desr  -18.99      15.17    27.5    ab\n",
       "           14.18   desz  -11.34      40.86    27.5    ab\n",
       "          20.145   desr  -2.909      5.493    27.5    ab\n",
       "           20.16   desz  -7.592      5.801    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "         111.891   desr   2.917      2.492    27.5    ab\n",
       "          111.91   desz   13.92      4.404    27.5    ab\n",
       "         119.895   desr    0.95      2.405    27.5    ab\n",
       "         119.926   desz   13.98      6.788    27.5    ab\n",
       "         127.922   desr   3.455       3.01    27.5    ab\n",
       "         128.918   desz   6.729       3.75    27.5    ab\n",
       "         136.895   desr   1.933      3.772    27.5    ab\n",
       "         139.867   desz   8.596      5.023    27.5    ab\n",
       "         141.867   desr   4.918      3.937    27.5    ab\n",
       "         142.863   desz    23.3      6.113    27.5    ab,\n",
       " 'DES_SN045040.DAT': <Table length=70>\n",
       "       mjd       filter   flux   flux_error    zp   zpsys\n",
       "     float64      str4  float64   float64   float64  str2\n",
       " --------------- ------ -------- ---------- ------- -----\n",
       " 0.0119999999952   desr   0.2102      1.485    27.5    ab\n",
       " 0.0469999999987   desz    -3.59      2.385    27.5    ab\n",
       "           7.918   desr   -2.565      2.323    27.5    ab\n",
       "   7.95699999999   desz   -3.277      2.711    27.5    ab\n",
       "           15.93   desr   -3.319      3.215    27.5    ab\n",
       "          15.976   desz -0.07772      3.845    27.5    ab\n",
       "          23.933   desr   0.8088     0.9561    27.5    ab\n",
       "          24.898   desz   -1.222      1.933    27.5    ab\n",
       "          27.023   desr    3.009      5.581    27.5    ab\n",
       "          27.066   desz    7.315      14.39    27.5    ab\n",
       "             ...    ...      ...        ...     ...   ...\n",
       "         137.793   desr  0.04514     0.9752    27.5    ab\n",
       "         137.836   desz   0.9079      2.577    27.5    ab\n",
       "         145.797   desr   -1.493      1.647    27.5    ab\n",
       "          145.84   desz    2.342      2.981    27.5    ab\n",
       "         153.812   desr   0.5033      2.099    27.5    ab\n",
       "         154.769   desz    4.894      2.429    27.5    ab\n",
       "         155.797   desr    1.296      1.814    27.5    ab\n",
       "          155.84   desz  0.06995      4.859    27.5    ab\n",
       "         156.797   desr   -3.201      1.761    27.5    ab\n",
       "          156.84   desz    5.636      4.782    27.5    ab,\n",
       " 'DES_SN064968.DAT': <Table length=74>\n",
       "       mjd       filter   flux   flux_error    zp   zpsys\n",
       "     float64      str4  float64   float64   float64  str2\n",
       " --------------- ------ -------- ---------- ------- -----\n",
       "  0.015999999996   desr  0.07073      5.569    27.5    ab\n",
       " 0.0779999999941   desz   -4.248        3.4    27.5    ab\n",
       "   1.10999999999   desr    3.627      3.268    27.5    ab\n",
       "   1.15199999999   desz   -2.788      2.964    27.5    ab\n",
       "   6.10999999999   desr 0.002954      1.485    27.5    ab\n",
       "           6.145   desz   -1.996      2.384    27.5    ab\n",
       "          14.016   desr    2.494      2.322    27.5    ab\n",
       "          14.055   desz    3.828      2.712    27.5    ab\n",
       "          22.028   desr   -2.366      3.215    27.5    ab\n",
       "          22.074   desz   0.2105      3.845    27.5    ab\n",
       "             ...    ...      ...        ...     ...   ...\n",
       "         143.891   desr  -0.8271     0.9739    27.5    ab\n",
       "         143.934   desz   0.8956      2.577    27.5    ab\n",
       "         151.895   desr   0.2299      1.646    27.5    ab\n",
       "         151.938   desz    2.814      2.981    27.5    ab\n",
       "          159.91   desr  -0.6158      2.099    27.5    ab\n",
       "         160.867   desz   -2.398      2.427    27.5    ab\n",
       "         161.895   desr   -1.268      1.814    27.5    ab\n",
       "         161.938   desz   -5.644       4.86    27.5    ab\n",
       "         162.895   desr  -0.6216      1.761    27.5    ab\n",
       "         162.938   desz    1.206       4.78    27.5    ab,\n",
       " 'DES_SN065493.DAT': <Table length=68>\n",
       "       mjd        filter   flux  flux_error    zp   zpsys\n",
       "     float64       str4  float64  float64   float64  str2\n",
       " ---------------- ------ ------- ---------- ------- -----\n",
       " 0.00800000000163   desr -0.1374      2.129    27.5    ab\n",
       "  0.0469999999987   desz  -1.843      2.483    27.5    ab\n",
       "            3.004   desr   2.969       1.71    27.5    ab\n",
       "            3.075   desz  -1.227      3.729    27.5    ab\n",
       "             4.09   desr  -0.253        1.3    27.5    ab\n",
       "            4.137   desz   1.766      4.486    27.5    ab\n",
       "           11.985   desr   2.633      5.569    27.5    ab\n",
       "           12.047   desz  -1.407      3.399    27.5    ab\n",
       "           13.079   desr  0.4093      3.267    27.5    ab\n",
       "           13.121   desz   7.108      2.967    27.5    ab\n",
       "              ...    ...     ...        ...     ...   ...\n",
       "           111.95   desz   13.97      2.598    27.5    ab\n",
       "          113.051   desr   2.692      2.702    27.5    ab\n",
       "          115.844   desz   8.543      2.069    27.5    ab\n",
       "          115.965   desr  -1.149      1.737    27.5    ab\n",
       "           126.86   desr   3.447      0.905    27.5    ab\n",
       "          126.903   desz   15.32      2.228    27.5    ab\n",
       "          134.864   desr   4.155        1.5    27.5    ab\n",
       "          134.907   desz   11.24      2.444    27.5    ab\n",
       "           140.95   desr    2.69      1.862    27.5    ab\n",
       "           141.95   desz    6.97      2.979    27.5    ab,\n",
       " 'DES_SN078241.DAT': <Table length=33>\n",
       "       mjd       filter   flux  flux_error    zp   zpsys\n",
       "     float64      str4  float64  float64   float64  str2\n",
       " --------------- ------ ------- ---------- ------- -----\n",
       " 0.0119999999952   desr -0.7614      4.253    27.5    ab\n",
       " 0.0270000000019   desz   9.087      7.806    27.5    ab\n",
       "           1.039   desr   7.869      40.68    27.5    ab\n",
       "           1.058   desz  -9.379      25.67    27.5    ab\n",
       "           2.058   desr  -8.067      13.68    27.5    ab\n",
       "           6.769   desz -0.3874      7.821    27.5    ab\n",
       "           7.769   desr  0.3629      4.105    27.5    ab\n",
       "           8.019   desz -0.1988      6.054    27.5    ab\n",
       "           9.051   desr   1.902       10.1    27.5    ab\n",
       "           9.777   desz  0.5156      4.292    27.5    ab\n",
       "             ...    ...     ...        ...     ...   ...\n",
       "          69.844   desr   26.37      3.266    27.5    ab\n",
       "          69.863   desz   89.01      5.652    27.5    ab\n",
       "          81.808   desr   3.386      4.574    27.5    ab\n",
       "          81.828   desz    54.9      6.456    27.5    ab\n",
       "          90.781   desr   6.055      2.804    27.5    ab\n",
       "          90.797   desz   27.13      4.785    27.5    ab\n",
       "          99.785   desr   15.03      3.096    27.5    ab\n",
       "         100.785   desz   28.01      4.078    27.5    ab\n",
       "         111.738   desr   8.557      2.488    27.5    ab\n",
       "          111.75   desz   23.18      6.579    27.5    ab,\n",
       " 'DES_SN149147.DAT': <Table length=38>\n",
       "       mjd        filter   flux   flux_error    zp   zpsys\n",
       "     float64       str4  float64   float64   float64  str2\n",
       " ---------------- ------ -------- ---------- ------- -----\n",
       " 0.00400000000081   desr   -3.677      3.585    27.5    ab\n",
       "  0.0190000000002   desz   -12.54      15.08    27.5    ab\n",
       "            4.988   desr    10.89       9.05    27.5    ab\n",
       "            5.004   desz   0.3649      7.694    27.5    ab\n",
       "           14.211   desr   -2.958      1.766    27.5    ab\n",
       "           15.148   desz    1.766      3.242    27.5    ab\n",
       "           22.211   desr 0.004955      1.881    27.5    ab\n",
       "            24.14   desz   -4.651      2.673    27.5    ab\n",
       "           27.004   desr   -2.277      3.577    27.5    ab\n",
       "           27.023   desz   -2.178      5.403    27.5    ab\n",
       "              ...    ...      ...        ...     ...   ...\n",
       "            82.16   desr    36.95      2.999    27.5    ab\n",
       "           84.152   desz    39.49      4.354    27.5    ab\n",
       "           94.031   desr    31.75      5.446    27.5    ab\n",
       "           94.054   desz    42.05      6.783    27.5    ab\n",
       "           98.113   desr    35.36      3.947    27.5    ab\n",
       "          102.039   desz    20.86      4.482    27.5    ab\n",
       "          111.039   desr    13.68      2.204    27.5    ab\n",
       "          111.058   desz    15.24      3.678    27.5    ab\n",
       "          123.031   desr   -3.385      6.188    27.5    ab\n",
       "          124.023   desz    35.55      13.47    27.5    ab}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def drop_lines(dataset, drop_fractions):\n",
    "    #inserting all supernovae into a single table\n",
    "    for supernova in dataset.data.keys():\n",
    "        new_sna=dataset.data[supernova].copy()\n",
    "        new_col=Table.Column(name='supernova', data=np.repeat(supernova, len(dat.data[supernova])))\n",
    "        new_sna.add_column(new_col, index=0)\n",
    "        if supernova==dataset.data.keys()[0]:\n",
    "            data_table=new_sna\n",
    "        else:\n",
    "            data_table=vstack([data_table, new_sna])\n",
    "    #in each band, determine the number of lines to drop, and drop them        \n",
    "    for band in unique(data_table, keys='filter')['filter']:\n",
    "        band_indices=np.where(data_table['filter']==band)\n",
    "        num_drops=int(np.floor(len(data_table[band_indices])*drop_fractions[band]))\n",
    "        if num_drops > 0:\n",
    "            lines_to_drop=np.random.choice(band_indices[0], num_drops, replace=False)\n",
    "        else:\n",
    "            lines_to_drop=[]\n",
    "        data_table.remove_rows(lines_to_drop)\n",
    "    #make copy of the original dataset, place rows from data_table in there\n",
    "    for supernova in dataset.data.keys():\n",
    "        dataset.data[supernova]=data_table[data_table['supernova']==supernova]\n",
    "        dataset.data[supernova].remove_column('supernova')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def produce_mock_data_set(drop_fractions, degrading_factor, drop_points=True, degrade_errors=False, realsperlc=100, prototypes_Ia=['DES_SN002542.DAT'], prototypes_II=['DES_SN002457.DAT'], prototypes_Ibc=['DES_SN005399.DAT']):\n",
    "    #Data root\n",
    "    dat=sndata.Dataset(rt)\n",
    "\n",
    "    types=dat.get_types()\n",
    "    types['Type'][np.floor(types['Type']/10)==2]=2\n",
    "    types['Type'][np.floor(types['Type']/10)==3]=3\n",
    "                             \n",
    "    _    \n",
    "    for prot in range(len(prototypes_II)):\n",
    "        type_II = []\n",
    "        if degrade_errors:\n",
    "            for i in range(len(dat.data[prototypes_II[prot]]['flux'])):\n",
    "                type_II.append(np.random.normal(dat.data[prototypes_II[prot]]['flux'][i], dat.data[prototypes_II[prot]]['flux_error'][i]*degrading_factor, realsperlc))\n",
    "        type_II = np.array(type_II)\n",
    "        filename_II = 'SPCC_SUBSET/'+prototypes_II[prot]\n",
    "        test_table_II = dat.data[prototypes_II[prot]]\n",
    "        test_table_II.rename_column('flux', 'FLUXCAL')\n",
    "        test_table_II.rename_column('flux_error', 'FLUXCALERR')\n",
    "        col_II = Table.Column(name='field',data=np.zeros(len(test_table_II)) )\n",
    "        test_table_II.add_column(col_II, index = 2)\n",
    "        for i in range(realsperlc):\n",
    "            test_table_II.replace_column('FLUXCAL', type_II[:,i])\n",
    "            if degrade_errors:\n",
    "                test_table_II.replace_column('FLUXCALERR', test_table_II['FLUXCALERR']*degrading_factor)\n",
    "            make_new_data(filename_II, test_table_II, 'II_%s_%s'%(i,prototypes_II[prot]), 2)\n",
    "            logfile.write('II_'+str(i)+'_'+prototypes_II[prot]+'\\n')\n",
    "\n",
    "    for prot in range(len(prototypes_Ia)):\n",
    "        type_Ia = []\n",
    "        if degrade_errors:\n",
    "            for i in range(len(dat.data[prototypes_Ia[prot]]['flux'])):\n",
    "                type_Ia.append(np.random.normal(dat.data[prototypes_Ia[prot]]['flux'][i], dat.data[prototypes_Ia[prot]]['flux_error'][i]*degrading_factor, realsperlc))\n",
    "        type_Ia = np.array(type_Ia)\n",
    "        filename_Ia = 'SPCC_SUBSET/'+prototypes_Ia[prot]\n",
    "        test_table_Ia = dat.data[prototypes_Ia[prot]]\n",
    "        test_table_Ia.rename_column('flux', 'FLUXCAL')\n",
    "        test_table_Ia.rename_column('flux_error', 'FLUXCALERR')\n",
    "        col_Ia = Table.Column(name='field',data=np.zeros(len(test_table_Ia)) )\n",
    "        test_table_Ia.add_column(col_Ia, index = 2)\n",
    "        for i in range(realsperlc):\n",
    "            test_table_Ia.replace_column('FLUXCAL', type_Ia[:,i])\n",
    "            if degrade_errors:\n",
    "                test_table_Ia.replace_column('FLUXCALERR', test_table_Ia['FLUXCALERR']*degrading_factor)\n",
    "            make_new_data(filename_Ia, test_table_Ia, 'Ia_%s_%s'%(i,prototypes_Ia[prot]), 1)\n",
    "            logfile.write('Ia_'+str(i)+'_'+prototypes_Ia[prot]+'\\n')\n",
    "        \n",
    "    for prot in range(len(prototypes_Ibc)):\n",
    "        type_Ibc = []\n",
    "        if degrade_errors:\n",
    "            for i in range(len(dat.data[prototypes_Ibc[prot]]['flux'])):\n",
    "                type_Ibc.append(np.random.normal(dat.data[prototypes_Ibc[prot]]['flux'][i], dat.data[prototypes_Ibc[prot]]['flux_error'][i]*degrading_factor, realsperlc))\n",
    "        type_Ibc = np.array(type_Ibc)\n",
    "        filename_Ibc = 'SPCC_SUBSET/'+prototypes_Ibc[prot]\n",
    "        test_table_Ibc = dat.data[prototypes_Ibc[prot]]\n",
    "        test_table_Ibc.rename_column('flux', 'FLUXCAL')\n",
    "        test_table_Ibc.rename_column('flux_error', 'FLUXCALERR')\n",
    "        col_Ibc = Table.Column(name='field',data=np.zeros(len(test_table_Ibc)) )\n",
    "        test_table_Ibc.add_column(col_Ibc, index = 3)\n",
    "        for i in range(realsperlc):\n",
    "            test_table_Ibc.replace_column('FLUXCAL', type_Ibc[:,i])\n",
    "            if degrade_errors:\n",
    "                test_table_Ibc.replace_column('FLUXCALERR', test_table_Ibc['FLUXCALERR']*degrading_factor)\n",
    "            make_new_data(filename_Ibc, test_table_Ibc, 'Ibc_%s_%s'%(i,prototypes_Ibc[prot]), 3)\n",
    "            logfile.write('Ibc_'+str(i)+'_'+prototypes_Ibc[prot]+'\\n')\n",
    "\n",
    "    logfile.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#produce_mock_data_set(1.1, 111, prototypes_Ia, prototypes_II, prototypes_Ibc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def AUC_from_mock_data_set(classifiers=['nb','knn','svm','neural_network','boost_dt']):\n",
    "    rt1=os.path.join('new_mocks_decrease_cadence','')\n",
    "    dat1=sndata.Dataset(rt1)\n",
    "    for obj in dat1.object_names:\n",
    "        for i in range(len(dat1.data[obj])):\n",
    "            dat1.data[obj]['filter'][i]=dat1.data[obj]['filter'][i][3:7]\n",
    "    types=dat1.get_types()\n",
    "    types['Type'][np.floor(types['Type']/10)==2]=2\n",
    "    types['Type'][np.floor(types['Type']/10)==3]=3\n",
    "    %%capture --no-stdout\n",
    " \n",
    "    mod1Feats=snfeatures.ParametricFeatures('newling',sampler='leastsq')\n",
    "    mod1_features=mod1Feats.extract_features(dat1,nprocesses=4,chain_directory=out_int)\n",
    "    mod1_features.write('%s_newling.dat' %run_name, format='ascii')\n",
    "    #Unfortunately, sometimes the fitting methods return NaN for some parameters for these models.\n",
    "    for c in mod1_features.colnames[1:]:\n",
    "        mod1_features[c][np.isnan(mod1_features[c])]=0\n",
    "    mod1Feats.fit_sn\n",
    "    dat1.set_model(mod1Feats.fit_sn,mod1_features)\n",
    "    \n",
    "    AUC=[]\n",
    "    nprocesses=4\n",
    "    return_classifier=False\n",
    "    columns=[]\n",
    "    training_set=0.7\n",
    "    param_dict={}\n",
    "    scale=True\n",
    "    \n",
    "    if isinstance(mod1_features,Table):\n",
    "        #The features are in astropy table format and must be converted to a numpy array before passing to sklearn\n",
    "\n",
    "        #We need to make sure we match the correct Y values to X values. The safest way to do this is to make types an\n",
    "        #astropy table as well.\n",
    "\n",
    "        if not isinstance(types,Table):\n",
    "            types=Table(data=[mod1_features['Object'],types],names=['Object','Type'])\n",
    "        feats=join(mod1_features,types,'Object')\n",
    "\n",
    "        if len(columns)==0:\n",
    "            columns=feats.columns[1:-1]\n",
    "\n",
    "        #Split into training and validation sets\n",
    "        if np.isscalar(training_set):\n",
    "            objs=feats['Object']\n",
    "            objs=np.random.permutation(objs)\n",
    "            training_set=objs[:(int)(training_set*len(objs))]\n",
    "\n",
    "        #Otherwise a training set has already been provided as a list of Object names and we can continue\n",
    "        feats_train=feats[np.in1d(feats['Object'],training_set)]\n",
    "        feats_test=feats[~np.in1d(feats['Object'],training_set)]\n",
    "\n",
    "        X_train=np.array([feats_train[c] for c in columns]).T\n",
    "        y_train=np.array(feats_train['Type'])\n",
    "        X_test=np.array([feats_test[c] for c in columns]).T\n",
    "        y_test=np.array(feats_test['Type'])\n",
    "\n",
    "    else:\n",
    "        #Otherwise the features are already in the form of a numpy array\n",
    "        if np.isscalar(training_set):\n",
    "            inds=np.random.permutation(range(len(features)))\n",
    "            train_inds=inds[:(int)(len(inds)*training_set)]\n",
    "            test_inds=inds[(int)(len(inds)*training_set):]\n",
    "\n",
    "        else:\n",
    "            #We assume the training set has been provided as indices\n",
    "            train_inds=training_set\n",
    "            test_inds=range(len(types))[~np.in1d(range(len(types)),training_set)]\n",
    "\n",
    "        X_train=mod1_features[train_inds]\n",
    "        y_train=types[train_inds]\n",
    "        X_test=mod1_features[test_inds]\n",
    "        y_test=types[test_inds]\n",
    "\n",
    "\n",
    "    #Rescale the data (highly recommended)\n",
    "    if scale:\n",
    "        scaler = sklearn.preprocessing.StandardScaler()\n",
    "        scaler.fit(np.vstack((X_train, X_test)))\n",
    "        X_train = scaler.transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "\n",
    "    probabilities={}\n",
    "    classifier_objects={}\n",
    "\n",
    "    if nprocesses>1 and return_classifier:\n",
    "        print \"Due to limitations with python's multiprocessing module, classifier objects cannot be returned if \" \\\n",
    "              \"multiple processors are used. Continuing serially...\"\n",
    "        print\n",
    "\n",
    "    if nprocesses>1 and not return_classifier:\n",
    "        partial_func=partial(snclassifier.__call_classifier,X_train=X_train, y_train=y_train, X_test=X_test,\n",
    "                             param_dict=param_dict,return_classifier=False)\n",
    "        p=Pool(nprocesses)\n",
    "        result=p.map(partial_func,classifiers)\n",
    "\n",
    "        for i in range(len(result)):\n",
    "            cls=classifiers[i]\n",
    "            probabilities[cls]=result[i]\n",
    "\n",
    "    else:\n",
    "        for cls in classifiers:\n",
    "            probs,clf=snclassifier.__call_classifier(cls, X_train, y_train, X_test, param_dict,return_classifier)\n",
    "            probabilities[cls]=probs\n",
    "            if return_classifier:\n",
    "                classifier_objects[cls]=clf\n",
    "\n",
    "    for i in range(len(classifiers)):\n",
    "        cls=classifiers[i]\n",
    "        probs=probabilities[cls]\n",
    "        fpr, tpr, auc=snclassifier.roc(probs, y_test, true_class=1)\n",
    "        AUC.append(auc)\n",
    "    return AUC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "classifiers=['nb','svm','boost_dt']\n",
    "#replace model\n",
    "#run with full size data set\n",
    "#put classifier names into plot\n",
    "auc_allclass={}\n",
    "\n",
    "for cl in classifiers:\n",
    "    auc_grid=[]\n",
    "    for i in range(11):\n",
    "        produce_mock_data_set(1.+2.*i/10., 111, prototypes_Ia, prototypes_II, prototypes_Ibc)\n",
    "        auc_grid=np.append(auc_grid, AUC_from_mock_data_set([cl]));\n",
    "    auc_allclass[cl]=auc_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "auc_allclass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for cl in classifiers:\n",
    "    plt.plot(np.linspace(1., 3., 11), auc_allclass[cl])\n",
    "plt.legend(auc_allclass)\n",
    "plt.xlim([1.,3.])\n",
    "plt.show()\n",
    "plt.savefig('blah.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dat1.plot_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "tsne_plot.plot(mod1_features,join(mod1_features,types)['Type'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "clss=snclassifier.run_pipeline(mod1_features,types,output_name=os.path.join(out_class,'model1'),\n",
    "                          classifiers=['nb','svm','boost_dt'], nprocesses=4)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "waveFeats=snfeatures.WaveletFeatures()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "%%capture --no-stdout\n",
    "if read_from_file:\n",
    "    wave_features=Table.read('%s_wavelets.dat' %run_name, format='ascii')\n",
    "    #Crucial for this format of id's\n",
    "    blah=wave_features['Object'].astype(str)\n",
    "    wave_features.replace_column('Object', blah)\n",
    "    PCA_vals=np.loadtxt('%s_wavelets_PCA_vals.dat' %run_name)\n",
    "    PCA_vec=np.loadtxt('%s_wavelets_PCA_vec.dat' %run_name)\n",
    "    PCA_mean=np.loadtxt('%s_wavelets_PCA_mean.dat' %run_name)\n",
    "else:\n",
    "    wave_features=waveFeats.extract_features(dat,nprocesses=6,output_root=out_int,save_output='all')\n",
    "    wave_features.write('%s_wavelets.dat' %run_name, format='ascii')\n",
    "    np.savetxt('%s_wavelets_PCA_vals.dat' %run_name,waveFeats.PCA_eigenvals)\n",
    "    np.savetxt('%s_wavelets_PCA_vec.dat' %run_name,waveFeats.PCA_eigenvectors)\n",
    "    np.savetxt('%s_wavelets_PCA_mean.dat' %run_name,waveFeats.PCA_mean)\n",
    "    \n",
    "    PCA_vals=waveFeats.PCA_eigenvals\n",
    "    PCA_vec=waveFeats.PCA_eigenvectors\n",
    "    PCA_mean=waveFeats.PCA_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.sort(dat.object_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dat.set_model(waveFeats.fit_sn,wave_features,PCA_vec,PCA_mean,0,dat.get_max_length(),dat.filter_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#dat.plot_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#wave_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#join(wave_features, types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#plt.figure()\n",
    "#tsne_plot.plot(wave_features,join(wave_features,types)['Type'])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "#This is a Features object, not yet the extracted features\n",
    "#We need to register the LSST bands with sncosmo (which is why lsst_bands=True)\n",
    "salt2Feats=snfeatures.TemplateFeatures(sampler='leastsq') \n",
    "\n",
    "#This performs the actual feature extraction. All feature extraction methods are parallelised with multiprocessing, \n",
    "#just set nprocesses>1 for parallelisation.\n",
    "if read_from_file:\n",
    "    salt2_features=Table.read('%s_templates.dat' %run_name, format='ascii')\n",
    "    blah=salt2_features['Object'].astype(str)\n",
    "    salt2_features.replace_column('Object', blah)\n",
    "else:\n",
    "    salt2_features=salt2Feats.extract_features(dat,use_redshift=True,nprocesses=6,chain_directory=out_int)\n",
    "    salt2_features.write('%s_templates.dat' %run_name, format='ascii')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#This code takes the fitted parameters and generates the model light curve for plotting purposes.\n",
    "#dat.set_model(salt2Feats.fit_sn,salt2_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#dat.plot_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#dat.data[dat.object_names[5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}