Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DRAFT - Implementing automatic download of templates for GW candidates #17

Open
wants to merge 21 commits into
base: master
Choose a base branch
from
Open

DRAFT - Implementing automatic download of templates for GW candidates #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
f4ead6f
something
Mar 8, 2024
c502140
Added functionlity to automatically fetch templates to be subtracted
gterreran Mar 22, 2024
0d46b94
fixing hooks
gterreran Mar 22, 2024
886d019
Changed templated folder to be defined by the environmental viariable…
gterreran Apr 2, 2024
c0eff4f
Changed DECam COLLECTION to ls_dr9
gterreran Apr 2, 2024
cfa8627
Added full functionality of SDSS, including variance image, but the r…
gterreran Apr 18, 2024
6f2ae56
broken
gterreran May 1, 2024
88b02ec
working on it
gterreran May 2, 2024
b3f9443
Full PanSTARRS and SDSS template retrieval complete.
gterreran May 14, 2024
9d2eb93
Full PanSTARRS and SDSS template retrieval complete.
gterreran May 14, 2024
3b5d010
Restructured scripts
gterreran May 20, 2024
5b84c02
Edited the views and the hooks to update all o4_galaxy targets at onc…
gterreran May 22, 2024
d08a6b0
Made small changes to make this version runnable locally
gterreran May 22, 2024
ec31233
Updated settings.py and requirements.txt to use asynchronous calls. T…
gterreran Jun 4, 2024
8f52ded
Removed some redundancies
gterreran Jun 4, 2024
a39f621
Fixed some import definitions
gterreran Jun 4, 2024
d50a4c5
bug fix
gterreran Jun 10, 2024
71545f2
Added telescopeid and instrumentid to the SURVEYS dictionary. Working…
gterreran Jun 12, 2024
7c222b1
tmp
gterreran Jun 12, 2024
aa4afc9
tmp, not working
gterreran Jun 13, 2024
aa5920c
added triplets functionality for GW observations with performed subtr…
gterreran Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,5 +109,9 @@ venv.bak/
src/
custom_code/brokers/queries/queries.json
gw/gw_config.ini
<<<<<<< HEAD

db_data
=======
>>>>>>> 2cdcce4 (Added functionlity to automatically fetch templates to be subtracted)
galaxy_database
170 changes: 120 additions & 50 deletions custom_code/facilities/gemini_facility.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,24 +39,53 @@ def get_site_code_from_program(program_id):

class OpticalImagingForm(BaseObservationForm):

north_south_choice = (
('north', 'North'),
('south', 'South'),
)
n_or_s = forms.ChoiceField(choices=north_south_choice, initial='north', widget=forms.Select(), required=True,
label='')

window_size = forms.FloatField(initial=1.0, min_value=0.0, label='')
max_airmass = forms.FloatField(min_value=1.0, max_value=5.0, initial=1.6, label='')

# Optical imaging
optical_phot_exptime_choices = (
(0, 0.0),
(100, 100.0),
(200, 200.0),
(300, 300.0),
(450, 450.0),
(600, 600.0),
(900, 900.0),
# optical_phot_exptime_choices = (
# (0, 0.0),
# (100, 100.0),
# (200, 200.0),
# (300, 300.0),
# (450, 450.0),
# (600, 600.0),
# (900, 900.0),
# )
#
# g_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
# r_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
# i_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
# z_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')

g_exptime = forms.FloatField(initial=0.0, min_value=0.0,
required=True, label='')
r_exptime = forms.FloatField(initial=0.0, min_value=0.0,
required=True, label='')
i_exptime = forms.FloatField(initial=0.0, min_value=0.0,
required=True, label='')
z_exptime = forms.FloatField(initial=0.0, min_value=0.0,
required=True, label='')

filter_choices = (
('U','U'),
('B','B'),
('g','g'),
('V','V'),
('r','r'),
('i','i'),
)

g_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
r_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
i_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
z_exptime = forms.ChoiceField(choices=optical_phot_exptime_choices, initial=0, widget=forms.Select(), required=True, label='')
mag_approx = forms.FloatField(initial=18.0, min_value=0.0, label='')
mag_approx_filter = forms.ChoiceField(choices=filter_choices, initial='g', widget=forms.Select(), required=True,
label='')

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
Expand All @@ -72,6 +101,12 @@ def __init__(self, *args, **kwargs):
PrependedText(
'max_airmass', 'Airmass <'
),
Div(
Div(PrependedText('mag_approx', 'Approximate mag: '),css_class='col-md-8'),
Div(HTML('<p style="text-align:center;">Filter:</p>'),css_class='col-md-2'),
Div('mag_approx_filter', css_class='col-md-2'),
css_class='form-row'
),
Div(
Div(HTML('<p style="text-align:center;">Filter</p>'),css_class='col-md-2'),
Div(HTML('<p style="text-align:center;">Exposure time (s)</p>'),css_class='col-md-10'), css_class='form-row',
Expand All @@ -92,6 +127,13 @@ def __init__(self, *args, **kwargs):
Div(HTML('<p style="text-align:center;">z</p>'),css_class='col-md-2'),
Div('z_exptime',css_class='col-md-10'), css_class='form-row'
),
Div(
Div(HTML('<p style="text-align:center;">Status</p>'), css_class='col-md-12'),
css_class='form-row',
),
Div(
Div('ready', css_class='col-md-12'), css_class='form-row'
),
), css_class='col-md-8'
), css_class='row justify-content-md-center'
)
Expand All @@ -108,13 +150,21 @@ def is_valid(self):

def _init_observation_payload(self, target):

wait = True #On Hold
#wait = True #On Hold
coords = SkyCoord(ra=target.ra*u.degree, dec=target.dec*u.degree)
now = datetime.utcnow()
sn_name = target.name

if self.data['n_or_s'] == 'north':
prog = os.getenv('GEMINI_NORTH_PROGRAMID')
pwd = os.getenv('GEMINI_NORTH_PASSWORD')
elif self.data['n_or_s'] == 'south':
prog = os.getenv('GEMINI_SOUTH_PROGRAMID')
pwd = os.getenv('GEMINI_SOUTH_PASSWORD')

payload = {
'ready': str(not wait).lower(),
#'ready': str(not wait).lower(),
'ready': self.data['ready'],
'prog': os.getenv('GEMINI_PROGRAMID',''),
'email': os.getenv('GEMINI_EMAIL',''),
'password': os.getenv('GEMINI_PASSWORD',''),
Expand All @@ -128,7 +178,7 @@ def _init_observation_payload(self, target):
'elevationType': 'airmass',
'elevationMin': 1.0,
'elevationMax': str(self.data['max_airmass']).strip(),
'note': 'API Test',
'note': 'This observation was submitted through the API, if anything is unusual contact [email protected]',
'posangle': 90.
}

Expand Down Expand Up @@ -174,12 +224,12 @@ class OpticalSpectraForm(BaseObservationForm):
window_size = forms.FloatField(initial=1.0, min_value=0.0, label='')
max_airmass = forms.FloatField(min_value=1.0, max_value=5.0, initial=1.6, label='')

optical_spec_exptime_choices = (
(0, 0.0),
(1200, 1200.0),
(1500, 1500.0),
(1700, 1700.0),
)
# optical_spec_exptime_choices = (
# (0, 0.0),
# (1200, 1200.0),
# (1500, 1500.0),
# (1700, 1700.0),
# )

optical_spec_slit_choices = (
(1, '1.0"'),
Expand Down Expand Up @@ -248,7 +298,7 @@ def __init__(self, *args, **kwargs):
css_class='form-row',
),
Div(
Div(HTML('<p style="text-align:center;">B600/500nm</p>'), css_class='col-md-4'),
Div(HTML('<p style="text-align:center;">B480/500nm</p>'), css_class='col-md-4'),
Div('b_exptime', css_class='col-md-8'), css_class='form-row'
),
Div(
Expand Down Expand Up @@ -283,7 +333,7 @@ def is_valid(self):
return not errors

def _init_observation_payload(self, target):
wait = True #On Hold
#wait = True #On Hold
coords = SkyCoord(ra=target.ra*u.degree, dec=target.dec*u.degree)
now = datetime.utcnow()
sn_name = target.name
Expand Down Expand Up @@ -324,55 +374,75 @@ def observation_payload(self):
if self.data['n_or_s'] == 'north':
if self.data['slit'] == '1.5':
obsid_map = {
'B600': {
'arc': '68',
'acquisition': '66',
'science_with_flats': '67',
# 'B600': {
# 'arc': '68',
# 'acquisition': '66',
# 'science_with_flats': '67',
# },
'B480': {
'arc': '104',
'acquisition': '102',
'science_with_flats': '103',
},
'R400': {
'arc': '74',
'acquisition': '72',
'science_with_flats': '73',
'arc': '110',
'acquisition': '108',
'science_with_flats': '109',
}
}
elif self.data['slit'] == '1':
obsid_map = {
'B600': {
'arc': '65',
'acquisition': '63',
'science_with_flats': '64',
# 'B600': {
# 'arc': '65',
# 'acquisition': '63',
# 'science_with_flats': '64',
# },
'B480': {
'arc': '101',
'acquisition': '99',
'science_with_flats': '100',
},
'R400': {
'arc': '71',
'acquisition': '69',
'science_with_flats': '70',
'arc': '107',
'acquisition': '105',
'science_with_flats': '106',
}
}
elif self.data['n_or_s'] == 'south':
if self.data['slit'] == '1.5':
obsid_map = {
'B600': {
'arc': '64',
'acquisition': '65',
'science_with_flats': '66',
'arc': '56',
'acquisition': '57',
'science_with_flats': '58',
},
'B480': {
'arc': '93',
'acquisition': '94',
'science_with_flats': '95',
},
'R400': {
'arc': '70',
'acquisition': '71',
'science_with_flats': '72',
'arc': '62',
'acquisition': '63',
'science_with_flats': '64',
}
}
elif self.data['slit'] == '1':
obsid_map = {
'B600': {
'arc': '61',
'acquisition': '62',
'science_with_flats': '63',
'arc': '53',
'acquisition': '54',
'science_with_flats': '55',
},
'B480': {
'arc': '90',
'acquisition': '91',
'science_with_flats': '92',
},
'R400': {
'arc': '67',
'acquisition': '68',
'science_with_flats': '69',
'arc': '59',
'acquisition': '60',
'science_with_flats': '61',
}
}

Expand All @@ -395,7 +465,7 @@ def observation_payload(self):
payloads[f'{color_grating}_{key}']['target'] = 'Acquisition'
elif key == 'science_with_flats':
payloads[f'{color_grating}_{key}']['target'] = target.name
if color_grating == 'B600':
if color_grating in ['B600', 'B480']:
payloads[f'{color_grating}_{key}']['exptime'] = int(float(self.data['b_exptime'])/2)
elif color_grating == 'R400':
payloads[f'{color_grating}_{key}']['exptime'] = int(float(self.data['r_exptime'])/2)
Expand Down
71 changes: 71 additions & 0 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,21 @@ services:
volumes:
- ${SNEX2_DB_DATA_PATH}:/var/lib/postgresql/data
ports:
<<<<<<< HEAD
- 5435:5432
#platform: linux/amd64
=======
- 5435:5432/tcp
platform: linux/amd64
>>>>>>> 4040213 (something)
snex2:
image: lcogt/snex2:latest
network_mode: bridge
links:
- "snex2-db:snex2-db"
ports:
- 8889:8080
platform: linux/amd64
restart: "always"
mem_limit: "4g"
logging:
Expand Down Expand Up @@ -75,6 +81,7 @@ services:
- ${SNEX_MUSCAT_FITS_PATH}:/snex2/data/fits/2m0a/
- ${SNEX_GW_FITS_PATH}:/snex2/data/fits/gw/
- ${SNEX_FLOYDS_PATH}:/snex2/data/floyds/
<<<<<<< HEAD
- ${SNEX_FLOYDS_WEB_PATH}:/snex2/data/WEB/floyds/
snex2-dev:
image: snex2-dev:latest
Expand Down Expand Up @@ -139,3 +146,67 @@ services:
- ${SNEX_GW_FITS_PATH}:/snex2/data/fits/gw/
- ${SNEX_FLOYDS_PATH}:/snex2/data/floyds/
- ${SNEX_FLOYDS_WEB_PATH}:/snex2/data/WEB/floyds/
=======
# snex2-dev:
# # image: snex2-dev:latest
# network_mode: bridge
# links:
# - "snex2-db:snex2-db"
# ports:
# - 8890:8080
# restart: "always"
# mem_limit: "12g"
# logging:
# options:
# max-file: "3"
# max-size: "10m"
# environment:
# - DB_NAME=snex2-dev
# - DB_HOST=snex2-db
# - DB_PORT=5432
# - SNEX2_USER=${SNEX2_USER}
# - SNEX2_DB_USER=${SNEX2_DB_USER}
# - SNEX2_DB_PASSWORD=${SNEX2_DB_PASSWORD}
# - SNEX1_DB_USER=${SNEX1_DB_USER}
# - SNEX1_DB_PASSWORD=${SNEX1_DB_PASSWORD}
# - SNEX2_DB_BACKEND=postgres
# - LCO_APIKEY=${SNEX2_LCO_APIKEY}
# - SNEXBOT_APIKEY=${SNEX2_SNEXBOT_APIKEY}
# - AWS_ACCESS_KEY_ID=${SNEX2_AWS_ACCESS_KEY_ID}
# - AWS_S3_REGION_NAME=${SNEX2_AWS_REGION_NAME}
# - AWS_SECRET_ACCESS_KEY=${SNEX2_AWS_SECRET_ACCESS_KEY}
# - AWS_STORAGE_BUCKET_NAME=${SNEX2_AWS_STORAGE_BUCKET_NAME}
# - TWITTER_APIKEY=${SNEX2_TWITTER_APIKEY}
# - TWITTER_SECRET=${SNEX2_TWITTER_SECRET}
# - TWITTER_ACCESSTOKEN=${SNEX2_TWITTER_ACCESSTOKEN}
# - TWITTER_ACCESSSECRET=${SNEX2_TWITTER_ACCESSSECRET}
# - GEMINI_EMAIL=${SNEX2_GEMINI_EMAIL}
# - GEMINI_SOUTH_PROGRAMID=${SNEX2_GEMINI_SOUTH_PROGRAMID}
# - GEMINI_SOUTH_PASSWORD=${SNEX2_GEMINI_SOUTH_PASSWORD}
# - GEMINI_SOUTH_SERVER=${SNEX2_GEMINI_SOUTH_SERVER}
# - GEMINI_NORTH_PROGRAMID=${SNEX2_GEMINI_NORTH_PROGRAMID}
# - GEMINI_NORTH_PASSWORD=${SNEX2_GEMINI_NORTH_PASSWORD}
# - GEMINI_NORTH_SERVER=${SNEX2_GEMINI_NORTH_SERVER}
# - SNEX_EMAIL_PASSWORD=${SNEX_EMAIL_PASSWORD}
# - TNS_APIKEY=${TNS_APIKEY}
# - TNS_APIID=${TNS_APIID}
# - SWIFT_USERNAME=${SWIFT_USERNAME}
# - SWIFT_SECRET=${SWIFT_SECRET}
# - LASAIR_IRIS_TOKEN=${LASAIR_IRIS_TOKEN}
# - SCIMMA_AUTH_USERNAME=${SCIMMA_AUTH_USERNAME}
# - SCIMMA_AUTH_PASSWORD=${SCIMMA_AUTH_PASSWORD}
# - GCN_CLASSIC_CLIENT_ID=${GCN_CLASSIC_CLIENT_ID}
# - GCN_CLASSIC_CLIENT_SECRET=${GCN_CLASSIC_CLIENT_SECRET}
# - CREDENTIAL_USERNAME=${CREDENTIAL_USERNAME}
# - CREDENTIAL_PASSWORD=${CREDENTIAL_PASSWORD}
# - TM_TOKEN=${TM_TOKEN}
# - HERMES_BASE_URL=${HERMES_BASE_URL}
# volumes:
# - ${SNEX_THUMBNAIL_PATH}:/snex2/data/thumbs/
# - ${SNEX_FITS_PATH}:/snex2/data/fits/
# - ${SNEX_0m4_FITS_PATH}:/snex2/data/fits/0m4/
# - ${SNEX_2m_FITS_PATH}:/snex2/data/fits/fts/
# - ${SNEX_MUSCAT_FITS_PATH}:/snex2/data/fits/2m0a/
# - ${SNEX_GW_FITS_PATH}:/snex2/data/fits/gw/
# - ${SNEX_FLOYDS_PATH}:/snex2/data/floyds/
>>>>>>> 4040213 (something)
Loading