diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..79a16af7e --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 \ No newline at end of file diff --git a/tom_base/settings.py.bak b/tom_base/settings.py.bak new file mode 100644 index 000000000..3636dfa82 --- /dev/null +++ b/tom_base/settings.py.bak @@ -0,0 +1,211 @@ +""" +Django settings for tom_base project. + +Generated by 'django-admin startproject' using Django 2.0.6. + +For more information on this file, see +https://docs.djangoproject.com/en/2.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.0/ref/settings/ +""" + +import os +import tempfile + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'dxja^_6p35x46dx0rx+c$(^31(10^n(twe1#ax3o8xl=n^p37q' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + 'django_extensions', + 'tom_common', + 'django_comments', + 'bootstrap4', + 'crispy_forms', + 'django_filters', + 'django_gravatar', + 'tom_targets', + 'tom_alerts', + 'tom_catalogs', + 'tom_observations', + 'tom_dataproducts', +] + +SITE_ID = 1 + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'tom_common.middleware.ExternalServiceMiddleware', + 'tom_common.middleware.AuthStrategyMiddleware', +] + +ROOT_URLCONF = 'tom_common.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +CRISPY_TEMPLATE_PACK = 'bootstrap4' + +WSGI_APPLICATION = 'tom_base.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = '/' + + +# Internationalization +# https://docs.djangoproject.com/en/2.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = False + +USE_TZ = True + +DATETIME_FORMAT = 'Y-m-d H:m:s' +DATE_FORMAT = 'Y-m-d' + + +# Caching +# https://docs.djangoproject.com/en/dev/topics/cache/#filesystem-caching + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': tempfile.gettempdir() + } +} + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.0/howto/static-files/ + +STATIC_URL = '/static/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'data') +MEDIA_URL = '/data/' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + '': { + 'handlers': ['console'], + 'level': 'INFO' + } + } +} + +TARGET_TYPE = 'NON_SIDEREAL' +FACILITIES = { + 'LCO': { + 'portal_url': 'https://observe.lco.global', + 'api_key': 'a4be47b88d02d69cbb2552f9699a223d4df84e8c', + } +} + +TOM_ALERT_CLASSES = [ + 'tom_alerts.brokers.mars.MARSBroker', + 'tom_alerts.brokers.lasair.LasairBroker' +] + +# Authentication strategy can either be LOCKED (required login for all views) +# or READ_ONLY (read only access to views) +AUTH_STRATEGY = 'READ_ONLY' + +# URLs that should be allowed access even with AUTH_STRATEGY = LOCKED +# for example: OPEN_URLS = ['/', '/about'] +OPEN_URLS = [] + +HOOKS = { + 'target_post_save': 'tom_common.hooks.target_post_save', + 'observation_change_state': 'tom_common.hooks.observation_change_state' +} + +DATA_TYPES = ( + ('SPECTROSCOPY', 'Spectroscopy'), + ('PHOTOMETRY', 'Photometry') +) + +try: + from local_settings import * # noqa +except ImportError: + pass diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 2df9fdd1c..7ba09aaf1 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -1,6 +1,6 @@ from django import forms -from .models import DataProductGroup, DataProduct +from .models import DataProductGroup, DataProduct, SPECTROSCOPY from tom_targets.models import Target from tom_observations.models import ObservationRecord @@ -29,4 +29,31 @@ class DataProductUploadForm(forms.Form): attrs={'multiple': True} ) ) - tag = forms.ChoiceField(choices=DataProduct.DATA_PRODUCT_TAGS) + tag = forms.ChoiceField(choices=DataProduct.DATA_PRODUCT_TYPES) + observation_timestamp = forms.SplitDateTimeField( + label='Observation Time', + widget=forms.SplitDateTimeWidget( + date_attrs={'placeholder': 'Observation Date', 'type': 'date'}, + time_attrs={'format': '%H:%M:%S', 'placeholder': 'Observation Time', + 'type': 'time', 'step': '1'} + ), + required=False + ) + referrer = forms.CharField( + widget=forms.HiddenInput() + ) + + def __init__(self, *args, **kwargs): + hide_timestamp = kwargs.pop('hide_timestamp', False) + super(DataProductUploadForm, self).__init__(*args, **kwargs) + if hide_timestamp: + self.fields['observation_timestamp'].widget = forms.HiddenInput() + + def clean(self): + cleaned_data = super().clean() + print(cleaned_data) + if cleaned_data.get('tag', '') != SPECTROSCOPY[0] and cleaned_data.get('observation_timestamp'): + raise forms.ValidationError('Observation timestamp is not valid for uploaded photometry') + # elif not cleaned_data.get('observation_timestamp'): + # raise forms.ValidationError('Observation timestamp is required for spectroscopy') + return cleaned_data \ No newline at end of file diff --git a/tom_dataproducts/management/commands/downloaddata.py b/tom_dataproducts/management/commands/downloaddata.py new file mode 100644 index 000000000..c764dd1fe --- /dev/null +++ b/tom_dataproducts/management/commands/downloaddata.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand +from tom_observations import facility +from tom_observations.models import ObservationRecord + + +class Command(BaseCommand): + help = 'Downloads data for all completed observations' + + def handle(self, *args, **options): + facility_classes = {} + for facility_name in facility.get_service_classes(): + facility_classes[facility_name] = facility.get_service_class(facility_name)() + observation_records = ObservationRecord.objects.all() + for record in observation_records: + if record.status not in facility_classes[record.facility].get_terminal_observing_states(): + facility_classes[record.facility].update_observation_status(record.observation_id) + facility_classes[record.facility].save_data_products(record) + + return 'completed command' \ No newline at end of file diff --git a/tom_dataproducts/migrations/0002_auto_20190515_0047.py b/tom_dataproducts/migrations/0002_auto_20190515_0047.py new file mode 100644 index 000000000..a69ab3e2f --- /dev/null +++ b/tom_dataproducts/migrations/0002_auto_20190515_0047.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.1 on 2019-05-15 00:47 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_dataproducts', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='reduceddatum', + name='timestamp', + field=models.DateTimeField(db_index=True, default=datetime.datetime.now), + ), + ] diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index b24baaae1..4bceceaa7 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -1,7 +1,9 @@ -from django.db import models +import os from io import BytesIO from base64 import b64encode -import os +from datetime import datetime + +from django.db import models from django.conf import settings import matplotlib @@ -40,7 +42,7 @@ def __str__(self): class DataProduct(models.Model): - DATA_PRODUCT_TAGS = ( + DATA_PRODUCT_TYPES = ( PHOTOMETRY, FITS_FILE, SPECTROSCOPY @@ -59,7 +61,7 @@ class DataProduct(models.Model): group = models.ManyToManyField(DataProductGroup) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - tag = models.CharField(max_length=50, blank=True, default='', choices=DATA_PRODUCT_TAGS) + tag = models.CharField(max_length=50, blank=True, default='', choices=DATA_PRODUCT_TYPES) featured = models.BooleanField(default=False) class Meta: @@ -106,5 +108,5 @@ class ReducedDatum(models.Model): ) source_name = models.CharField(max_length=100, default='') source_location = models.CharField(max_length=200, default='') - timestamp = models.DateTimeField(null=False, blank=False, db_index=True) + timestamp = models.DateTimeField(null=False, blank=False, default=datetime.now, db_index=True) value = models.TextField(null=False, blank=False) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html b/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html similarity index 79% rename from tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html rename to tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html index d9041702c..b1c8c4fc8 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html +++ b/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html @@ -1,6 +1,7 @@ {% load bootstrap4 %} {% if user.is_authenticated %}

Upload a data product for a target

+{% if response.GET.errors %}There was an error with your upload: {{ response.GET.errors }}{% endif %}
{% csrf_token %} {% bootstrap_form data_product_form %} diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 51041c0e5..0061ebd69 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -1,6 +1,7 @@ import json from django import template +from datetime import datetime from plotly import offline import plotly.graph_objs as go @@ -40,25 +41,6 @@ def dataproduct_list_all(saved, fields): return {'products': products} -@register.inclusion_tag('tom_dataproducts/partials/upload_dataproduct.html', takes_context=True) -def upload_dataproduct(context): - model_instance = context.get('object', None) - object_key = '' - if type(model_instance) == Target: - object_key = 'target' - elif type(model_instance) == ObservationRecord: - object_key = 'observation_record' - form = context.get( - 'data_product_form', - DataProductUploadForm(initial={object_key: model_instance}) - ) - user = context.get('user', None) - return { - 'data_product_form': form, - 'user': user - } - - @register.inclusion_tag('tom_dataproducts/partials/photometry_for_target.html') def photometry_for_target(target): photometry_data = {} @@ -97,17 +79,19 @@ def spectroscopy_for_target(target, dataproduct=None): if dataproduct: spectral_dataproducts = DataProduct.objects.get(dataproduct=dataproduct) for data in spectral_dataproducts: - datum = json.loads(ReducedDatum.objects.get(data_product=data).value) + datum = ReducedDatum.objects.get(data_product=data) + datum_value = json.loads(datum.value) wavelength = [] flux = [] - for key, value in datum.items(): + for key, value in datum_value.items(): wavelength.append(value['wavelength']) flux.append(float(value['flux'])) - spectra.append((wavelength, flux)) + spectra.append((wavelength, flux, datetime.strftime(datum.timestamp, '%Y-%m-%d %H:%M:%S'))) plot_data = [ go.Scatter( x=spectrum[0], - y=spectrum[1] + y=spectrum[1], + name=spectrum[2] ) for spectrum in spectra] layout = go.Layout( height=600, diff --git a/tom_dataproducts/utils.py b/tom_dataproducts/utils.py index 58651c8e6..017a03a01 100644 --- a/tom_dataproducts/utils.py +++ b/tom_dataproducts/utils.py @@ -2,12 +2,11 @@ import json from astropy.time import Time -from datetime import datetime from .models import ReducedDatum -from django.conf import settings -def process_data_product(data_product, target): +def process_data_product(data_product, target, timestamp=None): + # TODO: Validate data if data_product.tag == 'photometry': with data_product.data.file.open() as f: for line in f: @@ -42,6 +41,6 @@ def process_data_product(data_product, target): target=target, data_product=data_product, data_type=data_product.tag, - timestamp=datetime.now(), + timestamp=timestamp, value=json.dumps(spectrum) ) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 31b9a5e64..a54f6050d 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -14,6 +14,7 @@ from django.core.management import call_command from django.core.cache import cache from django.core.cache.utils import make_template_fragment_key +from django.core.exceptions import NON_FIELD_ERRORS from django.http import HttpResponseRedirect from guardian.shortcuts import get_objects_for_user @@ -54,32 +55,41 @@ def post(self, request, *args, **kwargs): class DataProductUploadView(LoginRequiredMixin, FormView): form_class = DataProductUploadForm - template_name = 'tom_dataproducts/partials/upload_dataproduct.html' - def post(self, request, *args, **kwargs): - form = self.get_form() - if form.is_valid(): - target = form.cleaned_data['target'] - if not target: - observation_record = form.cleaned_data['observation_record'] - target = observation_record.target - else: - observation_record = None - tag = form.cleaned_data['tag'] - data_product_files = request.FILES.getlist('files') - for f in data_product_files: - dp = DataProduct( - target=target, - observation_record=observation_record, - data=f, - product_id=None, - tag=tag - ) - dp.save() - process_data_product(dp, target) - return HttpResponseRedirect(request.META.get('HTTP_REFERER', '/')) + def form_valid(self, form): + print('form valid') + target = form.cleaned_data['target'] + observation_timestamp = form.cleaned_data.get('observation_timestamp', None) + if not target: + observation_record = form.cleaned_data['observation_record'] + target = observation_record.target else: - return super().form_invalid(form) + observation_record = None + tag = form.cleaned_data['tag'] + data_product_files = self.request.FILES.getlist('files') + for f in data_product_files: + dp = DataProduct( + target=target, + observation_record=observation_record, + data=f, + product_id=None, + tag=tag + ) + dp.save() + # try: + process_data_product(dp, target, timestamp=observation_timestamp) + # except: # TODO: more specific exception + # dp.delete() + # form.add_error(NON_FIELD_ERRORS, "Uploaded file used an invalid format. Please consult the docs.") + # return super().form_invalid(form) + return redirect(form.cleaned_data.get('referrer', '/')) + + + # TODO: ensure fits files can have obsv dates + # TODO: ensure photometry can't have dates + def form_invalid(self, form): + messages.error(self.request, 'There was a problem uploading your file: {}'.format(form.errors)) + return redirect(form.cleaned_data.get('referrer', '/')) class DataProductDeleteView(LoginRequiredMixin, DeleteView): diff --git a/tom_dataproducts/widgets.py b/tom_dataproducts/widgets.py new file mode 100644 index 000000000..c39cfaefc --- /dev/null +++ b/tom_dataproducts/widgets.py @@ -0,0 +1,23 @@ +from datetime import date +from django.forms import widgets + +class ObservationDateTimeWidget(widgets.SplitDateTimeWidget): + def __init__(self, attrs=None): + date_attrs = attrs + time_attrs = attrs + date_attrs['label'] = attrs.get('date-label', 'Observation Date') + time_attrs['label'] = attrs.get('time-label', 'Observation Time') + _widgets = ( + widgets.DateInput(attrs=date_attrs), + widgets.TimeInput(attrs=time_attrs) + ) + super().__init__(_widgets, attrs) + + def decompress(self, value): + if value: + return [value.date, value.time] + return [None, None] + + def compress(self, data_list): + if data_list: + return diff --git a/tom_observations/templates/tom_observations/observationrecord_detail.html b/tom_observations/templates/tom_observations/observationrecord_detail.html index 576fc7353..653b697c8 100644 --- a/tom_observations/templates/tom_observations/observationrecord_detail.html +++ b/tom_observations/templates/tom_observations/observationrecord_detail.html @@ -11,7 +11,7 @@

{{ object }} View at obse

Created: {{ object.created }} Modified: {{ object.modified }}

Status: {{ object.status }}

- {% upload_dataproduct %} + {% include 'tom_dataproducts/upload_dataproduct.html' %}
{% if image %} diff --git a/tom_observations/views.py b/tom_observations/views.py index 7c5817259..ecb891662 100644 --- a/tom_observations/views.py +++ b/tom_observations/views.py @@ -12,7 +12,7 @@ from .models import ObservationRecord from .forms import ManualObservationForm -from tom_dataproducts.forms import AddProductToGroupForm +from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_targets.models import Target from tom_observations.facility import get_service_class @@ -159,4 +159,12 @@ def get_context_data(self, *args, **kwargs): data_product.get_file_extension() == '.fits' else newest_image if newest_image: context['image'] = newest_image.get_image_data() + data_product_upload_form = DataProductUploadForm( + initial={ + 'observation_record': self.get_object(), + 'referrer': reverse('tom_observations:detail', args=(self.get_object().id,)) + }, + hide_timestamp=True + ) + context['data_product_form'] = data_product_upload_form return context diff --git a/tom_targets/migrations/0007_auto_20190515_0047.py b/tom_targets/migrations/0007_auto_20190515_0047.py new file mode 100644 index 000000000..b811ed16c --- /dev/null +++ b/tom_targets/migrations/0007_auto_20190515_0047.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.1 on 2019-05-15 00:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_targets', '0006_auto_20190403_1659'), + ] + + operations = [ + migrations.AlterField( + model_name='target', + name='epoch_of_perihelion', + field=models.FloatField(blank=True, help_text='Julian Date.', null=True, verbose_name='Epoch of Perihelion'), + ), + ] diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 7a0b0599a..30f4ba3ac 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -45,7 +45,7 @@

Observations

{% photometry_for_target target %} - {% upload_dataproduct %} + {% include 'tom_dataproducts/upload_dataproduct.html' %} {% dataproduct_list_for_target object %} {% spectroscopy_for_target target %}
diff --git a/tom_targets/views.py b/tom_targets/views.py index befa499f8..09914d99e 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -14,6 +14,7 @@ from guardian.shortcuts import get_objects_for_user, get_groups_with_perms from .models import Target +from tom_dataproducts.forms import DataProductUploadForm from .forms import SiderealTargetCreateForm, NonSiderealTargetCreateForm from .import_targets import import_targets from .filters import TargetFilter @@ -114,6 +115,18 @@ class TargetDetailView(PermissionRequiredMixin, DetailView): permission_required = 'tom_targets.view_target' model = Target + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + data_product_upload_form = DataProductUploadForm( + initial={ + 'target': self.get_object(), + 'referrer': reverse('tom_targets:detail', args=(self.get_object().id,)) + }, + hide_timestamp=False + ) + context['data_product_form'] = data_product_upload_form + return context + def get(self, request, *args, **kwargs): update_status = request.GET.get('update_status', False) if update_status: