Added observation time field to dataproduct upload

Refactored upload logic

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120

tom_base/settings.py.bak

@@ -0,0 +1,211 @@
+"""
+Django settings for tom_base project.
+
+Generated by 'django-admin startproject' using Django 2.0.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/2.0/ref/settings/
+"""
+
+import os
+import tempfile
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'dxja^_6p35x46dx0rx+c$(^31(10^n(twe1#ax3o8xl=n^p37q'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.sites',
+    'django_extensions',
+    'tom_common',
+    'django_comments',
+    'bootstrap4',
+    'crispy_forms',
+    'django_filters',
+    'django_gravatar',
+    'tom_targets',
+    'tom_alerts',
+    'tom_catalogs',
+    'tom_observations',
+    'tom_dataproducts',
+]
+
+SITE_ID = 1
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'tom_common.middleware.ExternalServiceMiddleware',
+    'tom_common.middleware.AuthStrategyMiddleware',
+]
+
+ROOT_URLCONF = 'tom_common.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+CRISPY_TEMPLATE_PACK = 'bootstrap4'
+
+WSGI_APPLICATION = 'tom_base.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+LOGIN_REDIRECT_URL = '/'
+LOGOUT_REDIRECT_URL = '/'
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/2.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = False
+
+USE_TZ = True
+
+DATETIME_FORMAT = 'Y-m-d H:m:s'
+DATE_FORMAT = 'Y-m-d'
+
+
+# Caching
+# https://docs.djangoproject.com/en/dev/topics/cache/#filesystem-caching
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': tempfile.gettempdir()
+    }
+}
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/2.0/howto/static-files/
+
+STATIC_URL = '/static/'
+MEDIA_ROOT = os.path.join(BASE_DIR, 'data')
+MEDIA_URL = '/data/'
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        }
+    },
+    'loggers': {
+        '': {
+            'handlers': ['console'],
+            'level': 'INFO'
+        }
+    }
+}
+
+TARGET_TYPE = 'NON_SIDEREAL'
+FACILITIES = {
+    'LCO': {
+        'portal_url': 'https://observe.lco.global',
+        'api_key': 'a4be47b88d02d69cbb2552f9699a223d4df84e8c',
+    }
+}
+
+TOM_ALERT_CLASSES = [
+    'tom_alerts.brokers.mars.MARSBroker',
+    'tom_alerts.brokers.lasair.LasairBroker'
+]
+
+# Authentication strategy can either be LOCKED (required login for all views)
+# or READ_ONLY (read only access to views)
+AUTH_STRATEGY = 'READ_ONLY'
+
+# URLs that should be allowed access even with AUTH_STRATEGY = LOCKED
+# for example: OPEN_URLS = ['/', '/about']
+OPEN_URLS = []
+
+HOOKS = {
+    'target_post_save': 'tom_common.hooks.target_post_save',
+    'observation_change_state': 'tom_common.hooks.observation_change_state'
+}
+
+DATA_TYPES = (
+    ('SPECTROSCOPY', 'Spectroscopy'),
+    ('PHOTOMETRY', 'Photometry')
+)
+
+try:
+    from local_settings import * # noqa
+except ImportError:
+    pass

tom_dataproducts/forms.py

@@ -1,6 +1,6 @@
 from django import forms
 
-from .models import DataProductGroup, DataProduct
+from .models import DataProductGroup, DataProduct, SPECTROSCOPY
 from tom_targets.models import Target
 from tom_observations.models import ObservationRecord
 
@@ -29,4 +29,31 @@ class DataProductUploadForm(forms.Form):
             attrs={'multiple': True}
         )
     )
-    tag = forms.ChoiceField(choices=DataProduct.DATA_PRODUCT_TAGS)
+    tag = forms.ChoiceField(choices=DataProduct.DATA_PRODUCT_TYPES)
+    observation_timestamp = forms.SplitDateTimeField(
+        label='Observation Time',
+        widget=forms.SplitDateTimeWidget(
+            date_attrs={'placeholder': 'Observation Date', 'type': 'date'},
+            time_attrs={'format': '%H:%M:%S', 'placeholder': 'Observation Time',
+                        'type': 'time', 'step': '1'}
+        ),
+        required=False
+    )
+    referrer = forms.CharField(
+        widget=forms.HiddenInput()
+    )
+
+    def __init__(self, *args, **kwargs):
+        hide_timestamp = kwargs.pop('hide_timestamp', False)
+        super(DataProductUploadForm, self).__init__(*args, **kwargs)
+        if hide_timestamp:
+            self.fields['observation_timestamp'].widget = forms.HiddenInput()
+
+    def clean(self):
+        cleaned_data = super().clean()
+        print(cleaned_data)
+        if cleaned_data.get('tag', '') != SPECTROSCOPY[0] and cleaned_data.get('observation_timestamp'):
+            raise forms.ValidationError('Observation timestamp is not valid for uploaded photometry')
+        # elif not cleaned_data.get('observation_timestamp'):
+        #     raise forms.ValidationError('Observation timestamp is required for spectroscopy')
+        return cleaned_data

tom_dataproducts/management/commands/downloaddata.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from tom_observations import facility
+from tom_observations.models import ObservationRecord
+
+
+class Command(BaseCommand):
+    help = 'Downloads data for all completed observations'
+
+    def handle(self, *args, **options):
+        facility_classes = {}
+        for facility_name in facility.get_service_classes():
+            facility_classes[facility_name] = facility.get_service_class(facility_name)()
+        observation_records = ObservationRecord.objects.all()
+        for record in observation_records:
+            if record.status not in facility_classes[record.facility].get_terminal_observing_states():
+                facility_classes[record.facility].update_observation_status(record.observation_id)
+                facility_classes[record.facility].save_data_products(record)
+
+        return 'completed command'

tom_dataproducts/migrations/0002_auto_20190515_0047.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.1 on 2019-05-15 00:47
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tom_dataproducts', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='reduceddatum',
+            name='timestamp',
+            field=models.DateTimeField(db_index=True, default=datetime.datetime.now),
+        ),
+    ]

tom_dataproducts/models.py

@@ -1,7 +1,9 @@
-from django.db import models
+import os
 from io import BytesIO
 from base64 import b64encode
-import os
+from datetime import datetime
+
+from django.db import models
 from django.conf import settings
 
 import matplotlib
@@ -40,7 +42,7 @@ def __str__(self):
 
 
 class DataProduct(models.Model):
-    DATA_PRODUCT_TAGS = (
+    DATA_PRODUCT_TYPES = (
         PHOTOMETRY,
         FITS_FILE,
         SPECTROSCOPY
@@ -59,7 +61,7 @@ class DataProduct(models.Model):
     group = models.ManyToManyField(DataProductGroup)
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
-    tag = models.CharField(max_length=50, blank=True, default='', choices=DATA_PRODUCT_TAGS)
+    tag = models.CharField(max_length=50, blank=True, default='', choices=DATA_PRODUCT_TYPES)
     featured = models.BooleanField(default=False)
 
     class Meta:
@@ -106,5 +108,5 @@ class ReducedDatum(models.Model):
     )
     source_name = models.CharField(max_length=100, default='')
     source_location = models.CharField(max_length=200, default='')
-    timestamp = models.DateTimeField(null=False, blank=False, db_index=True)
+    timestamp = models.DateTimeField(null=False, blank=False, default=datetime.now, db_index=True)
     value = models.TextField(null=False, blank=False)

tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html → tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html

@@ -1,6 +1,7 @@
 {% load bootstrap4 %}
 {% if user.is_authenticated %}
 <h4>Upload a data product for a target</h4>
+{% if response.GET.errors %}There was an error with your upload: {{ response.GET.errors }}{% endif %}
 <form method="POST" action="{% url 'tom_dataproducts:upload' %}" enctype="multipart/form-data">
   {% csrf_token %}
   {% bootstrap_form data_product_form %}