diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..79a16af7e --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 \ No newline at end of file diff --git a/tom_base/settings.py.bak b/tom_base/settings.py.bak new file mode 100644 index 000000000..3636dfa82 --- /dev/null +++ b/tom_base/settings.py.bak @@ -0,0 +1,211 @@ +""" +Django settings for tom_base project. + +Generated by 'django-admin startproject' using Django 2.0.6. + +For more information on this file, see +https://docs.djangoproject.com/en/2.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.0/ref/settings/ +""" + +import os +import tempfile + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'dxja^_6p35x46dx0rx+c$(^31(10^n(twe1#ax3o8xl=n^p37q' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + 'django_extensions', + 'tom_common', + 'django_comments', + 'bootstrap4', + 'crispy_forms', + 'django_filters', + 'django_gravatar', + 'tom_targets', + 'tom_alerts', + 'tom_catalogs', + 'tom_observations', + 'tom_dataproducts', +] + +SITE_ID = 1 + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'tom_common.middleware.ExternalServiceMiddleware', + 'tom_common.middleware.AuthStrategyMiddleware', +] + +ROOT_URLCONF = 'tom_common.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +CRISPY_TEMPLATE_PACK = 'bootstrap4' + +WSGI_APPLICATION = 'tom_base.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = '/' + + +# Internationalization +# https://docs.djangoproject.com/en/2.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = False + +USE_TZ = True + +DATETIME_FORMAT = 'Y-m-d H:m:s' +DATE_FORMAT = 'Y-m-d' + + +# Caching +# https://docs.djangoproject.com/en/dev/topics/cache/#filesystem-caching + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': tempfile.gettempdir() + } +} + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.0/howto/static-files/ + +STATIC_URL = '/static/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'data') +MEDIA_URL = '/data/' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + '': { + 'handlers': ['console'], + 'level': 'INFO' + } + } +} + +TARGET_TYPE = 'NON_SIDEREAL' +FACILITIES = { + 'LCO': { + 'portal_url': 'https://observe.lco.global', + 'api_key': 'a4be47b88d02d69cbb2552f9699a223d4df84e8c', + } +} + +TOM_ALERT_CLASSES = [ + 'tom_alerts.brokers.mars.MARSBroker', + 'tom_alerts.brokers.lasair.LasairBroker' +] + +# Authentication strategy can either be LOCKED (required login for all views) +# or READ_ONLY (read only access to views) +AUTH_STRATEGY = 'READ_ONLY' + +# URLs that should be allowed access even with AUTH_STRATEGY = LOCKED +# for example: OPEN_URLS = ['/', '/about'] +OPEN_URLS = [] + +HOOKS = { + 'target_post_save': 'tom_common.hooks.target_post_save', + 'observation_change_state': 'tom_common.hooks.observation_change_state' +} + +DATA_TYPES = ( + ('SPECTROSCOPY', 'Spectroscopy'), + ('PHOTOMETRY', 'Photometry') +) + +try: + from local_settings import * # noqa +except ImportError: + pass diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 2df9fdd1c..7ba09aaf1 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -1,6 +1,6 @@ from django import forms -from .models import DataProductGroup, DataProduct +from .models import DataProductGroup, DataProduct, SPECTROSCOPY from tom_targets.models import Target from tom_observations.models import ObservationRecord @@ -29,4 +29,31 @@ class DataProductUploadForm(forms.Form): attrs={'multiple': True} ) ) - tag = forms.ChoiceField(choices=DataProduct.DATA_PRODUCT_TAGS) + tag = forms.ChoiceField(choices=DataProduct.DATA_PRODUCT_TYPES) + observation_timestamp = forms.SplitDateTimeField( + label='Observation Time', + widget=forms.SplitDateTimeWidget( + date_attrs={'placeholder': 'Observation Date', 'type': 'date'}, + time_attrs={'format': '%H:%M:%S', 'placeholder': 'Observation Time', + 'type': 'time', 'step': '1'} + ), + required=False + ) + referrer = forms.CharField( + widget=forms.HiddenInput() + ) + + def __init__(self, *args, **kwargs): + hide_timestamp = kwargs.pop('hide_timestamp', False) + super(DataProductUploadForm, self).__init__(*args, **kwargs) + if hide_timestamp: + self.fields['observation_timestamp'].widget = forms.HiddenInput() + + def clean(self): + cleaned_data = super().clean() + print(cleaned_data) + if cleaned_data.get('tag', '') != SPECTROSCOPY[0] and cleaned_data.get('observation_timestamp'): + raise forms.ValidationError('Observation timestamp is not valid for uploaded photometry') + # elif not cleaned_data.get('observation_timestamp'): + # raise forms.ValidationError('Observation timestamp is required for spectroscopy') + return cleaned_data \ No newline at end of file diff --git a/tom_dataproducts/management/commands/downloaddata.py b/tom_dataproducts/management/commands/downloaddata.py new file mode 100644 index 000000000..c764dd1fe --- /dev/null +++ b/tom_dataproducts/management/commands/downloaddata.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand +from tom_observations import facility +from tom_observations.models import ObservationRecord + + +class Command(BaseCommand): + help = 'Downloads data for all completed observations' + + def handle(self, *args, **options): + facility_classes = {} + for facility_name in facility.get_service_classes(): + facility_classes[facility_name] = facility.get_service_class(facility_name)() + observation_records = ObservationRecord.objects.all() + for record in observation_records: + if record.status not in facility_classes[record.facility].get_terminal_observing_states(): + facility_classes[record.facility].update_observation_status(record.observation_id) + facility_classes[record.facility].save_data_products(record) + + return 'completed command' \ No newline at end of file diff --git a/tom_dataproducts/migrations/0002_auto_20190515_0047.py b/tom_dataproducts/migrations/0002_auto_20190515_0047.py new file mode 100644 index 000000000..a69ab3e2f --- /dev/null +++ b/tom_dataproducts/migrations/0002_auto_20190515_0047.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.1 on 2019-05-15 00:47 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_dataproducts', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='reduceddatum', + name='timestamp', + field=models.DateTimeField(db_index=True, default=datetime.datetime.now), + ), + ] diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index b24baaae1..4bceceaa7 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -1,7 +1,9 @@ -from django.db import models +import os from io import BytesIO from base64 import b64encode -import os +from datetime import datetime + +from django.db import models from django.conf import settings import matplotlib @@ -40,7 +42,7 @@ def __str__(self): class DataProduct(models.Model): - DATA_PRODUCT_TAGS = ( + DATA_PRODUCT_TYPES = ( PHOTOMETRY, FITS_FILE, SPECTROSCOPY @@ -59,7 +61,7 @@ class DataProduct(models.Model): group = models.ManyToManyField(DataProductGroup) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - tag = models.CharField(max_length=50, blank=True, default='', choices=DATA_PRODUCT_TAGS) + tag = models.CharField(max_length=50, blank=True, default='', choices=DATA_PRODUCT_TYPES) featured = models.BooleanField(default=False) class Meta: @@ -106,5 +108,5 @@ class ReducedDatum(models.Model): ) source_name = models.CharField(max_length=100, default='') source_location = models.CharField(max_length=200, default='') - timestamp = models.DateTimeField(null=False, blank=False, db_index=True) + timestamp = models.DateTimeField(null=False, blank=False, default=datetime.now, db_index=True) value = models.TextField(null=False, blank=False) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html b/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html similarity index 79% rename from tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html rename to tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html index d9041702c..b1c8c4fc8 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/upload_dataproduct.html +++ b/tom_dataproducts/templates/tom_dataproducts/upload_dataproduct.html @@ -1,6 +1,7 @@ {% load bootstrap4 %} {% if user.is_authenticated %}