Merge branch 'dev' into dependabot/pip/dev/sphinx-gte-4-and-lt-8

docs/targets/target_matcher.rst

@@ -47,10 +47,10 @@ See the following example for only checking for exact name matches:
     def check_for_fuzzy_match(self, name):
         """
         Returns a queryset exactly matching name that is received
-        :param name: The string against which target names and aliases will be matched.
-        :return: queryset containing matching Targets. Will return targets even when matched value is an alias.
+        :param name: The string against which target names will be matched.
+        :return: queryset containing matching Target(s).
         """
-        queryset = Target.objects.filter(name=name)
+        queryset = super().get_queryset().filter(name=name)
         return queryset
 
 This might be useful if a user is experiencing performance issues when ingesting targets or does not wish to allow for

tom_base/settings.py

@@ -211,8 +211,7 @@
 # MATCH_MANAGERS = {
 #    'Target': 'custom_code_directory.match_managers.MyCustomTargetMatchManager'
 # }
-MATCH_MANAGERS = {
-}
+MATCH_MANAGERS = {}
 
 #
 # tom_alerts configuration

tom_dataproducts/data_processor.py

@@ -37,7 +37,7 @@ def run_data_processor(dp):
     data = data_processor.process_data(dp)
 
     reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=dp.data_product_type,
-                                   timestamp=datum[0], value=datum[1]) for datum in data]
+                                   timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data]
     ReducedDatum.objects.bulk_create(reduced_datums)
 
     return ReducedDatum.objects.filter(data_product=dp)

tom_dataproducts/processors/photometry_processor.py

@@ -3,6 +3,7 @@
 from astropy import units
 from astropy.io import ascii
 from astropy.time import Time, TimezoneInfo
+import numpy as np
 
 from tom_dataproducts.data_processor import DataProcessor
 from tom_dataproducts.exceptions import InvalidFileFormatException
@@ -25,7 +26,7 @@ def process_data(self, data_product):
         mimetype = mimetypes.guess_type(data_product.data.path)[0]
         if mimetype in self.PLAINTEXT_MIMETYPES:
             photometry = self._process_photometry_from_plaintext(data_product)
-            return [(datum.pop('timestamp'), datum) for datum in photometry]
+            return [(datum.pop('timestamp'), datum, datum.pop('source', '')) for datum in photometry]
         else:
             raise InvalidFileFormatException('Unsupported file type')
 
@@ -57,7 +58,8 @@ def _process_photometry_from_plaintext(self, data_product):
                 'timestamp': time.to_datetime(timezone=utc),
             }
             for column_name in datum.colnames:
-                value[column_name] = datum[column_name]
+                if not np.ma.is_masked(datum[column_name]):
+                    value[column_name] = datum[column_name]
             photometry.append(value)
 
         return photometry

tom_dataproducts/static/tom_dataproducts/photometry_sample.csv

@@ -1,4 +1,5 @@
-time,filter,magnitude,error
-55959.06999999983,r,15.582,0.005
-55959.06999999983,V,15.676,0.007
-55959.06999999983,B,15.591,0.008
+time,filter,magnitude,error,limit,source
+55959.06999999983,r,15.582,0.005,,ZTF
+55959.06999999983,V,15.676,0.007,,CSS
+55959.06999999983,B,15.591,0.008,,Las Cumbres
+55959.06999999983,B,,,18.0,DLT40

tom_observations/facility.py

@@ -218,6 +218,15 @@ def get_form(self, observation_type):
         """
         pass
 
+    def get_facility_context_data(self, **kwargs):
+        """
+        This method provides an opportunity for the Facility subclass to add additional
+        context data. It will be called by the OberservationCreateView.get_context_data()
+        method and the context dictionary passed to the template will be updated with the
+        returned facility context dictionary.
+        """
+        return kwargs
+
     # TODO: consider making submit_observation create ObservationRecords as well
     @abstractmethod
     def submit_observation(self, observation_payload):

tom_observations/views.py

@@ -1,6 +1,7 @@
 from io import StringIO
 from urllib.parse import urlencode
 import logging
+from typing import List
 
 from crispy_forms.bootstrap import FormActions
 from crispy_forms.layout import HTML, Layout, Submit
@@ -125,6 +126,27 @@ class ObservationCreateView(LoginRequiredMixin, FormView):
     """
     template_name = 'tom_observations/observation_form.html'
 
+    def get_template_names(self) -> List[str]:
+        """Override the base class method to ask the Facility if it has
+        specified a Facility-specific template to use. If so, put it at the
+        front of the returned list of template_names.
+        """
+        template_names = super().get_template_names()
+
+        # get the facility_class and its template_name, if defined
+        facility_class = self.get_facility_class()
+        try:
+            if facility_class.template_name:
+                # add to front of list b/c first template will be tried first
+                template_names.insert(0, facility_class.template_name)
+        except AttributeError:
+            # some Facilities won't have a custom template_name defined and so
+            # we will just use the one defined above.
+            pass
+
+        logger.debug(f'ObservationCreateView.get_template_name template_names: {template_names}')
+        return template_names
+
     def get_target_id(self):
         """
         Parses the target id for the given observation from the query parameters.
@@ -198,6 +220,12 @@ def get_context_data(self, **kwargs):
 
         target = Target.objects.get(pk=self.get_target_id())
         context['target'] = target
+
+        # allow the Facility class to add data to the context
+        facility_class = self.get_facility_class()
+        facility_context = facility_class().get_facility_context_data()
+        context.update(facility_context)
+
         return context
 
     def get_form_class(self):
@@ -229,6 +257,9 @@ def get_form(self):
         except Exception as ex:
             logger.error(f"Error loading {self.get_facility()} form: {repr(ex)}")
             raise BadRequest(f"Error loading {self.get_facility()} form: {repr(ex)}")
+
+        # tom_observations/facility.BaseObservationForm.__init__ to see how
+        # groups is added to common_layout
         if not settings.TARGET_PERMISSIONS_ONLY:
             form.fields['groups'].queryset = self.request.user.groups.all()
         form.helper.form_action = reverse(

tom_setup/templates/tom_setup/settings.tmpl

@@ -48,6 +48,7 @@ INSTALLED_APPS = [
     'tom_common',
     'django_comments',
     'bootstrap4',
+    'crispy_bootstrap4',
     'crispy_forms',
     'rest_framework',
     'rest_framework.authtoken',
@@ -246,8 +247,6 @@ TOM_ALERT_CLASSES = [
     'tom_alerts.brokers.antares.ANTARESBroker',
     'tom_alerts.brokers.gaia.GaiaBroker',
     'tom_alerts.brokers.lasair.LasairBroker',
-    'tom_alerts.brokers.mars.MARSBroker',
-    'tom_alerts.brokers.scimma.SCIMMABroker',
     'tom_alerts.brokers.scout.ScoutBroker',
     'tom_alerts.brokers.tns.TNSBroker',
     'tom_alerts.brokers.fink.FinkBroker',

tom_targets/models.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from dateutil.parser import parse
+import logging
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -10,6 +11,8 @@
 
 from tom_common.hooks import run_hook
 
+logger = logging.getLogger(__name__)
+
 GLOBAL_TARGET_FIELDS = ['name', 'type']
 
 SIDEREAL_FIELDS = GLOBAL_TARGET_FIELDS + [
@@ -257,15 +260,16 @@ class Target(models.Model):
     )
 
     objects = models.Manager()
-    target_match_manager = settings.MATCH_MANAGERS.get('Target', None)
-    if target_match_manager:
+    try:
+        target_match_manager = settings.MATCH_MANAGERS.get('Target')
         try:
             manager = import_string(target_match_manager)
             matches = manager()
         except (ImportError, AttributeError):
-            raise ImportError(f'Could not import {target_match_manager}. Did you provide the correct path in '
-                              f'settings.py?')
-    else:
+            logger.debug(f'Could not import a Target Match Manager from {target_match_manager}. Did you provide the'
+                         f'correct path in settings.py?')
+            raise ImportError
+    except (ImportError, AttributeError):
         matches = TargetMatchManager()
 
     @transaction.atomic
@@ -313,7 +317,7 @@ def validate_unique(self, *args, **kwargs):
         matches = Target.matches.check_for_fuzzy_match(self.name)
         for match in matches:
             # Ignore the fact that this target's name matches itself.
-            if match.id is not self.id:
+            if match.id != self.id:
                 raise ValidationError(f'Target with Name or alias similar to {self.name} already exists')
         # Alias Check only necessary when updating target existing target. Reverse relationships require Primary Key.
         # If nothing has changed for the Target, do not validate against existing aliases.

tom_targets/tests/test_utils.py

@@ -0,0 +1,16 @@
+from django.db import models
+
+
+class StrictMatch(models.Manager):
+    """
+    Return Queryset for target with name matching string.
+    """
+
+    def check_for_fuzzy_match(self, name):
+        """
+        Returns a queryset exactly matching name that is received
+        :param name: The string against which target names will be matched.
+        :return: queryset containing matching Target(s).
+        """
+        queryset = super().get_queryset().filter(name=name)
+        return queryset

tom_targets/tests/tests.py

@@ -838,6 +838,26 @@ def test_raw_update_process(self):
         with self.assertRaises(ValidationError):
             new_alias.full_clean()
 
+    @override_settings(MATCH_MANAGERS={'Target': 'tom_targets.tests.test_utils.StrictMatch'})
+    def test_update_with_strict_matching(self):
+        self.form_data.update({
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': 'redshift',
+            'targetextra_set-0-value': '3',
+            'aliases-TOTAL_FORMS': 1,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+            'aliases-0-name': 'testtargetname2'
+        })
+        self.client.post(reverse('targets:update', kwargs={'pk': self.target.id}), data=self.form_data)
+        self.target.refresh_from_db()
+        self.assertTrue(self.target.targetextra_set.filter(key='redshift').exists())
+        self.assertTrue(self.target.aliases.filter(name='testtargetname2').exists())
+
 
 class TestTargetImport(TestCase):
     def setUp(self):