Merge branch 'dev' of https://github.com/TOMToolkit/tom_base into dev

setup.py

@@ -45,7 +45,7 @@
         'fits2image>=0.4.2',
         'Markdown~=3.4',  # django-rest-framework doc headers require this to support Markdown
         'numpy~=1.20',
-        'pillow~=9.2',
+        'pillow>=9.2,<11.0',
         'plotly~=5.0',
         'python-dateutil~=2.8',
         'requests~=2.25',
@@ -55,7 +55,7 @@
         'test': ['factory_boy~=3.2.1'],
         'docs': [
             'recommonmark~=0.7',
-            'sphinx>=4,<7',
+            'sphinx>=4,<8',
             'tom_antares',
             'tom_scimma'
         ]

tom_alerts/brokers/tns.py

@@ -32,9 +32,9 @@ class TNSForm(GenericQueryForm):
     days_ago = forms.FloatField(required=False, min_value=0.,
                                 label='Discovered in the Last __ Days',
                                 help_text='Leave blank to use the "Discovered After" field')
-    min_date = forms.DateTimeField(required=False,
-                                   label='Discovered After',
-                                   help_text='Most valid date formats are recognized')
+    min_date = forms.CharField(required=False,
+                               label='Discovered After',
+                               help_text='Most valid date formats are recognized')
     days_from_nondet = forms.FloatField(required=False, min_value=0.,
                                         label='Days From Nondetection',
                                         help_text='Maximum time between last nondetection and first detection')
@@ -106,7 +106,7 @@ def fetch_alerts(cls, parameters):
             public_timestamp = (datetime.utcnow() - timedelta(days=parameters['days_ago']))\
                 .strftime('%Y-%m-%d %H:%M:%S')
         elif parameters['min_date'] is not None:
-            public_timestamp = parameters['min_date'].strftime('%Y-%m-%d %H:%M:%S')
+            public_timestamp = parameters['min_date']
         else:
             public_timestamp = ''
         data = {

tom_dataproducts/data_processor.py

@@ -37,7 +37,7 @@ def run_data_processor(dp):
     data = data_processor.process_data(dp)
 
     reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=dp.data_product_type,
-                                   timestamp=datum[0], value=datum[1]) for datum in data]
+                                   timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data]
     ReducedDatum.objects.bulk_create(reduced_datums)
 
     return ReducedDatum.objects.filter(data_product=dp)

tom_dataproducts/processors/photometry_processor.py

@@ -3,6 +3,7 @@
 from astropy import units
 from astropy.io import ascii
 from astropy.time import Time, TimezoneInfo
+import numpy as np
 
 from tom_dataproducts.data_processor import DataProcessor
 from tom_dataproducts.exceptions import InvalidFileFormatException
@@ -25,7 +26,7 @@ def process_data(self, data_product):
         mimetype = mimetypes.guess_type(data_product.data.path)[0]
         if mimetype in self.PLAINTEXT_MIMETYPES:
             photometry = self._process_photometry_from_plaintext(data_product)
-            return [(datum.pop('timestamp'), datum) for datum in photometry]
+            return [(datum.pop('timestamp'), datum, datum.pop('source', '')) for datum in photometry]
         else:
             raise InvalidFileFormatException('Unsupported file type')
 
@@ -57,7 +58,8 @@ def _process_photometry_from_plaintext(self, data_product):
                 'timestamp': time.to_datetime(timezone=utc),
             }
             for column_name in datum.colnames:
-                value[column_name] = datum[column_name]
+                if not np.ma.is_masked(datum[column_name]):
+                    value[column_name] = datum[column_name]
             photometry.append(value)
 
         return photometry

tom_dataproducts/static/tom_dataproducts/photometry_sample.csv

@@ -1,4 +1,5 @@
-time,filter,magnitude,error
-55959.06999999983,r,15.582,0.005
-55959.06999999983,V,15.676,0.007
-55959.06999999983,B,15.591,0.008
+time,filter,magnitude,error,limit,source
+55959.06999999983,r,15.582,0.005,,ZTF
+55959.06999999983,V,15.676,0.007,,CSS
+55959.06999999983,B,15.591,0.008,,Las Cumbres
+55959.06999999983,B,,,18.0,DLT40

tom_targets/models.py

@@ -317,7 +317,7 @@ def validate_unique(self, *args, **kwargs):
         matches = Target.matches.check_for_fuzzy_match(self.name)
         for match in matches:
             # Ignore the fact that this target's name matches itself.
-            if match.id is not self.id:
+            if match.id != self.id:
                 raise ValidationError(f'Target with Name or alias similar to {self.name} already exists')
         # Alias Check only necessary when updating target existing target. Reverse relationships require Primary Key.
         # If nothing has changed for the Target, do not validate against existing aliases.

tom_targets/tests/test_utils.py

@@ -0,0 +1,16 @@
+from django.db import models
+
+
+class StrictMatch(models.Manager):
+    """
+    Return Queryset for target with name matching string.
+    """
+
+    def check_for_fuzzy_match(self, name):
+        """
+        Returns a queryset exactly matching name that is received
+        :param name: The string against which target names will be matched.
+        :return: queryset containing matching Target(s).
+        """
+        queryset = super().get_queryset().filter(name=name)
+        return queryset

tom_targets/tests/tests.py

@@ -838,6 +838,26 @@ def test_raw_update_process(self):
         with self.assertRaises(ValidationError):
             new_alias.full_clean()
 
+    @override_settings(MATCH_MANAGERS={'Target': 'tom_targets.tests.test_utils.StrictMatch'})
+    def test_update_with_strict_matching(self):
+        self.form_data.update({
+            'targetextra_set-TOTAL_FORMS': 1,
+            'targetextra_set-INITIAL_FORMS': 0,
+            'targetextra_set-MIN_NUM_FORMS': 0,
+            'targetextra_set-MAX_NUM_FORMS': 1000,
+            'targetextra_set-0-key': 'redshift',
+            'targetextra_set-0-value': '3',
+            'aliases-TOTAL_FORMS': 1,
+            'aliases-INITIAL_FORMS': 0,
+            'aliases-MIN_NUM_FORMS': 0,
+            'aliases-MAX_NUM_FORMS': 1000,
+            'aliases-0-name': 'testtargetname2'
+        })
+        self.client.post(reverse('targets:update', kwargs={'pk': self.target.id}), data=self.form_data)
+        self.target.refresh_from_db()
+        self.assertTrue(self.target.targetextra_set.filter(key='redshift').exists())
+        self.assertTrue(self.target.aliases.filter(name='testtargetname2').exists())
+
 
 class TestTargetImport(TestCase):
     def setUp(self):