tom2tom data sharing

setup.py

@@ -49,6 +49,7 @@
         'plotly~=5.0',
         'python-dateutil~=2.8',
         'requests~=2.25',
+        'responses~=0.23',
         'specutils~=1.8',
     ],
     extras_require={

tom_dataproducts/alertstreams/hermes.py

@@ -136,7 +136,7 @@ def get_hermes_topics(**kwargs):
         response = requests.get(url=submit_url, headers=headers)
 
         topics = response.json()['writable_topics']
-    except KeyError:
+    except (KeyError, requests.exceptions.JSONDecodeError):
         topics = settings.DATA_SHARING['hermes']['USER_TOPICS']
     return topics
 

tom_dataproducts/api_views.py

@@ -4,15 +4,15 @@
 from guardian.shortcuts import assign_perm, get_objects_for_user
 from rest_framework import status
 from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin
-from rest_framework.parsers import MultiPartParser
+from rest_framework.parsers import MultiPartParser, FormParser, JSONParser
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
 
 from tom_common.hooks import run_hook
 from tom_dataproducts.data_processor import run_data_processor
 from tom_dataproducts.filters import DataProductFilter
 from tom_dataproducts.models import DataProduct, ReducedDatum
-from tom_dataproducts.serializers import DataProductSerializer
+from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer
 
 
 class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin):
@@ -38,6 +38,7 @@ def create(self, request, *args, **kwargs):
         response = super().create(request, *args, **kwargs)
 
         if response.status_code == status.HTTP_201_CREATED:
+            response.data['message'] = 'Data product successfully uploaded.'
             dp = DataProduct.objects.get(pk=response.data['id'])
             try:
                 run_hook('data_product_post_upload', dp)
@@ -68,3 +69,29 @@ def get_queryset(self):
             )
         else:
             return get_objects_for_user(self.request.user, 'tom_dataproducts.view_dataproduct')
+
+
+class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin):
+    """
+    Viewset for ReducedDatum objects. Supports list, create, and delete.
+
+    To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI.
+
+    **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The groups parameter
+    will specify which ``groups`` can view the created ``DataProduct``. If no ``groups`` are specified, the
+    ``ReducedDatum`` will only be visible to the user that created the ``DataProduct``. Make sure to check your
+    ``groups``!!**
+    """
+    queryset = ReducedDatum.objects.all()
+    serializer_class = ReducedDatumSerializer
+    filter_backends = (drf_filters.DjangoFilterBackend,)
+    permission_required = 'tom_dataproducts.view_reduceddatum'
+    parser_classes = [FormParser, JSONParser]
+
+    def create(self, request, *args, **kwargs):
+        response = super().create(request, *args, **kwargs)
+
+        if response.status_code == status.HTTP_201_CREATED:
+            response.data['message'] = 'Data successfully uploaded.'
+
+        return response

tom_dataproducts/forms.py

@@ -8,36 +8,6 @@
 from tom_dataproducts.alertstreams.hermes import get_hermes_topics
 
 
-def get_sharing_destination_options():
-    """
-    Build the Display options and headers for the dropdown form for choosing sharing topics.
-    Customize for a different selection experience.
-    :return: Tuple: Possible Destinations and their Display Names
-    """
-    choices = []
-    try:
-        for destination, details in settings.DATA_SHARING.items():
-            new_destination = [details.get('DISPLAY_NAME', destination)]
-            if details.get('USER_TOPICS', None):
-                # If topics exist for a destination (Such as HERMES) give topics as sub-choices
-                #   for non-selectable Destination
-                if destination == "hermes":
-                    destination_topics = get_hermes_topics()
-                else:
-                    destination_topics = details['USER_TOPICS']
-                topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics]
-                new_destination.append(tuple(topic_list))
-            else:
-                # Otherwise just use destination as option
-                new_destination.insert(0, destination)
-            choices.append(tuple(new_destination))
-    except AttributeError:
-        pass
-    return tuple(choices)
-
-
-DESTINATION_OPTIONS = get_sharing_destination_options()
-
 DATA_TYPE_OPTIONS = (('photometry', 'Photometry'),
                      ('spectroscopy', 'Spectroscopy'))
 
@@ -82,7 +52,7 @@ def __init__(self, *args, **kwargs):
 
 
 class DataShareForm(forms.Form):
-    share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination")
+    share_destination = forms.ChoiceField(required=True, choices=[], label="Destination")
     share_title = forms.CharField(required=False, label="Title")
     share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea())
     share_authors = forms.CharField(required=False, widget=forms.HiddenInput())
@@ -95,6 +65,33 @@ class DataShareForm(forms.Form):
         widget=forms.HiddenInput()
     )
 
+    def get_sharing_destination_options(self):
+        """
+        Build the Display options and headers for the dropdown form for choosing sharing topics.
+        Customize for a different selection experience.
+        :return: Tuple: Possible Destinations and their Display Names
+        """
+        choices = []
+        try:
+            for destination, details in settings.DATA_SHARING.items():
+                new_destination = [details.get('DISPLAY_NAME', destination)]
+                if details.get('USER_TOPICS', None):
+                    # If topics exist for a destination (Such as HERMES) give topics as sub-choices
+                    #   for non-selectable Destination
+                    if destination == "hermes":
+                        destination_topics = get_hermes_topics()
+                    else:
+                        destination_topics = details['USER_TOPICS']
+                    topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics]
+                    new_destination.append(tuple(topic_list))
+                else:
+                    # Otherwise just use destination as option
+                    new_destination.insert(0, destination)
+                choices.append(tuple(new_destination))
+        except AttributeError:
+            pass
+        return tuple(choices)
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.fields['share_destination'].choices = DESTINATION_OPTIONS
+        self.fields['share_destination'].choices = self.get_sharing_destination_options()

tom_dataproducts/models.py

@@ -331,7 +331,7 @@ class ReducedDatum(models.Model):
     """
 
     target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE)
-    data_product = models.ForeignKey(DataProduct, null=True, on_delete=models.CASCADE)
+    data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE)
     data_type = models.CharField(
         max_length=100,
         default=''
@@ -352,3 +352,12 @@ def save(self, *args, **kwargs):
         else:
             raise ValidationError('Not a valid DataProduct type.')
         return super().save()
+
+    def validate_unique(self, *args, **kwargs):
+        super().validate_unique(*args, **kwargs)
+        model_dict = self.__dict__.copy()
+        del model_dict['_state']
+        del model_dict['id']
+        obs = ReducedDatum.objects.filter(**model_dict)
+        if obs:
+            raise ValidationError('Data point already exists.')

tom_dataproducts/serializers.py

@@ -18,6 +18,8 @@ class Meta:
 
 
 class ReducedDatumSerializer(serializers.ModelSerializer):
+    target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all())
+
     class Meta:
         model = ReducedDatum
         fields = (
@@ -26,9 +28,31 @@ class Meta:
             'source_name',
             'source_location',
             'timestamp',
-            'value'
+            'value',
+            'target'
         )
 
+    def create(self, validated_data):
+        """DRF requires explicitly handling writeable nested serializers,
+        here we pop the groups data and save it using its serializer.
+        """
+        groups = validated_data.pop('groups', [])
+
+        rd = ReducedDatum(**validated_data)
+        rd.full_clean()
+        rd.save()
+
+        # Save groups for this target
+        group_serializer = GroupSerializer(data=groups, many=True)
+        if group_serializer.is_valid() and settings.TARGET_PERMISSIONS_ONLY is False:
+            for group in groups:
+                group_instance = Group.objects.get(pk=group['id'])
+                assign_perm('tom_dataproducts.view_dataproduct', group_instance, rd)
+                assign_perm('tom_dataproducts.change_dataproduct', group_instance, rd)
+                assign_perm('tom_dataproducts.delete_dataproduct', group_instance, rd)
+
+        return rd
+
 
 class DataProductSerializer(serializers.ModelSerializer):
     target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all())

tom_dataproducts/sharing.py

@@ -0,0 +1,172 @@
+import requests
+import os
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+from tom_targets.models import Target
+from tom_dataproducts.models import DataProduct, ReducedDatum
+from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage
+from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer
+
+
+def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None):
+    """
+
+    :param share_destination:
+    :param form_data:
+    :param product_id:
+    :param target_id:
+    :param selected_data:
+    :return:
+    """
+    # Query relevant Reduced Datums Queryset
+    accepted_data_types = ['photometry']
+    if product_id:
+        product = DataProduct.objects.get(pk=product_id)
+        reduced_datums = ReducedDatum.objects.filter(data_product=product)
+    elif selected_data:
+        reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data)
+    elif target_id:
+        target = Target.objects.get(pk=target_id)
+        data_type = form_data['data_type']
+        reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type)
+    else:
+        reduced_datums = ReducedDatum.objects.none()
+
+    reduced_datums.filter(data_type__in=accepted_data_types)
+
+    # Build and submit hermes table from Reduced Datums
+    hermes_topic = share_destination.split(':')[1]
+    destination = share_destination.split(':')[0]
+    message_info = BuildHermesMessage(title=form_data['share_title'],
+                                      submitter=form_data['submitter'],
+                                      authors=form_data['share_authors'],
+                                      message=form_data['share_message'],
+                                      topic=hermes_topic
+                                      )
+    # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share.
+    filtered_reduced_datums = check_for_share_safe_datums(destination, reduced_datums, topic=hermes_topic)
+    if filtered_reduced_datums.count() > 0:
+        response = publish_photometry_to_hermes(message_info, filtered_reduced_datums)
+    else:
+        return {'message': f'ERROR: No valid data to share. (Check Sharing Protocol. Note that data types must be in '
+                           f'{accepted_data_types})'}
+    return response
+
+
+def share_data_with_tom(share_destination, form_data, product_id=None, target_id=None, selected_data=None):
+    """
+
+    :param share_destination:
+    :param form_data:
+    :param product_id:
+    :param target_id:
+    :param selected_data:
+    :return:
+    """
+    try:
+        destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL']
+        username = settings.DATA_SHARING[share_destination]['USERNAME']
+        password = settings.DATA_SHARING[share_destination]['PASSWORD']
+    except KeyError as err:
+        raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.')
+    auth = (username, password)
+    headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
+
+    dataproducts_url = destination_tom_base_url + 'api/dataproducts/'
+    targets_url = destination_tom_base_url + 'api/targets/'
+    reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/'
+    reduced_datums = ReducedDatum.objects.none()
+
+    if product_id:
+        product = DataProduct.objects.get(pk=product_id)
+        target = product.target
+        serialized_data = DataProductSerializer(product).data
+        destination_target_id = get_destination_target(target, targets_url, headers, auth)
+        if destination_target_id is None:
+            return {'message': 'ERROR: No matching target found.'}
+        serialized_data['target'] = destination_target_id
+        # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage
+        dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name)
+        # Save DataProduct in Destination TOM
+        with open(dataproduct_filename, 'rb') as dataproduct_filep:
+            files = {'file': (product.data.name, dataproduct_filep, 'text/csv')}
+            headers = {'Media-Type': 'multipart/form-data'}
+            response = requests.post(dataproducts_url, data=serialized_data, files=files, headers=headers, auth=auth)
+    elif selected_data or target_id:
+        if selected_data:
+            reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data)
+            targets = set(reduced_datum.target for reduced_datum in reduced_datums)
+            target_dict = {}
+            for target in targets:
+                # get destination Target
+                destination_target_id = get_destination_target(target, targets_url, headers, auth)
+                target_dict[target.name] = destination_target_id
+            if all(value is None for value in target_dict.values()):
+                return {'message': 'ERROR: No matching targets found.'}
+        else:
+            target = Target.objects.get(pk=target_id)
+            reduced_datums = ReducedDatum.objects.filter(target=target)
+            destination_target_id = get_destination_target(target, targets_url, headers, auth)
+            if destination_target_id is None:
+                return {'message': 'ERROR: No matching target found.'}
+            target_dict = {target.name:  destination_target_id}
+        response_codes = []
+        reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums)
+        for datum in reduced_datums:
+            if target_dict[datum.target.name]:
+                serialized_data = ReducedDatumSerializer(datum).data
+                serialized_data['target'] = target_dict[datum.target.name]
+                serialized_data['data_product'] = ''
+                if not serialized_data['source_name']:
+                    serialized_data['source_name'] = settings.TOM_NAME
+                    serialized_data['source_location'] = "TOM-TOM Direct Sharing"
+                response = requests.post(reduced_datums_url, json=serialized_data, headers=headers, auth=auth)
+                response_codes.append(response.status_code)
+        failed_data_count = response_codes.count(500)
+        if failed_data_count < len(response_codes):
+            return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} '
+                               'datums successfully saved.'}
+        else:
+            return {'message': 'ERROR: No valid data shared. These data may already exist in target TOM.'}
+    else:
+        return {'message': 'ERROR: No valid data to share.'}
+
+    return response
+
+
+def get_destination_target(target, targets_url, headers, auth):
+    target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth)
+    target_response_json = target_response.json()
+    try:
+        if target_response_json['results']:
+            destination_target_id = target_response_json['results'][0]['id']
+            return destination_target_id
+        else:
+            return None
+    except KeyError:
+        return None
+
+
+def check_for_share_safe_datums(destination, reduced_datums, **kwargs):
+    """
+    Custom sharing protocols used to determine when data is shared with a destination.
+    This example prevents sharing if a datum has already been published to the given Hermes topic.
+    :param destination: sharing destination string
+    :param reduced_datums: selected input datums
+    :return: queryset of reduced datums to be shared
+    """
+    return reduced_datums
+    # if 'hermes' in destination:
+    #     message_topic = kwargs.get('topic', None)
+    #     # Remove data points previously shared to the given topic
+    #     filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published')
+    #                                              & Q(message__topic=message_topic))
+    # else:
+    #     filtered_datums = reduced_datums
+    # return filtered_datums
+
+
+def check_for_save_safe_datums():
+    return