From 25e848f742b09d61d7b887cea1cc17d441131525 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 15 May 2023 19:17:06 -0700 Subject: [PATCH 01/37] refactor sharing view (broken) --- tom_dataproducts/sharing.py | 98 +++++++++++++++++++++++++ tom_dataproducts/tests/tests.py | 54 ++++++++++++++ tom_dataproducts/views.py | 123 ++++++++------------------------ 3 files changed, 183 insertions(+), 92 deletions(-) create mode 100644 tom_dataproducts/sharing.py diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py new file mode 100644 index 000000000..b1bd45483 --- /dev/null +++ b/tom_dataproducts/sharing.py @@ -0,0 +1,98 @@ + +def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): + # Query relevant Reduced Datums Queryset + accepted_data_types = ['photometry'] + if product_id: + product = DataProduct.objects.get(pk=product_id) + reduced_datums = ReducedDatum.objects.filter(data_product=product) + elif selected_data: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + elif target_id: + target = Target.objects.get(pk=target_id) + data_type = form_data['data_type'] + reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + else: + reduced_datums = ReducedDatum.objects.none() + + reduced_datums.filter(data_type__in=accepted_data_types) + + # Build and submit hermes table from Reduced Datums + hermes_topic = share_destination.split(':')[1] + destination = share_destination.split(':')[0] + message_info = BuildHermesMessage(title=form_data['share_title'], + submitter=form_data['submitter'], + authors=form_data['share_authors'], + message=form_data['share_message'], + topic=hermes_topic + ) + # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. + filtered_reduced_datums = get_share_safe_datums(destination, reduced_datums, topic=hermes_topic) + if filtered_reduced_datums.count() > 0: + response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) + else: + messages.error(self.request, f'No Data to share. (Check sharing Protocol, note that data types must be ' + f'in {accepted_data_types})') + + +def share_data_with_tom(destination, datums, product=None): + """ + When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other + TOM process it rather than share the Reduced Datums + :param destination: name of destination tom in settings.DATA_SHARING + :param datums: Queryset of ReducedDatum Instances + :param product: DataProduct model instance + :return: + """ + try: + destination_tom_base_url = settings.DATA_SHARING[destination]['BASE_URL'] + username = settings.DATA_SHARING[destination]['USERNAME'] + password = settings.DATA_SHARING[destination]['PASSWORD'] + except KeyError as err: + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {destination}: Key {err} not found.') + auth = (username, password) + headers = {'Media-Type': 'application/json'} + target = product.target + serialized_target_data = TargetSerializer(target).data + targets_url = destination_tom_base_url + 'api/targets/' + # TODO: Make sure aliases are checked before creating new target + # Attempt to create Target in Destination TOM + # response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) + # try: + # target_response = response.json() + # destination_target_id = target_response['id'] + # except KeyError: + # # If Target already exists at destination, find ID + # response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) + # target_response = response.json() + # destination_target_id = target_response['results'][0]['id'] + + print(serialized_target_data) + + response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) + target_response = response.json() + if target_response['results']: + destination_target_id = target_response['results'][0]['id'] + else: + return response + print("------------------------") + print(target_response) + serialized_dataproduct_data = DataProductSerializer(product).data + serialized_dataproduct_data['target'] = destination_target_id + dataproducts_url = destination_tom_base_url + 'api/dataproducts/' + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + # with open(dataproduct_filename, 'rb') as dataproduct_filep: + # files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + # headers = {'Media-Type': 'multipart/form-data'} + # response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, + # headers=headers, auth=auth) + return response + + +def check_for_share_safe_datums(): + return + + +def check_for_save_safe_datums(): + return diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index bf671c3c8..9c6679f7b 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -489,3 +489,57 @@ def test_create_thumbnail(self, mock_is_fits_image_file): 'ignore_missing_simple=True') self.assertIn(expected, logs.output) + + +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True) +@patch('tom_dataproducts.views.run_data_processor') +class TestShareDataProducts(TestCase): + def setUp(self): + self.target = SiderealTargetFactory.create() + self.observation_record = ObservingRecordFactory.create( + target_id=self.target.id, + facility=FakeRoboticFacility.name, + parameters={} + ) + self.data_product = DataProduct.objects.create( + product_id='testproductid', + target=self.target, + observation_record=self.observation_record, + data=SimpleUploadedFile('afile.fits', b'somedata') + ) + self.user = User.objects.create_user(username='test', email='test@example.com') + assign_perm('tom_targets.view_target', self.user, self.target) + self.client.force_login(self.user) + + def test_share_dataproduct(self, run_data_processor_mock): + + response = self.client.post( + reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': ['local_host'], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'TOM-TOM sharing is not yet supported.') + + def test_share_data_for_target(self, run_data_processor_mock): + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': ['local_host'], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'TOM-TOM sharing is not yet supported.') \ No newline at end of file diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 73cb920a8..207b77c76 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -314,109 +314,48 @@ def post(self, request, *args, **kwargs): Handles Data Products and All the data of a type for a target as well as individual Reduced Datums. Submit to Hermes, or Share with TOM (soon). """ - data_share_form = DataShareForm(request.POST, request.FILES) - # Check if data points have been selected. - selected_data = request.POST.getlist("share-box") + if data_share_form.is_valid(): form_data = data_share_form.cleaned_data - # 1st determine if pk is data product, Reduced Datum, or Target. - # Then query relevant Reduced Datums Queryset + share_destination = form_data['share_destination'] product_id = kwargs.get('dp_pk', None) - if product_id: - product = DataProduct.objects.get(pk=product_id) - data_type = product.data_product_type - reduced_datums = ReducedDatum.objects.filter(data_product=product) + target_id = kwargs.get('tg_pk', None) + + + # Check if data points have been selected. + selected_data = request.POST.getlist("share-box") + + # Check Destination + if 'HERMES' in share_destination.upper(): + response = share_data_with_hermes(share_destination, form_data, product_id, target_id, selected_data) else: - target_id = kwargs.get('tg_pk', None) - target = Target.objects.get(pk=target_id) - data_type = form_data['data_type'] - if request.POST.get("share-box", None) is None: - reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + response = share_data_with_tom(share_destination, form_data, product_id, target_id, selected_data) + try: + if 'message' in response.json(): + publish_feedback = response.json()['message'] else: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - if data_type == 'photometry': - share_destination = form_data['share_destination'] - if 'HERMES' in share_destination.upper(): - # Build and submit hermes table from Reduced Datums - hermes_topic = share_destination.split(':')[1] - destination = share_destination.split(':')[0] - message_info = BuildHermesMessage(title=form_data['share_title'], - submitter=form_data['submitter'], - authors=form_data['share_authors'], - message=form_data['share_message'], - topic=hermes_topic - ) - # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. - filtered_reduced_datums = self.get_share_safe_datums(destination, reduced_datums, - topic=hermes_topic) - if filtered_reduced_datums.count() > 0: - response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) - else: - messages.error(self.request, 'No Data to share. (Check sharing Protocol.)') + publish_feedback = f"ERROR: {response.text}" + except ValueError: + publish_feedback = f"ERROR: Returned Response code {response.status_code}" + if "ERROR" in publish_feedback.upper(): + messages.error(self.request, publish_feedback) + else: + messages.success(self.request, publish_feedback) + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + + + if data_share_form.is_valid(): + form_data = data_share_form.cleaned_data + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) else: - messages.error(self.request, 'TOM-TOM sharing is not yet supported.') + # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') + response = self.share_with_tom(share_destination, product) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) # response = self.share_with_tom(share_destination, product) - try: - if 'message' in response.json(): - publish_feedback = response.json()['message'] - else: - publish_feedback = f"ERROR: {response.text}" - except ValueError: - publish_feedback = f"ERROR: Returned Response code {response.status_code}" - if "ERROR" in publish_feedback.upper(): - messages.error(self.request, publish_feedback) - else: - messages.success(self.request, publish_feedback) - else: - messages.error(self.request, f'Publishing {data_type} data is not yet supported.') - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - def share_with_tom(self, tom_name, product): - """ - When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other - TOM process it rather than share the Reduced Datums - :param tom_name: name of destination tom in settings.DATA_SHARING - :param product: DataProduct model instance - :return: - """ - try: - destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] - username = settings.DATA_SHARING[tom_name]['USERNAME'] - password = settings.DATA_SHARING[tom_name]['PASSWORD'] - except KeyError as err: - raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {tom_name}: Key {err} not found.') - auth = (username, password) - headers = {'Media-Type': 'application/json'} - target = product.target - serialized_target_data = TargetSerializer(target).data - targets_url = destination_tom_base_url + 'api/targets/' - # TODO: Make sure aliases are checked before creating new target - # Attempt to create Target in Destination TOM - response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) - try: - target_response = response.json() - destination_target_id = target_response['id'] - except KeyError: - # If Target already exists at destination, find ID - response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) - target_response = response.json() - destination_target_id = target_response['results'][0]['id'] - - serialized_dataproduct_data = DataProductSerializer(product).data - serialized_dataproduct_data['target'] = destination_target_id - dataproducts_url = destination_tom_base_url + 'api/dataproducts/' - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - with open(dataproduct_filename, 'rb') as dataproduct_filep: - files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - headers = {'Media-Type': 'multipart/form-data'} - response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, - headers=headers, auth=auth) - return response + def get_share_safe_datums(self, destination, reduced_datums, **kwargs): """ From 035dea33018285def9ddafefc8910511d3c0ec18 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 16 May 2023 12:30:26 -0700 Subject: [PATCH 02/37] add hermes error message --- tom_dataproducts/sharing.py | 7 +++++-- tom_dataproducts/views.py | 22 +++++++++------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index b1bd45483..a24adc80e 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -30,8 +30,11 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target if filtered_reduced_datums.count() > 0: response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) else: - messages.error(self.request, f'No Data to share. (Check sharing Protocol, note that data types must be ' - f'in {accepted_data_types})') + def response(): + def json(): + return {'message': f'No Data to share. (Check sharing Protocol, note that data types must be in ' + f'{accepted_data_types})'} + return response def share_data_with_tom(destination, datums, product=None): diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 207b77c76..62e2ee849 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -322,7 +322,6 @@ def post(self, request, *args, **kwargs): product_id = kwargs.get('dp_pk', None) target_id = kwargs.get('tg_pk', None) - # Check if data points have been selected. selected_data = request.POST.getlist("share-box") @@ -344,18 +343,15 @@ def post(self, request, *args, **kwargs): messages.success(self.request, publish_feedback) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - - if data_share_form.is_valid(): - form_data = data_share_form.cleaned_data - - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - else: - # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') - response = self.share_with_tom(share_destination, product) - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # response = self.share_with_tom(share_destination, product) - - + # if data_share_form.is_valid(): + # form_data = data_share_form.cleaned_data + # + # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + # else: + # # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') + # response = self.share_with_tom(share_destination, product) + # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + # # response = self.share_with_tom(share_destination, product) def get_share_safe_datums(self, destination, reduced_datums, **kwargs): """ From 4da20e85a3811d6492e7ed49507933dbed12df6b Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 16 May 2023 15:48:05 -0700 Subject: [PATCH 03/37] fix some imports --- tom_dataproducts/sharing.py | 35 ++++++++++++++++++++++++----------- tom_dataproducts/views.py | 22 +++------------------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index a24adc80e..f1c7ad9ad 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -1,3 +1,7 @@ +from tom_targets.models import Target +from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage + def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): # Query relevant Reduced Datums Queryset @@ -26,14 +30,12 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target topic=hermes_topic ) # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. - filtered_reduced_datums = get_share_safe_datums(destination, reduced_datums, topic=hermes_topic) + filtered_reduced_datums = check_for_share_safe_datums(destination, reduced_datums, topic=hermes_topic) if filtered_reduced_datums.count() > 0: response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) else: - def response(): - def json(): - return {'message': f'No Data to share. (Check sharing Protocol, note that data types must be in ' - f'{accepted_data_types})'} + return {'message': f'ERROR: No valid data to share. (Check Sharing Protocol. Note that data types must be in ' + f'{accepted_data_types})'} return response @@ -69,16 +71,12 @@ def share_data_with_tom(destination, datums, product=None): # target_response = response.json() # destination_target_id = target_response['results'][0]['id'] - print(serialized_target_data) - response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) target_response = response.json() if target_response['results']: destination_target_id = target_response['results'][0]['id'] else: return response - print("------------------------") - print(target_response) serialized_dataproduct_data = DataProductSerializer(product).data serialized_dataproduct_data['target'] = destination_target_id dataproducts_url = destination_tom_base_url + 'api/dataproducts/' @@ -93,8 +91,23 @@ def share_data_with_tom(destination, datums, product=None): return response -def check_for_share_safe_datums(): - return +def check_for_share_safe_datums(destination, reduced_datums, **kwargs): + """ + Custom sharing protocols used to determine when data is shared with a destination. + This example prevents sharing if a datum has already been published to the given Hermes topic. + :param destination: sharing destination string + :param reduced_datums: selected input datums + :return: queryset of reduced datums to be shared + """ + return reduced_datums + # if 'hermes' in destination: + # message_topic = kwargs.get('topic', None) + # # Remove data points previously shared to the given topic + # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') + # & Q(message__topic=message_topic)) + # else: + # filtered_datums = reduced_datums + # return filtered_datums def check_for_save_safe_datums(): diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 62e2ee849..d6ac8dd80 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -32,10 +32,10 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.sharing import share_data_with_hermes import requests @@ -335,6 +335,8 @@ def post(self, request, *args, **kwargs): publish_feedback = response.json()['message'] else: publish_feedback = f"ERROR: {response.text}" + except AttributeError: + publish_feedback = response['message'] except ValueError: publish_feedback = f"ERROR: Returned Response code {response.status_code}" if "ERROR" in publish_feedback.upper(): @@ -353,24 +355,6 @@ def post(self, request, *args, **kwargs): # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) # # response = self.share_with_tom(share_destination, product) - def get_share_safe_datums(self, destination, reduced_datums, **kwargs): - """ - Custom sharing protocols used to determine when data is shared with a destination. - This example prevents sharing if a datum has already been published to the given Hermes topic. - :param destination: sharing destination string - :param reduced_datums: selected input datums - :return: queryset of reduced datums to be shared - """ - return reduced_datums - # if 'hermes' in destination: - # message_topic = kwargs.get('topic', None) - # # Remove data points previously shared to the given topic - # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') - # & Q(message__topic=message_topic)) - # else: - # filtered_datums = reduced_datums - # return filtered_datums - class DataProductGroupDetailView(DetailView): """ From e801e815fbb39a0424ec1fe24c8e68cf861b2d11 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 17 May 2023 17:07:24 -0700 Subject: [PATCH 04/37] finish Dataproduct sharing --- tom_dataproducts/alertstreams/hermes.py | 2 +- tom_dataproducts/api_views.py | 1 + tom_dataproducts/sharing.py | 93 ++++++++++++++++--------- tom_dataproducts/views.py | 3 +- 4 files changed, 65 insertions(+), 34 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index d2589af22..cd1c2742a 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -136,7 +136,7 @@ def get_hermes_topics(**kwargs): response = requests.get(url=submit_url, headers=headers) topics = response.json()['writable_topics'] - except KeyError: + except (KeyError, requests.exceptions.JSONDecodeError): topics = settings.DATA_SHARING['hermes']['USER_TOPICS'] return topics diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 8f59138ae..964df7a82 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -38,6 +38,7 @@ def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Data product successfully uploaded.' dp = DataProduct.objects.get(pk=response.data['id']) try: run_hook('data_product_post_upload', dp) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index f1c7ad9ad..9bf67f4ec 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -1,9 +1,24 @@ +import requests +import os + +from django.conf import settings + from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage +from tom_dataproducts.serializers import DataProductSerializer def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): + """ + + :param share_destination: + :param form_data: + :param product_id: + :param target_id: + :param selected_data: + :return: + """ # Query relevant Reduced Datums Queryset accepted_data_types = ['photometry'] if product_id: @@ -39,26 +54,61 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target return response -def share_data_with_tom(destination, datums, product=None): +def share_data_with_tom(share_destination, form_data, product_id=None, target_id=None, selected_data=None): """ - When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other - TOM process it rather than share the Reduced Datums - :param destination: name of destination tom in settings.DATA_SHARING - :param datums: Queryset of ReducedDatum Instances - :param product: DataProduct model instance + + :param share_destination: + :param form_data: + :param product_id: + :param target_id: + :param selected_data: :return: """ try: - destination_tom_base_url = settings.DATA_SHARING[destination]['BASE_URL'] - username = settings.DATA_SHARING[destination]['USERNAME'] - password = settings.DATA_SHARING[destination]['PASSWORD'] + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + username = settings.DATA_SHARING[share_destination]['USERNAME'] + password = settings.DATA_SHARING[share_destination]['PASSWORD'] except KeyError as err: - raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {destination}: Key {err} not found.') + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.') auth = (username, password) headers = {'Media-Type': 'application/json'} - target = product.target - serialized_target_data = TargetSerializer(target).data + + dataproducts_url = destination_tom_base_url + 'api/dataproducts/' targets_url = destination_tom_base_url + 'api/targets/' + reduced_datums = ReducedDatum.objects.none() + if product_id: + product = DataProduct.objects.get(pk=product_id) + target = product.target + serialized_data = DataProductSerializer(product).data + # elif selected_data: + # reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + # elif target_id: + # target = Target.objects.get(pk=target_id) + # data_type = form_data['data_type'] + # reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + else: + return {'message': f'ERROR: No valid data to share.'} + + # get destination Target + target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) + target_response_json = target_response.json() + if target_response_json['results']: + destination_target_id = target_response_json['results'][0]['id'] + else: + return target_response + + serialized_data['target'] = destination_target_id + + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + headers = {'Media-Type': 'multipart/form-data'} + response = requests.post(dataproducts_url, data=serialized_data, files=files, + headers=headers, auth=auth) + return response + # serialized_target_data = TargetSerializer(target).data # TODO: Make sure aliases are checked before creating new target # Attempt to create Target in Destination TOM # response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) @@ -71,25 +121,6 @@ def share_data_with_tom(destination, datums, product=None): # target_response = response.json() # destination_target_id = target_response['results'][0]['id'] - response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) - target_response = response.json() - if target_response['results']: - destination_target_id = target_response['results'][0]['id'] - else: - return response - serialized_dataproduct_data = DataProductSerializer(product).data - serialized_dataproduct_data['target'] = destination_target_id - dataproducts_url = destination_tom_base_url + 'api/dataproducts/' - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - # with open(dataproduct_filename, 'rb') as dataproduct_filep: - # files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - # headers = {'Media-Type': 'multipart/form-data'} - # response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, - # headers=headers, auth=auth) - return response - def check_for_share_safe_datums(destination, reduced_datums, **kwargs): """ diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d6ac8dd80..093ef8a6a 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -34,8 +34,7 @@ from tom_dataproducts.data_processor import run_data_processor from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class -from tom_dataproducts.serializers import DataProductSerializer -from tom_dataproducts.sharing import share_data_with_hermes +from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom import requests From 008ac5729bce52377186b055e404a442708db4d0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 22 Jun 2023 14:27:24 -0700 Subject: [PATCH 05/37] add reduced datum sharing --- tom_dataproducts/api_views.py | 30 ++++++++++- tom_dataproducts/forms.py | 61 +++++++++++----------- tom_dataproducts/models.py | 11 +++- tom_dataproducts/serializers.py | 26 +++++++++- tom_dataproducts/sharing.py | 83 ++++++++++++++++++------------ tom_dataproducts/tests/test_api.py | 40 ++++++++++++++ tom_dataproducts/urls.py | 3 +- 7 files changed, 183 insertions(+), 71 deletions(-) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 964df7a82..5661098b8 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -4,7 +4,7 @@ from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin -from rest_framework.parsers import MultiPartParser +from rest_framework.parsers import MultiPartParser, FormParser, JSONParser from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet @@ -12,7 +12,7 @@ from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.models import DataProduct, ReducedDatum -from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): @@ -69,3 +69,29 @@ def get_queryset(self): ) else: return get_objects_for_user(self.request.user, 'tom_dataproducts.view_dataproduct') + + +class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): + """ + Viewset for ReducedDatum objects. Supports list, create, and delete. + + To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + + **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The groups parameter + will specify which ``groups`` can view the created ``DataProduct``. If no ``groups`` are specified, the + ``ReducedDatum`` will only be visible to the user that created the ``DataProduct``. Make sure to check your + ``groups``!!** + """ + queryset = ReducedDatum.objects.all() + serializer_class = ReducedDatumSerializer + filter_backends = (drf_filters.DjangoFilterBackend,) + permission_required = 'tom_dataproducts.view_reduceddatum' + parser_classes = [FormParser, JSONParser] + + def create(self, request, *args, **kwargs): + response = super().create(request, *args, **kwargs) + + if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Data successfully uploaded.' + + return response diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index db39c1e75..e0b615a96 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -8,36 +8,6 @@ from tom_dataproducts.alertstreams.hermes import get_hermes_topics -def get_sharing_destination_options(): - """ - Build the Display options and headers for the dropdown form for choosing sharing topics. - Customize for a different selection experience. - :return: Tuple: Possible Destinations and their Display Names - """ - choices = [] - try: - for destination, details in settings.DATA_SHARING.items(): - new_destination = [details.get('DISPLAY_NAME', destination)] - if details.get('USER_TOPICS', None): - # If topics exist for a destination (Such as HERMES) give topics as sub-choices - # for non-selectable Destination - if destination == "hermes": - destination_topics = get_hermes_topics() - else: - destination_topics = details['USER_TOPICS'] - topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] - new_destination.append(tuple(topic_list)) - else: - # Otherwise just use destination as option - new_destination.insert(0, destination) - choices.append(tuple(new_destination)) - except AttributeError: - pass - return tuple(choices) - - -DESTINATION_OPTIONS = get_sharing_destination_options() - DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), ('spectroscopy', 'Spectroscopy')) @@ -82,7 +52,7 @@ def __init__(self, *args, **kwargs): class DataShareForm(forms.Form): - share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination") + share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") share_title = forms.CharField(required=False, label="Title") share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea()) share_authors = forms.CharField(required=False, widget=forms.HiddenInput()) @@ -95,6 +65,33 @@ class DataShareForm(forms.Form): widget=forms.HiddenInput() ) + def get_sharing_destination_options(self): + """ + Build the Display options and headers for the dropdown form for choosing sharing topics. + Customize for a different selection experience. + :return: Tuple: Possible Destinations and their Display Names + """ + choices = [] + try: + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + # If topics exist for a destination (Such as HERMES) give topics as sub-choices + # for non-selectable Destination + if destination == "hermes": + destination_topics = get_hermes_topics() + else: + destination_topics = details['USER_TOPICS'] + topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] + new_destination.append(tuple(topic_list)) + else: + # Otherwise just use destination as option + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + except AttributeError: + pass + return tuple(choices) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['share_destination'].choices = DESTINATION_OPTIONS + self.fields['share_destination'].choices = self.get_sharing_destination_options() diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index d8fbef1ae..6e2664471 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -331,7 +331,7 @@ class ReducedDatum(models.Model): """ target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE) - data_product = models.ForeignKey(DataProduct, null=True, on_delete=models.CASCADE) + data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE) data_type = models.CharField( max_length=100, default='' @@ -352,3 +352,12 @@ def save(self, *args, **kwargs): else: raise ValidationError('Not a valid DataProduct type.') return super().save() + + def validate_unique(self, *args, **kwargs): + super().validate_unique(*args, **kwargs) + model_dict = self.__dict__.copy() + del model_dict['_state'] + del model_dict['id'] + obs = ReducedDatum.objects.filter(**model_dict) + if obs: + raise ValidationError('Data point already exists.') diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 223747567..7d6ba9993 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -18,6 +18,8 @@ class Meta: class ReducedDatumSerializer(serializers.ModelSerializer): + target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) + class Meta: model = ReducedDatum fields = ( @@ -26,9 +28,31 @@ class Meta: 'source_name', 'source_location', 'timestamp', - 'value' + 'value', + 'target' ) + def create(self, validated_data): + """DRF requires explicitly handling writeable nested serializers, + here we pop the groups data and save it using its serializer. + """ + groups = validated_data.pop('groups', []) + + rd = ReducedDatum(**validated_data) + rd.full_clean() + rd.save() + + # Save groups for this target + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid() and settings.TARGET_PERMISSIONS_ONLY is False: + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_dataproducts.view_dataproduct', group_instance, rd) + assign_perm('tom_dataproducts.change_dataproduct', group_instance, rd) + assign_perm('tom_dataproducts.delete_dataproduct', group_instance, rd) + + return rd + class DataProductSerializer(serializers.ModelSerializer): target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 9bf67f4ec..cf0ddab52 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -6,7 +6,7 @@ from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage -from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): @@ -71,55 +71,70 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id except KeyError as err: raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.') auth = (username, password) - headers = {'Media-Type': 'application/json'} + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} dataproducts_url = destination_tom_base_url + 'api/dataproducts/' targets_url = destination_tom_base_url + 'api/targets/' + reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/' reduced_datums = ReducedDatum.objects.none() + if product_id: product = DataProduct.objects.get(pk=product_id) target = product.target serialized_data = DataProductSerializer(product).data - # elif selected_data: - # reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - # elif target_id: - # target = Target.objects.get(pk=target_id) - # data_type = form_data['data_type'] - # reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + destination_target_id = get_destination_target(target, targets_url, headers, auth) + serialized_data['target'] = destination_target_id + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + headers = {'Media-Type': 'multipart/form-data'} + response = requests.post(dataproducts_url, data=serialized_data, files=files, headers=headers, auth=auth) + elif selected_data or target_id: + if selected_data: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + targets = set(reduced_datum.target for reduced_datum in reduced_datums) + target_dict = {} + for target in targets: + # get destination Target + destination_target_id = get_destination_target(target, targets_url, headers, auth) + target_dict[target.name] = destination_target_id + else: + target = Target.objects.get(pk=target_id) + reduced_datums = ReducedDatum.objects.filter(target=target) + destination_target_id = get_destination_target(target, targets_url, headers, auth) + target_dict = {target.name: destination_target_id} + response_codes = [] + for datum in reduced_datums: + if target_dict[datum.target.name]: + serialized_data = ReducedDatumSerializer(datum).data + serialized_data['target'] = target_dict[datum.target.name] + serialized_data['data_product'] = '' + if not serialized_data['source_name']: + serialized_data['source_name'] = settings.TOM_NAME + serialized_data['source_location'] = "TOM-TOM Direct Sharing" + response = requests.post(reduced_datums_url, json=serialized_data, headers=headers, auth=auth) + response_codes.append(response.status_code) + failed_data_count = response_codes.count(500) + if failed_data_count < len(response_codes): + return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} datums successfully saved.'} + else: + return {'message': f'ERROR: No valid data shared. These data may already exist in target TOM.'} else: return {'message': f'ERROR: No valid data to share.'} - # get destination Target + return response + + +def get_destination_target(target, targets_url, headers, auth): target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) target_response_json = target_response.json() if target_response_json['results']: destination_target_id = target_response_json['results'][0]['id'] + return destination_target_id else: - return target_response - - serialized_data['target'] = destination_target_id - - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - with open(dataproduct_filename, 'rb') as dataproduct_filep: - files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - headers = {'Media-Type': 'multipart/form-data'} - response = requests.post(dataproducts_url, data=serialized_data, files=files, - headers=headers, auth=auth) - return response - # serialized_target_data = TargetSerializer(target).data - # TODO: Make sure aliases are checked before creating new target - # Attempt to create Target in Destination TOM - # response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) - # try: - # target_response = response.json() - # destination_target_id = target_response['id'] - # except KeyError: - # # If Target already exists at destination, find ID - # response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) - # target_response = response.json() - # destination_target_id = target_response['results'][0]['id'] + return None def check_for_share_safe_datums(destination, reduced_datums, **kwargs): diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index fb86fdecd..c1eb5636a 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,6 +1,8 @@ +from datetime import datetime from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse +from django.core.exceptions import ValidationError from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase @@ -102,3 +104,41 @@ def test_data_product_list(self): response = self.client.get(reverse('api:dataproducts-list')) self.assertContains(response, dp.product_id, status_code=status.HTTP_200_OK) + + +class TestReducedDatumViewset(APITestCase): + def setUp(self): + self.user = User.objects.create(username='testuser') + self.client.force_login(self.user) + self.st = SiderealTargetFactory.create() + self.obsr = ObservingRecordFactory.create(target_id=self.st.id) + self.rd_data = { + 'data_product': '', + 'data_type': 'photometry', + 'source_name': 'TOM Toolkit', + 'source_location': 'TOM-TOM Direct Sharing', + 'value': {'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + 'target': self.st.id, + 'timestamp': '2012-02-12T01:40:47Z' + } + + assign_perm('tom_dataproducts.add_reduceddatum', self.user) + assign_perm('tom_targets.add_target', self.user, self.st) + assign_perm('tom_targets.view_target', self.user, self.st) + assign_perm('tom_targets.change_target', self.user, self.st) + + def test_upload_reduced_datum(self): + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.assertContains(response, self.rd_data['source_name'], status_code=status.HTTP_201_CREATED) + + def test_upload_same_reduced_datum_twice(self): + """ + Test that identical data raises a validation error while similar but different JSON will make it through. + """ + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + with self.assertRaises(ValidationError): + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} + response3 = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + rd_queryset = ReducedDatum.objects.all() + self.assertEqual(rd_queryset.count(), 2) diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index ebd4b450b..c08dddde5 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -7,10 +7,11 @@ from tom_dataproducts.views import DataShareView from tom_common.api_router import SharedAPIRootRouter -from tom_dataproducts.api_views import DataProductViewSet +from tom_dataproducts.api_views import DataProductViewSet, ReducedDatumViewSet router = SharedAPIRootRouter() router.register(r'dataproducts', DataProductViewSet, 'dataproducts') +router.register(r'reduceddatums', ReducedDatumViewSet, 'reduceddatums') app_name = 'tom_dataproducts' From b758531241149338d55d67821fed2bcaa02f55f6 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 22 Jun 2023 14:45:57 -0700 Subject: [PATCH 06/37] fix some lint issues --- tom_dataproducts/sharing.py | 11 +++++++---- tom_dataproducts/tests/test_api.py | 5 ++--- tom_dataproducts/tests/tests.py | 2 +- tom_dataproducts/views.py | 6 ------ 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index cf0ddab52..04e38f408 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -2,9 +2,10 @@ import os from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from tom_targets.models import Target -from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer @@ -106,6 +107,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id destination_target_id = get_destination_target(target, targets_url, headers, auth) target_dict = {target.name: destination_target_id} response_codes = [] + reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums) for datum in reduced_datums: if target_dict[datum.target.name]: serialized_data = ReducedDatumSerializer(datum).data @@ -118,11 +120,12 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id response_codes.append(response.status_code) failed_data_count = response_codes.count(500) if failed_data_count < len(response_codes): - return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} datums successfully saved.'} + return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} ' + 'datums successfully saved.'} else: - return {'message': f'ERROR: No valid data shared. These data may already exist in target TOM.'} + return {'message': 'ERROR: No valid data shared. These data may already exist in target TOM.'} else: - return {'message': f'ERROR: No valid data to share.'} + return {'message': 'ERROR: No valid data to share.'} return response diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index c1eb5636a..5bb39b4d8 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,4 +1,3 @@ -from datetime import datetime from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse @@ -135,10 +134,10 @@ def test_upload_same_reduced_datum_twice(self): """ Test that identical data raises a validation error while similar but different JSON will make it through. """ - response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') with self.assertRaises(ValidationError): self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} - response3 = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') rd_queryset = ReducedDatum.objects.all() self.assertEqual(rd_queryset.count(), 2) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 9c6679f7b..a00049391 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -542,4 +542,4 @@ def test_share_data_for_target(self, run_data_processor_mock): }, follow=True ) - self.assertContains(response, 'TOM-TOM sharing is not yet supported.') \ No newline at end of file + self.assertContains(response, 'TOM-TOM sharing is not yet supported.') diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 093ef8a6a..b9d489795 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,6 +1,5 @@ from io import StringIO import logging -import os from urllib.parse import urlencode, urlparse from django.conf import settings @@ -9,7 +8,6 @@ from django.contrib.auth.models import Group from django.core.cache import cache from django.core.cache.utils import make_template_fragment_key -from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.http import HttpResponseRedirect from django.shortcuts import redirect @@ -25,8 +23,6 @@ from tom_common.hooks import run_hook from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_targets.serializers import TargetSerializer -from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm @@ -36,8 +32,6 @@ from tom_observations.facility import get_service_class from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom -import requests - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From 89b08763126f7aeb834face75e6f60792455bf98 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 12:59:43 -0700 Subject: [PATCH 07/37] add tests for no connection to target tom --- setup.py | 1 + tom_dataproducts/sharing.py | 17 +++-- tom_dataproducts/tests/tests.py | 112 ++++++++++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index ef8ff72b0..538655a77 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ 'plotly~=5.0', 'python-dateutil~=2.8', 'requests~=2.25', + 'responses~=0.23', 'specutils~=1.8', ], extras_require={ diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 04e38f408..8995bd39e 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -84,6 +84,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id target = product.target serialized_data = DataProductSerializer(product).data destination_target_id = get_destination_target(target, targets_url, headers, auth) + if destination_target_id is None: + return {'message': 'ERROR: No matching target found.'} serialized_data['target'] = destination_target_id # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) @@ -101,10 +103,14 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id # get destination Target destination_target_id = get_destination_target(target, targets_url, headers, auth) target_dict[target.name] = destination_target_id + if all(value is None for value in target_dict.values()): + return {'message': 'ERROR: No matching targets found.'} else: target = Target.objects.get(pk=target_id) reduced_datums = ReducedDatum.objects.filter(target=target) destination_target_id = get_destination_target(target, targets_url, headers, auth) + if destination_target_id is None: + return {'message': 'ERROR: No matching target found.'} target_dict = {target.name: destination_target_id} response_codes = [] reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums) @@ -133,10 +139,13 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id def get_destination_target(target, targets_url, headers, auth): target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) target_response_json = target_response.json() - if target_response_json['results']: - destination_target_id = target_response_json['results'][0]['id'] - return destination_target_id - else: + try: + if target_response_json['results']: + destination_target_id = target_response_json['results'][0]['id'] + return destination_target_id + else: + return None + except KeyError: return None diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index a00049391..75a41f841 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -1,6 +1,7 @@ import os from http import HTTPStatus import tempfile +import responses from astropy import units from astropy.io import fits @@ -18,7 +19,7 @@ from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import DataProductUploadForm -from tom_dataproducts.models import DataProduct, is_fits_image_file +from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_dataproducts.processors.spectroscopy_processor import SpectroscopyProcessor @@ -493,7 +494,6 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) -@patch('tom_dataproducts.views.run_data_processor') class TestShareDataProducts(TestCase): def setUp(self): self.target = SiderealTargetFactory.create() @@ -512,7 +512,40 @@ def setUp(self): assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - def test_share_dataproduct(self, run_data_processor_mock): + self.rd1 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + ) + self.rd2 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + ) + self.rd3 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + ) + + @responses.activate + def test_share_dataproduct_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) response = self.client.post( reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), @@ -520,13 +553,80 @@ def test_share_dataproduct(self, run_data_processor_mock): 'share_authors': ['test_author'], 'target': self.target.id, 'submitter': ['test_submitter'], - 'share_destination': ['local_host'], + 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'] }, follow=True ) - self.assertContains(response, 'TOM-TOM sharing is not yet supported.') + self.assertContains(response, 'ERROR: No matching target found.') + + @responses.activate + def test_share_reduceddatums_target_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching target found.') + + @responses.activate + def test_share_reduced_datums_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching targets found.') def test_share_data_for_target(self, run_data_processor_mock): @@ -542,4 +642,4 @@ def test_share_data_for_target(self, run_data_processor_mock): }, follow=True ) - self.assertContains(response, 'TOM-TOM sharing is not yet supported.') + self.assertContains(response, 'Data successfully uploaded.') From 06d2f70846e973820573f0975ab707b682a00d75 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 13:14:27 -0700 Subject: [PATCH 08/37] fix tests to spoof settings --- tom_dataproducts/tests/tests.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 75a41f841..a00ad8589 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -493,7 +493,8 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], - TARGET_PERMISSIONS_ONLY=True) + TARGET_PERMISSIONS_ONLY=True, + DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example'}}) class TestShareDataProducts(TestCase): def setUp(self): self.target = SiderealTargetFactory.create() @@ -628,18 +629,18 @@ def test_share_reduced_datums_no_valid_responses(self): ) self.assertContains(response, 'ERROR: No matching targets found.') - def test_share_data_for_target(self, run_data_processor_mock): - - response = self.client.post( - reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), - { - 'share_authors': ['test_author'], - 'target': self.target.id, - 'submitter': ['test_submitter'], - 'share_destination': ['local_host'], - 'share_title': ['Updated data for thingy.'], - 'share_message': ['test_message'] - }, - follow=True - ) - self.assertContains(response, 'Data successfully uploaded.') + # def test_share_data_for_target(self, run_data_processor_mock): + # + # response = self.client.post( + # reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + # { + # 'share_authors': ['test_author'], + # 'target': self.target.id, + # 'submitter': ['test_submitter'], + # 'share_destination': ['local_host'], + # 'share_title': ['Updated data for thingy.'], + # 'share_message': ['test_message'] + # }, + # follow=True + # ) + # self.assertContains(response, 'Data successfully uploaded.') From f4deef8204949fe0162128402543f6a6564b0c40 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 13:22:55 -0700 Subject: [PATCH 09/37] add username/password to settings override --- tom_dataproducts/tests/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index a00ad8589..2d90ab838 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -494,7 +494,9 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, - DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example'}}) + DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example', + 'USERNAME': 'fake_user', + 'PASSWORD': 'password'}}) class TestShareDataProducts(TestCase): def setUp(self): self.target = SiderealTargetFactory.create() From 71ef6acfe6b5008705eaf88d6df6bf600ccc4138 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 13:35:59 -0700 Subject: [PATCH 10/37] make properly formatted fake url --- tom_dataproducts/tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 2d90ab838..9832b0c1a 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -494,7 +494,7 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, - DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example', + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example', 'USERNAME': 'fake_user', 'PASSWORD': 'password'}}) class TestShareDataProducts(TestCase): From 17a7a03ceec1da2c180d3fe7edbfedfef904a6e0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 28 Jun 2023 16:19:04 -0700 Subject: [PATCH 11/37] add tests for succesful sharing --- tom_dataproducts/tests/tests.py | 134 ++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 16 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 9832b0c1a..a06d39245 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -494,7 +494,7 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, - DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example', + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example/', 'USERNAME': 'fake_user', 'PASSWORD': 'password'}}) class TestShareDataProducts(TestCase): @@ -631,18 +631,120 @@ def test_share_reduced_datums_no_valid_responses(self): ) self.assertContains(response, 'ERROR: No matching targets found.') - # def test_share_data_for_target(self, run_data_processor_mock): - # - # response = self.client.post( - # reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), - # { - # 'share_authors': ['test_author'], - # 'target': self.target.id, - # 'submitter': ['test_submitter'], - # 'share_destination': ['local_host'], - # 'share_title': ['Updated data for thingy.'], - # 'share_message': ['test_message'] - # }, - # follow=True - # ) - # self.assertContains(response, 'Data successfully uploaded.') + @responses.activate + def test_share_dataproduct_valid_target_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/dataproducts/', + json={"message": "Data product successfully uploaded."}, + status=200, + ) + + response = self.client.post( + reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'Data product successfully uploaded.') + + @responses.activate + def test_share_reduceddatums_target_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, '3 of 3 datums successfully saved.') + + @responses.activate + def test_share_reduced_datums_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + }, + follow=True + ) + self.assertContains(response, '2 of 2 datums successfully saved.') From e6e7e9567a503e181dfd95f91d44c8cbbd8cbf69 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 5 Jul 2023 17:05:31 -0700 Subject: [PATCH 12/37] set up ui buttons for sharing --- tom_dataproducts/views.py | 10 ---- .../tom_targets/partials/target_buttons.html | 5 +- tom_targets/urls.py | 1 + tom_targets/views.py | 58 +++++++++++++++++++ 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index b9d489795..314519b07 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -338,16 +338,6 @@ def post(self, request, *args, **kwargs): messages.success(self.request, publish_feedback) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # if data_share_form.is_valid(): - # form_data = data_share_form.cleaned_data - # - # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # else: - # # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') - # response = self.share_with_tom(share_destination, product) - # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # # response = self.share_with_tom(share_destination, product) - class DataProductGroupDetailView(DetailView): """ diff --git a/tom_targets/templates/tom_targets/partials/target_buttons.html b/tom_targets/templates/tom_targets/partials/target_buttons.html index 5cd86e4f4..3f5216c37 100644 --- a/tom_targets/templates/tom_targets/partials/target_buttons.html +++ b/tom_targets/templates/tom_targets/partials/target_buttons.html @@ -1,2 +1,3 @@ -Update Target -Delete Target \ No newline at end of file +Update +Delete +Share \ No newline at end of file diff --git a/tom_targets/urls.py b/tom_targets/urls.py index 33e98aa81..09bb8228f 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -24,6 +24,7 @@ path('name/', TargetNameSearchView.as_view(), name='name-search'), path('/update/', TargetUpdateView.as_view(), name='update'), path('/delete/', TargetDeleteView.as_view(), name='delete'), + path('/share/', TargetUpdateView.as_view(), name='share'), path('/', TargetDetailView.as_view(), name='detail'), path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') diff --git a/tom_targets/views.py b/tom_targets/views.py index 500c1e472..d762e7e47 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -321,6 +321,64 @@ def get_form(self, *args, **kwargs): return form +# class TargetShareView(FormView): +# """ +# View that handles the sharing of data either through HERMES or with another TOM. +# """ +# +# form_class = DataShareForm +# +# def get_form(self, *args, **kwargs): +# # TODO: Add permissions +# form = super().get_form(*args, **kwargs) +# return form +# +# def form_invalid(self, form): +# """ +# Adds errors to Django messaging framework in the case of an invalid form and redirects to the previous page. +# """ +# # TODO: Format error messages in a more human-readable way +# messages.error(self.request, 'There was a problem sharing your Data: {}'.format(form.errors.as_json())) +# return redirect(form.cleaned_data.get('referrer', '/')) +# +# def post(self, request, *args, **kwargs): +# """ +# Method that handles the POST requests for sharing data. +# Handles Data Products and All the data of a type for a target as well as individual Reduced Datums. +# Submit to Hermes, or Share with TOM (soon). +# """ +# data_share_form = DataShareForm(request.POST, request.FILES) +# +# if data_share_form.is_valid(): +# form_data = data_share_form.cleaned_data +# share_destination = form_data['share_destination'] +# product_id = kwargs.get('dp_pk', None) +# target_id = kwargs.get('tg_pk', None) +# +# # Check if data points have been selected. +# selected_data = request.POST.getlist("share-box") +# +# # Check Destination +# if 'HERMES' in share_destination.upper(): +# response = share_data_with_hermes(share_destination, form_data, product_id, target_id, selected_data) +# else: +# response = share_data_with_tom(share_destination, form_data, product_id, target_id, selected_data) +# try: +# if 'message' in response.json(): +# publish_feedback = response.json()['message'] +# else: +# publish_feedback = f"ERROR: {response.text}" +# except AttributeError: +# publish_feedback = response['message'] +# except ValueError: +# publish_feedback = f"ERROR: Returned Response code {response.status_code}" +# if "ERROR" in publish_feedback.upper(): +# messages.error(self.request, publish_feedback) +# else: +# messages.success(self.request, publish_feedback) +# return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + + class TargetDeleteView(Raise403PermissionRequiredMixin, DeleteView): """ View for deleting a target. Requires authorization. From d342af2b0f946af2e9ed7cae99437d03b0ce4bb4 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 13 Jul 2023 16:42:06 -0700 Subject: [PATCH 13/37] Build Target sharing form page --- tom_dataproducts/forms.py | 31 +----- tom_dataproducts/sharing.py | 30 +++++- .../photometry_datalist_for_target.html | 36 +++---- .../templatetags/dataproduct_extras.py | 5 +- tom_targets/forms.py | 16 +++ .../tom_targets/partials/target_buttons.html | 4 +- tom_targets/urls.py | 4 +- tom_targets/views.py | 101 +++++++----------- 8 files changed, 111 insertions(+), 116 deletions(-) diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index e0b615a96..43fbbf309 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -5,7 +5,7 @@ from tom_dataproducts.models import DataProductGroup, DataProduct from tom_observations.models import ObservationRecord from tom_targets.models import Target -from tom_dataproducts.alertstreams.hermes import get_hermes_topics +from tom_dataproducts.sharing import get_sharing_destination_options DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), @@ -65,33 +65,6 @@ class DataShareForm(forms.Form): widget=forms.HiddenInput() ) - def get_sharing_destination_options(self): - """ - Build the Display options and headers for the dropdown form for choosing sharing topics. - Customize for a different selection experience. - :return: Tuple: Possible Destinations and their Display Names - """ - choices = [] - try: - for destination, details in settings.DATA_SHARING.items(): - new_destination = [details.get('DISPLAY_NAME', destination)] - if details.get('USER_TOPICS', None): - # If topics exist for a destination (Such as HERMES) give topics as sub-choices - # for non-selectable Destination - if destination == "hermes": - destination_topics = get_hermes_topics() - else: - destination_topics = details['USER_TOPICS'] - topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] - new_destination.append(tuple(topic_list)) - else: - # Otherwise just use destination as option - new_destination.insert(0, destination) - choices.append(tuple(new_destination)) - except AttributeError: - pass - return tuple(choices) - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['share_destination'].choices = self.get_sharing_destination_options() + self.fields['share_destination'].choices = get_sharing_destination_options() diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 8995bd39e..f6a7cda1e 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -6,7 +6,7 @@ from tom_targets.models import Target from tom_dataproducts.models import DataProduct, ReducedDatum -from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage +from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage, get_hermes_topics from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer @@ -170,3 +170,31 @@ def check_for_share_safe_datums(destination, reduced_datums, **kwargs): def check_for_save_safe_datums(): return + + +def get_sharing_destination_options(): + """ + Build the Display options and headers for the dropdown form for choosing sharing topics. + Customize for a different selection experience. + :return: Tuple: Possible Destinations and their Display Names + """ + choices = [] + try: + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + # If topics exist for a destination (Such as HERMES) give topics as sub-choices + # for non-selectable Destination + if destination == "hermes": + destination_topics = get_hermes_topics() + else: + destination_topics = details['USER_TOPICS'] + topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] + new_destination.append(tuple(topic_list)) + else: + # Otherwise just use destination as option + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + except AttributeError: + pass + return tuple(choices) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html index 953af3b88..1a2e499f5 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -46,25 +46,25 @@ {% endfor %} -
-
- Share Selected Data -
- {% if sharing_destinations %} -
-
- {% bootstrap_field target_data_share_form.share_destination %} -
-
- -
-
- {% else %} - Not Configured - {% endif %} - - + {% if not target_share %} +
+
+ Share Selected Data +
+ {% if sharing_destinations %} +
+
+ {% bootstrap_field target_data_share_form.share_destination %} +
+
+ +
+
+ {% else %} + Not Configured + {% endif %}
+ {% endif %}