From 2cdfde03e5fb4587ea14787e2c4b9ee5c8518848 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 23 Oct 2023 14:52:11 -0700 Subject: [PATCH 1/4] handle other responses than 500 --- tom_base/settings.py | 1 + tom_dataproducts/admin.py | 3 +- tom_dataproducts/sharing.py | 5 ++- tom_dataproducts/tests/tests.py | 70 +++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 8eef5e1fb..1d63d7440 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -201,6 +201,7 @@ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', 'tom_observations.facilities.soar.SOARFacility', + 'tom_swift.swift.SwiftFacility', ] # Define MATCH_MANAGERS here. diff --git a/tom_dataproducts/admin.py b/tom_dataproducts/admin.py index 1f78419c3..2cdcb3f87 100644 --- a/tom_dataproducts/admin.py +++ b/tom_dataproducts/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from tom_dataproducts.models import DataProduct, DataProductGroup +from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum admin.site.register(DataProduct) admin.site.register(DataProductGroup) +admin.site.register(ReducedDatum) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index d85c3524c..ae9d23980 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -144,10 +144,11 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id serialized_data['data_product'] = '' if not serialized_data['source_name']: serialized_data['source_name'] = settings.TOM_NAME - serialized_data['source_location'] = "TOM-TOM Direct Sharing" + serialized_data['source_location'] = f"ReducedDatum shared from " \ + f"<{settings.TOM_NAME}.url>/api/reduceddatums/{datum.id}/" response = requests.post(reduced_datums_url, json=serialized_data, headers=headers, auth=auth) response_codes.append(response.status_code) - failed_data_count = response_codes.count(500) + failed_data_count = len([rc for rc in response_codes if rc >= 300]) if failed_data_count < len(response_codes): return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} ' 'datums successfully saved.'} diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index a06d39245..eae6b4cc4 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -748,3 +748,73 @@ def test_share_reduced_datums_valid_responses(self): follow=True ) self.assertContains(response, '2 of 2 datums successfully saved.') + + @responses.activate + def test_share_reduced_datums_invalid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + sharing_dict = { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + } + # Check 500 error + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=500, + ) + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + sharing_dict, + follow=True + ) + self.assertContains(response, 'No valid data shared. These data may already exist in target TOM.') + + # Check 400 error + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=400, + ) + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + sharing_dict, + follow=True + ) + self.assertContains(response, 'No valid data shared. These data may already exist in target TOM.') + + # Check 300 error + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=300, + ) + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + sharing_dict, + follow=True + ) + self.assertContains(response, 'No valid data shared. These data may already exist in target TOM.') From e5c3f9c80c89d4e769ec406debf593c06198aa4f Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 23 Oct 2023 15:08:43 -0700 Subject: [PATCH 2/4] allow for blank fields in the ReducedDatum Model --- ...lter_reduceddatum_data_product_and_more.py | 29 +++++++++++++++++++ tom_dataproducts/models.py | 4 +-- tom_dataproducts/tests/test_api.py | 16 ++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tom_dataproducts/migrations/0012_alter_reduceddatum_data_product_and_more.py diff --git a/tom_dataproducts/migrations/0012_alter_reduceddatum_data_product_and_more.py b/tom_dataproducts/migrations/0012_alter_reduceddatum_data_product_and_more.py new file mode 100644 index 000000000..5696653f4 --- /dev/null +++ b/tom_dataproducts/migrations/0012_alter_reduceddatum_data_product_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.1 on 2023-10-23 22:03 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_dataproducts', '0011_reduceddatum_message'), + ] + + operations = [ + migrations.AlterField( + model_name='reduceddatum', + name='data_product', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct'), + ), + migrations.AlterField( + model_name='reduceddatum', + name='source_location', + field=models.CharField(blank=True, default='', max_length=200), + ), + migrations.AlterField( + model_name='reduceddatum', + name='source_name', + field=models.CharField(blank=True, default='', max_length=100), + ), + ] diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 6e2664471..1aa3da09f 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -336,8 +336,8 @@ class ReducedDatum(models.Model): max_length=100, default='' ) - source_name = models.CharField(max_length=100, default='') - source_location = models.CharField(max_length=200, default='') + source_name = models.CharField(max_length=100, default='', blank=True) + source_location = models.CharField(max_length=200, default='', blank=True) timestamp = models.DateTimeField(null=False, blank=False, default=datetime.now, db_index=True) value = models.JSONField(null=False, blank=False) message = models.ManyToManyField(AlertStreamMessage) diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index 5bb39b4d8..2ad4bc694 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -141,3 +141,19 @@ def test_upload_same_reduced_datum_twice(self): self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') rd_queryset = ReducedDatum.objects.all() self.assertEqual(rd_queryset.count(), 2) + + def test_upload_reduced_datum_no_sharing_location(self): + """ + Test that a reduced datum can be uploaded without a source_location. + """ + del self.rd_data['source_location'] + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + def test_upload_reduced_datum_no_sharing_name(self): + """ + Test that a reduced datum can be uploaded without a source_name. + """ + del self.rd_data['source_name'] + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) From 355830f3b4614b21ac2b2d4763d9b91681503b49 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 23 Oct 2023 15:57:44 -0700 Subject: [PATCH 3/4] change publish to share for consistency --- .../tom_dataproducts/partials/dataproduct_list_for_target.html | 2 +- .../templates/tom_dataproducts/partials/share_target_data.html | 2 +- tom_dataproducts/templatetags/dataproduct_extras.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index eb119fd8a..4c7331e5c 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -31,7 +31,7 @@

Data

{% if sharing_destinations %} - + {% else %}

Not Configured

{% endif %} diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html index bcccaabc4..33d8deffe 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html @@ -3,7 +3,7 @@ {% load tom_common_extras %}
- Publish Data for {{ target.name }}: + Share Data for {{ target.name }}:
{% if sharing_destinations %}
diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 59cbc1894..7c237b5e2 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -127,7 +127,7 @@ def upload_dataproduct(context, obj): @register.inclusion_tag('tom_dataproducts/partials/share_target_data.html', takes_context=True) def share_data(context, target): """ - Publish data to Hermes + Share data to Hermes or another TOM """ initial = {'submitter': context['request'].user, From 8f7221e51bfe75b49b454a5ba3e437d6076e0344 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 23 Oct 2023 16:02:50 -0700 Subject: [PATCH 4/4] revert accidental settings.py changes --- tom_base/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 1d63d7440..8eef5e1fb 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -201,7 +201,6 @@ 'tom_observations.facilities.lco.LCOFacility', 'tom_observations.facilities.gemini.GEMFacility', 'tom_observations.facilities.soar.SOARFacility', - 'tom_swift.swift.SwiftFacility', ] # Define MATCH_MANAGERS here.