From f4849e862184c83e20e115f2ce2beffb38daf914 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Tue, 29 Oct 2024 23:09:16 -0500 Subject: [PATCH 01/21] add PairedFieldDescriptor two-column tag model --- ...ection_tdamm_manual_collection_tdamm_ml.py | 23 ++++++++++++++++++ sde_collections/models/collection.py | 5 ++++ .../utils/paired_field_descriptor.py | 24 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py create mode 100644 sde_collections/utils/paired_field_descriptor.py diff --git a/sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py b/sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py new file mode 100644 index 00000000..557ad13e --- /dev/null +++ b/sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.9 on 2024-10-30 00:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0058_candidateurl_division_collection_is_multi_division_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="collection", + name="tdamm_manual", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="collection", + name="tdamm_ml", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index 31306b8c..a2d3181c 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -26,6 +26,7 @@ UpdateFrequencies, WorkflowStatusChoices, ) +from ..utils.paired_field_descriptor import PairedFieldDescriptor User = get_user_model() @@ -33,6 +34,10 @@ class Collection(models.Model): """Model definition for Collection.""" + tdamm_manual = models.CharField(max_length=255, null=True, blank=True) + tdamm_ml = models.CharField(max_length=255, null=True, blank=True) + tdamm = PairedFieldDescriptor('tdamm') + name = models.CharField("Name", max_length=1024) config_folder = models.CharField("Config Folder", max_length=2048, unique=True, editable=False) url = models.URLField("URL", max_length=2048) diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py new file mode 100644 index 00000000..e07d41dc --- /dev/null +++ b/sde_collections/utils/paired_field_descriptor.py @@ -0,0 +1,24 @@ +from django.db import models + + +class PairedFieldDescriptor: + def __init__(self, field_name): + self.manual_field_name = f"{field_name}_manual" + self.ml_field_name = f"{field_name}_ml" + + def __get__(self, instance, owner): + if instance is None: + return self + # Return manual tag if available, otherwise ML tag + manual_value = getattr(instance, self.manual_field_name, None) + machine_learning_value = getattr(instance, self.ml_field_name, None) + return manual_value if manual_value is not None else machine_learning_value + + def __set__(self, instance, value): + # Set the value of the manual field + setattr(instance, self.manual_field_name, value) + + def __delete__(self, instance): + # Delete both manual and ML fields + delattr(instance, self.manual_field_name) + delattr(instance, self.ml_field_name) From a469ef1242824645885433ac0d3ecd8d4a23a7fe Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 30 Oct 2024 16:14:07 -0500 Subject: [PATCH 02/21] add fields to admin panel --- sde_collections/admin.py | 35 +++++++++++++++++++ ...remove_collection_tdamm_manual_and_more.py | 31 ++++++++++++++++ sde_collections/models/collection.py | 6 ++-- sde_collections/serializers.py | 1 + 4 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py diff --git a/sde_collections/admin.py b/sde_collections/admin.py index cb105f80..add9a906 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -7,6 +7,7 @@ from .models.collection import Collection, WorkflowHistory from .models.pattern import DivisionPattern, IncludePattern, TitlePattern from .tasks import import_candidate_urls_from_api +from django import forms @admin.action(description="Generate deployment message") @@ -174,10 +175,34 @@ def update_config(self, request, queryset): update_config.short_description = "Update configs of selected" +class CollectionForm(forms.ModelForm): + tdamm_tag = forms.CharField(required=False, label="TDAMM Tag") + + class Meta: + model = Collection + fields = "__all__" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.instance and hasattr(self.instance, "tdamm_tag"): + # Set the initial value of tdamm_tag to the computed value + self.fields["tdamm_tag"].initial = self.instance.tdamm_tag + + def clean(self): + cleaned_data = super().clean() + tdamm_value = cleaned_data.get("tdamm_tag") + if tdamm_value: + # Set the manual field with the value from tdamm + cleaned_data["tdamm_tag_manual"] = tdamm_value + return cleaned_data + + @admin.register(Collection) class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): """Admin View for Collection""" + form = CollectionForm + fieldsets = ( ( "Essential information", @@ -187,6 +212,9 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): "config_folder", "url", "division", + "tdamm_tag", + "tdamm_tag_ml", + "tdamm_tag_manual", "document_type", "update_frequency", "source", @@ -215,15 +243,22 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): ), ) + def tdamm_tag(self, obj): + return obj.tdamm_tag + list_display = ( "name", "candidate_urls_count", "config_folder", "url", + "tdamm_tag", + "tdamm_tag_ml", + "tdamm_tag_manual", "division", "new_collection", "is_multi_division", ) + readonly_fields = ("config_folder",) list_filter = ("division", "curation_status", "workflow_status", "turned_on", "is_multi_division") search_fields = ("name", "url", "config_folder") diff --git a/sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py b/sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py new file mode 100644 index 00000000..37b817a7 --- /dev/null +++ b/sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.9 on 2024-10-30 21:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0059_collection_tdamm_manual_collection_tdamm_ml"), + ] + + operations = [ + migrations.RemoveField( + model_name="collection", + name="tdamm_manual", + ), + migrations.RemoveField( + model_name="collection", + name="tdamm_ml", + ), + migrations.AddField( + model_name="collection", + name="tdamm_tag_manual", + field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM Manual Tag"), + ), + migrations.AddField( + model_name="collection", + name="tdamm_tag_ml", + field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM ML Tag"), + ), + ] diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index a2d3181c..1d140889 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -34,9 +34,9 @@ class Collection(models.Model): """Model definition for Collection.""" - tdamm_manual = models.CharField(max_length=255, null=True, blank=True) - tdamm_ml = models.CharField(max_length=255, null=True, blank=True) - tdamm = PairedFieldDescriptor('tdamm') + tdamm_tag_manual = models.CharField(max_length=255, null=True, blank=True, verbose_name="TDAMM Manual Tag") + tdamm_tag_ml = models.CharField(max_length=255, null=True, blank=True, verbose_name="TDAMM ML Tag") + tdamm_tag = PairedFieldDescriptor('tdamm_tag') name = models.CharField("Name", max_length=1024) config_folder = models.CharField("Config Folder", max_length=2048, unique=True, editable=False) diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py index 9623e85d..19717818 100644 --- a/sde_collections/serializers.py +++ b/sde_collections/serializers.py @@ -26,6 +26,7 @@ class Meta: "workflow_status_display", "curated_by", "division", + "tdamm_tag", "document_type", "name", ) From 8e8e0ac743f6a915e8196c6dc9914060766315eb Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Mon, 4 Nov 2024 00:13:31 -0600 Subject: [PATCH 03/21] moved tdamm_tags feature from collection to candidate_url --- sde_collections/admin.py | 127 ++++++++++++----- ..._candidateurl_tdamm_tag_manual_and_more.py | 134 ++++++++++++++++++ ...ection_tdamm_manual_collection_tdamm_ml.py | 23 --- ...remove_collection_tdamm_manual_and_more.py | 31 ---- sde_collections/models/candidate_url.py | 60 +++++++- sde_collections/models/collection.py | 5 - sde_collections/serializers.py | 2 +- 7 files changed, 287 insertions(+), 95 deletions(-) create mode 100644 sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py delete mode 100644 sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py delete mode 100644 sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py diff --git a/sde_collections/admin.py b/sde_collections/admin.py index add9a906..73576899 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -8,6 +8,7 @@ from .models.pattern import DivisionPattern, IncludePattern, TitlePattern from .tasks import import_candidate_urls_from_api from django import forms +from django.contrib.postgres.fields import ArrayField @admin.action(description="Generate deployment message") @@ -175,34 +176,10 @@ def update_config(self, request, queryset): update_config.short_description = "Update configs of selected" -class CollectionForm(forms.ModelForm): - tdamm_tag = forms.CharField(required=False, label="TDAMM Tag") - - class Meta: - model = Collection - fields = "__all__" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if self.instance and hasattr(self.instance, "tdamm_tag"): - # Set the initial value of tdamm_tag to the computed value - self.fields["tdamm_tag"].initial = self.instance.tdamm_tag - - def clean(self): - cleaned_data = super().clean() - tdamm_value = cleaned_data.get("tdamm_tag") - if tdamm_value: - # Set the manual field with the value from tdamm - cleaned_data["tdamm_tag_manual"] = tdamm_value - return cleaned_data - - @admin.register(Collection) class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): """Admin View for Collection""" - form = CollectionForm - fieldsets = ( ( "Essential information", @@ -212,9 +189,6 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): "config_folder", "url", "division", - "tdamm_tag", - "tdamm_tag_ml", - "tdamm_tag_manual", "document_type", "update_frequency", "source", @@ -243,17 +217,11 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin): ), ) - def tdamm_tag(self, obj): - return obj.tdamm_tag - list_display = ( "name", "candidate_urls_count", "config_folder", "url", - "tdamm_tag", - "tdamm_tag_ml", - "tdamm_tag_manual", "division", "new_collection", "is_multi_division", @@ -296,13 +264,104 @@ def exclude_and_delete_children(modeladmin, request, queryset): for candidate_url in queryset.all(): candidate_url.get_children().delete() +class CandidateURLForm(forms.ModelForm): + # tdamm_tag = forms.MultipleChoiceField( + # choices=CandidateURL.TDAMM_TAG_CHOICES, + # required=False, + # label="TDAMM Tags", + # widget=forms.CheckboxSelectMultiple, + # ) + + tdamm_tag_ml = forms.MultipleChoiceField( + choices=CandidateURL.TDAMM_TAG_CHOICES, + required=False, + label="TDAMM ML Tags", + widget=forms.CheckboxSelectMultiple, + ) + + tdamm_tag_manual = forms.MultipleChoiceField( + choices=CandidateURL.TDAMM_TAG_CHOICES, + required=False, + label="TDAMM Manual Tags", + widget=forms.CheckboxSelectMultiple, + ) + + class Meta: + model = CandidateURL + fields = '__all__' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Initialize tdamm_tag + # if self.instance and hasattr(self.instance, 'tdamm_tag'): + # self.fields['tdamm_tag'].initial = self.instance.tdamm_tag or [] + + # Initialize tdamm_tag_ml + if self.instance and self.instance.tdamm_tag_ml: + self.fields['tdamm_tag_ml'].initial = self.instance.tdamm_tag_ml + + # Initialize tdamm_tag_manual + if self.instance and self.instance.tdamm_tag_manual: + self.fields['tdamm_tag_manual'].initial = self.instance.tdamm_tag_manual + + def clean(self): + cleaned_data = super().clean() + + # Handle tdamm_tag + # tdamm_tag_value = cleaned_data.get('tdamm_tag', []) + # if not tdamm_tag_value: + # cleaned_data['tdamm_tag_manual'] = None + # else: + # cleaned_data['tdamm_tag_manual'] = tdamm_tag_value + + # Handle tdamm_tag_ml + tdamm_tag_ml_value = cleaned_data.get('tdamm_tag_ml', []) + if not tdamm_tag_ml_value: + cleaned_data['tdamm_tag_ml'] = None + + # Handle tdamm_tag_manual + tdamm_tag_manual_value = cleaned_data.get('tdamm_tag_manual', []) + if not tdamm_tag_manual_value: + cleaned_data['tdamm_tag_manual'] = None + + return cleaned_data class CandidateURLAdmin(admin.ModelAdmin): """Admin View for CandidateURL""" - list_display = ("url", "scraped_title", "collection") + form = CandidateURLForm + + list_display = ( + "url", + "scraped_title", + "collection", + # "tdamm_tag_display", + "tdamm_tag_ml_display", + "tdamm_tag_manual_display" + ) list_filter = ("collection",) + # @admin.display(description='TDAMM Tags') + # def tdamm_tag_display(self, obj): + # if obj.tdamm_tag: + # readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag] + # return ", ".join(readable_tags) + # return "" + + @admin.display(description='TDAMM ML Tags') + def tdamm_tag_ml_display(self, obj): + if obj.tdamm_tag_ml: + readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag_ml] + return ", ".join(readable_tags) + return "" + + @admin.display(description='TDAMM Manual Tags') + def tdamm_tag_manual_display(self, obj): + if obj.tdamm_tag_manual: + readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag_manual] + return ", ".join(readable_tags) + return "" + class TitlePatternAdmin(admin.ModelAdmin): """Admin View for TitlePattern""" diff --git a/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py new file mode 100644 index 00000000..057f1ed6 --- /dev/null +++ b/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py @@ -0,0 +1,134 @@ +# Generated by Django 4.2.9 on 2024-11-02 04:36 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0058_candidateurl_division_collection_is_multi_division_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="candidateurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + null=True, + size=None, + verbose_name="TDAMM Manual Tags", + ), + ), + migrations.AddField( + model_name="candidateurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + null=True, + size=None, + verbose_name="TDAMM ML Tags", + ), + ), + migrations.AddField( + model_name="collection", + name="tdamm_tag_manual", + field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM Manual Tag"), + ), + migrations.AddField( + model_name="collection", + name="tdamm_tag_ml", + field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM ML Tag"), + ), + ] diff --git a/sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py b/sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py deleted file mode 100644 index 557ad13e..00000000 --- a/sde_collections/migrations/0059_collection_tdamm_manual_collection_tdamm_ml.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.9 on 2024-10-30 00:44 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("sde_collections", "0058_candidateurl_division_collection_is_multi_division_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="collection", - name="tdamm_manual", - field=models.CharField(blank=True, max_length=255, null=True), - ), - migrations.AddField( - model_name="collection", - name="tdamm_ml", - field=models.CharField(blank=True, max_length=255, null=True), - ), - ] diff --git a/sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py b/sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py deleted file mode 100644 index 37b817a7..00000000 --- a/sde_collections/migrations/0060_remove_collection_tdamm_manual_and_more.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 4.2.9 on 2024-10-30 21:05 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("sde_collections", "0059_collection_tdamm_manual_collection_tdamm_ml"), - ] - - operations = [ - migrations.RemoveField( - model_name="collection", - name="tdamm_manual", - ), - migrations.RemoveField( - model_name="collection", - name="tdamm_ml", - ), - migrations.AddField( - model_name="collection", - name="tdamm_tag_manual", - field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM Manual Tag"), - ), - migrations.AddField( - model_name="collection", - name="tdamm_tag_ml", - field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM ML Tag"), - ), - ] diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index 51c3a28b..f8c91a97 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -7,7 +7,8 @@ from .collection import Collection from .collection_choice_fields import Divisions, DocumentTypes from .pattern import ExcludePattern, TitlePattern - +from ..utils.paired_field_descriptor import PairedFieldDescriptor +from django.contrib.postgres.fields import ArrayField class CandidateURLQuerySet(models.QuerySet): def with_exclusion_status(self): @@ -80,6 +81,63 @@ class CandidateURL(models.Model): help_text="Helps keep track if the Current URL is present in production or not", ) + TDAMM_TAG_CHOICES = [ + ('MMA_M_EM', 'Messenger - EM Radiation'), + ('MMA_M_EM_G', 'Messenger - EM Radiation - Gamma rays'), + ('MMA_M_EM_X', 'Messenger - EM Radiation - X-rays'), + ('MMA_M_EM_U', 'Messenger - EM Radiation - Ultraviolet'), + ('MMA_M_EM_O', 'Messenger - EM Radiation - Optical'), + ('MMA_M_EM_I', 'Messenger - EM Radiation - Infrared'), + ('MMA_M_EM_M', 'Messenger - EM Radiation - Microwave'), + ('MMA_M_EM_R', 'Messenger - EM Radiation - Radio'), + ('MMA_M_G', 'Messenger - Gravitational Waves'), + ('MMA_M_G_CBI', 'Messenger - Gravitational Waves - Compact Binary Inspiral'), + ('MMA_M_G_S', 'Messenger - Gravitational Waves - Stochastic'), + ('MMA_M_G_CON', 'Messenger - Gravitational Waves - Continuous'), + ('MMA_M_G_B', 'Messenger - Gravitational Waves - Burst'), + ('MMA_M_C', 'Messenger - Cosmic Rays'), + ('MMA_M_N', 'Messenger - Neutrinos'), + ('MMA_O_BI', 'Objects - Binaries'), + ('MMA_O_BI_BBH', 'Objects - Binaries - Binary Black Holes'), + ('MMA_O_BI_BNS', 'Objects - Binaries - Binary Neutron Stars'), + ('MMA_O_BI_C', 'Objects - Binaries - Cataclysmic Variables'), + ('MMA_O_BI_N', 'Objects - Binaries - Neutron Star-Black Hole'), + ('MMA_O_BI_B', 'Objects - Binaries - Binary Pulsars'), + ('MMA_O_BI_W', 'Objects - Binaries - White Dwarf Binaries'), + ('MMA_O_BH', 'Objects - Black Holes'), + ('MMA_O_BH_AGN', 'Objects - Black Holes - Active Galactic Nuclei'), + ('MMA_O_BH_IM', 'Objects - Black Holes - Intermediate mass'), + ('MMA_O_BH_STM', 'Objects - Black Holes - Stellar mass'), + ('MMA_O_BH_SUM', 'Objects - Black Holes - Supermassive'), + ('MMA_O_E', 'Objects - Exoplanets'), + ('MMA_O_N', 'Objects - Neutron Stars'), + ('MMA_O_N_M', 'Objects - Neutron Stars - Magnetars'), + ('MMA_O_N_P', 'Objects - Neutron Stars - Pulsars'), + ('MMA_O_N_PWN', 'Objects - Neutron Stars - Pulsar Wind Nebula'), + ('MMA_O_S', 'Objects - Supernova Remnants'), + ('MMA_S_F', 'Signals - Fast Radio Bursts'), + ('MMA_S_G', 'Signals - Gamma-ray Bursts'), + ('MMA_S_K', 'Signals - Kilonovae'), + ('MMA_S_N', 'Signals - Novae'), + ('MMA_S_P', 'Signals - Pevatrons'), + ('MMA_S_ST', 'Signals - Stellar flares'), + ('MMA_S_SU', 'Signals - Supernovae'), + ] + + tdamm_tag_manual = ArrayField( + models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), + blank=True, + null=True, + verbose_name="TDAMM Manual Tags" + ) + tdamm_tag_ml = ArrayField( + models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), + blank=True, + null=True, + verbose_name="TDAMM ML Tags" + ) + tdamm_tag = PairedFieldDescriptor('tdamm_tag') + class Meta: """Meta definition for Candidate URL.""" diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index 1d140889..31306b8c 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -26,7 +26,6 @@ UpdateFrequencies, WorkflowStatusChoices, ) -from ..utils.paired_field_descriptor import PairedFieldDescriptor User = get_user_model() @@ -34,10 +33,6 @@ class Collection(models.Model): """Model definition for Collection.""" - tdamm_tag_manual = models.CharField(max_length=255, null=True, blank=True, verbose_name="TDAMM Manual Tag") - tdamm_tag_ml = models.CharField(max_length=255, null=True, blank=True, verbose_name="TDAMM ML Tag") - tdamm_tag = PairedFieldDescriptor('tdamm_tag') - name = models.CharField("Name", max_length=1024) config_folder = models.CharField("Config Folder", max_length=2048, unique=True, editable=False) url = models.URLField("URL", max_length=2048) diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py index 19717818..b7bb3b25 100644 --- a/sde_collections/serializers.py +++ b/sde_collections/serializers.py @@ -26,7 +26,6 @@ class Meta: "workflow_status_display", "curated_by", "division", - "tdamm_tag", "document_type", "name", ) @@ -123,6 +122,7 @@ class Meta: "hash", "file_extension", "tree_root", + "tdamm_tag" ) def get_document_type(self, obj): From f5c69bd4ce64c1edcfdd700e15e0e0404b19ce67 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Mon, 4 Nov 2024 14:46:19 -0600 Subject: [PATCH 04/21] refactor code --- sde_collections/admin.py | 171 ++++++++++++------ ..._candidateurl_tdamm_tag_manual_and_more.py | 151 ++++++++++++++++ sde_collections/models/candidate_url.py | 120 +++++++----- sde_collections/serializers.py | 21 ++- .../utils/paired_field_descriptor.py | 3 - 5 files changed, 349 insertions(+), 117 deletions(-) create mode 100644 sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 73576899..0860d0e5 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -8,7 +8,6 @@ from .models.pattern import DivisionPattern, IncludePattern, TitlePattern from .tasks import import_candidate_urls_from_api from django import forms -from django.contrib.postgres.fields import ArrayField @admin.action(description="Generate deployment message") @@ -264,14 +263,8 @@ def exclude_and_delete_children(modeladmin, request, queryset): for candidate_url in queryset.all(): candidate_url.get_children().delete() -class CandidateURLForm(forms.ModelForm): - # tdamm_tag = forms.MultipleChoiceField( - # choices=CandidateURL.TDAMM_TAG_CHOICES, - # required=False, - # label="TDAMM Tags", - # widget=forms.CheckboxSelectMultiple, - # ) +class CandidateURLForm(forms.ModelForm): tdamm_tag_ml = forms.MultipleChoiceField( choices=CandidateURL.TDAMM_TAG_CHOICES, required=False, @@ -285,83 +278,141 @@ class CandidateURLForm(forms.ModelForm): label="TDAMM Manual Tags", widget=forms.CheckboxSelectMultiple, ) - + class Meta: model = CandidateURL - fields = '__all__' + fields = "__all__" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Initialize tdamm_tag - # if self.instance and hasattr(self.instance, 'tdamm_tag'): - # self.fields['tdamm_tag'].initial = self.instance.tdamm_tag or [] - - # Initialize tdamm_tag_ml - if self.instance and self.instance.tdamm_tag_ml: - self.fields['tdamm_tag_ml'].initial = self.instance.tdamm_tag_ml - - # Initialize tdamm_tag_manual - if self.instance and self.instance.tdamm_tag_manual: - self.fields['tdamm_tag_manual'].initial = self.instance.tdamm_tag_manual + instance = kwargs.get("instance") + + # Only show TDAMM fields if is_tdamm is True + if not instance or not instance.is_tdamm: + if "tdamm_tag_ml" in self.fields: + del self.fields["tdamm_tag_ml"] + if "tdamm_tag_manual" in self.fields: + del self.fields["tdamm_tag_manual"] + else: + # Initialize tdamm fields only if is_tdamm is True + if hasattr(self.instance, "tdamm_tag_ml"): + self.fields["tdamm_tag_ml"].initial = self.instance.tdamm_tag_ml or [] + + if hasattr(self.instance, "tdamm_tag_manual"): + self.fields["tdamm_tag_manual"].initial = self.instance.tdamm_tag_manual or [] def clean(self): cleaned_data = super().clean() - - # Handle tdamm_tag - # tdamm_tag_value = cleaned_data.get('tdamm_tag', []) - # if not tdamm_tag_value: - # cleaned_data['tdamm_tag_manual'] = None - # else: - # cleaned_data['tdamm_tag_manual'] = tdamm_tag_value - - # Handle tdamm_tag_ml - tdamm_tag_ml_value = cleaned_data.get('tdamm_tag_ml', []) - if not tdamm_tag_ml_value: - cleaned_data['tdamm_tag_ml'] = None - - # Handle tdamm_tag_manual - tdamm_tag_manual_value = cleaned_data.get('tdamm_tag_manual', []) - if not tdamm_tag_manual_value: - cleaned_data['tdamm_tag_manual'] = None - return cleaned_data + def save(self, commit=True): + instance = super().save(commit=False) + + # Handle TDAMM fields if is_tdamm is True + if instance.is_tdamm: + # Get values from the form + tdamm_tag_ml = self.cleaned_data.get("tdamm_tag_ml", []) + tdamm_tag_manual = self.cleaned_data.get("tdamm_tag_manual", []) + + # Set the values directly on the instance + instance.tdamm_tag_ml = tdamm_tag_ml or None + instance.tdamm_tag_manual = tdamm_tag_manual or None + else: + # Clear TDAMM fields if is_tdamm is False + instance.tdamm_tag_ml = None + instance.tdamm_tag_manual = None + + if commit: + instance.save() + + return instance + + class CandidateURLAdmin(admin.ModelAdmin): """Admin View for CandidateURL""" form = CandidateURLForm - list_display = ( - "url", - "scraped_title", - "collection", - # "tdamm_tag_display", - "tdamm_tag_ml_display", - "tdamm_tag_manual_display" - ) - list_filter = ("collection",) - - # @admin.display(description='TDAMM Tags') - # def tdamm_tag_display(self, obj): - # if obj.tdamm_tag: - # readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag] - # return ", ".join(readable_tags) - # return "" - - @admin.display(description='TDAMM ML Tags') + def get_list_display(self, request): + list_display = [ + "url", + "scraped_title", + "collection", + "is_tdamm", + ] + # Add TDAMM-related fields only if any TDAMM-enabled URLs exist + if CandidateURL.objects.filter(is_tdamm=True).exists(): + list_display.extend(["tdamm_tag_ml_display", "tdamm_tag_manual_display"]) + return list_display + + list_filter = ("collection", "is_tdamm") + + @admin.display(description="TDAMM ML Tags") def tdamm_tag_ml_display(self, obj): - if obj.tdamm_tag_ml: + if obj.is_tdamm and obj.tdamm_tag_ml: readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag_ml] return ", ".join(readable_tags) return "" - @admin.display(description='TDAMM Manual Tags') + @admin.display(description="TDAMM Manual Tags") def tdamm_tag_manual_display(self, obj): - if obj.tdamm_tag_manual: + if obj.is_tdamm and obj.tdamm_tag_manual: readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag_manual] return ", ".join(readable_tags) return "" + def get_fieldsets(self, request, obj=None): + """Dynamically adjust fieldsets based on is_tdamm""" + fieldsets = [ + ( + "Essential Information", + { + "fields": ( + "collection", + "url", + "hash", + "scraped_title", + "generated_title", + "test_title", + "production_title", + "level", + "visited", + "document_type", + "division", + "inferenced_by", + "is_pdf", + "present_on_test", + "present_on_prod", + "is_tdamm", + ) + }, + ), + ] + + # Add TDAMM fields only if is_tdamm is True + if obj and obj.is_tdamm: + fieldsets.append( + ( + "TDAMM Tags", + { + "fields": ( + "tdamm_tag_ml", + "tdamm_tag_manual", + ), + "classes": ("collapse",), + }, + ) + ) + + return fieldsets + + def save_model(self, request, obj, form, change): + """Ensure proper saving of the model""" + if not obj.is_tdamm: + obj.tdamm_tag_ml = None + obj.tdamm_tag_manual = None + super().save_model(request, obj, form, change) + class TitlePatternAdmin(admin.ModelAdmin): """Admin View for TitlePattern""" diff --git a/sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py new file mode 100644 index 00000000..d8a0a4a7 --- /dev/null +++ b/sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py @@ -0,0 +1,151 @@ +# Generated by Django 4.2.9 on 2024-11-04 06:33 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0059_candidateurl_tdamm_tag_manual_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="candidateurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + verbose_name="TDAMM Manual Tags", + ), + ), + migrations.RenameField( + model_name="candidateurl", + old_name="tdamm_tag_manual", + new_name="_tdamm_tag_manual", + ), + migrations.AlterField( + model_name="candidateurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + verbose_name="TDAMM ML Tags", + ), + ), + migrations.RenameField( + model_name="candidateurl", + old_name="tdamm_tag_ml", + new_name="_tdamm_tag_ml", + ), + migrations.RemoveField( + model_name="collection", + name="tdamm_tag_manual", + ), + migrations.RemoveField( + model_name="collection", + name="tdamm_tag_ml", + ), + migrations.AddField( + model_name="candidateurl", + name="is_tdamm", + field=models.BooleanField( + default=False, help_text="Enable TDAMM tagging for this URL", verbose_name="Is TDAMM" + ), + ), + ] diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index f8c91a97..41c1072f 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -10,6 +10,7 @@ from ..utils.paired_field_descriptor import PairedFieldDescriptor from django.contrib.postgres.fields import ArrayField + class CandidateURLQuerySet(models.QuerySet): def with_exclusion_status(self): return self.annotate( @@ -80,63 +81,91 @@ class CandidateURL(models.Model): default=False, help_text="Helps keep track if the Current URL is present in production or not", ) + is_tdamm = models.BooleanField("Is TDAMM?", default=False, help_text="Enable TDAMM tagging for this URL") TDAMM_TAG_CHOICES = [ - ('MMA_M_EM', 'Messenger - EM Radiation'), - ('MMA_M_EM_G', 'Messenger - EM Radiation - Gamma rays'), - ('MMA_M_EM_X', 'Messenger - EM Radiation - X-rays'), - ('MMA_M_EM_U', 'Messenger - EM Radiation - Ultraviolet'), - ('MMA_M_EM_O', 'Messenger - EM Radiation - Optical'), - ('MMA_M_EM_I', 'Messenger - EM Radiation - Infrared'), - ('MMA_M_EM_M', 'Messenger - EM Radiation - Microwave'), - ('MMA_M_EM_R', 'Messenger - EM Radiation - Radio'), - ('MMA_M_G', 'Messenger - Gravitational Waves'), - ('MMA_M_G_CBI', 'Messenger - Gravitational Waves - Compact Binary Inspiral'), - ('MMA_M_G_S', 'Messenger - Gravitational Waves - Stochastic'), - ('MMA_M_G_CON', 'Messenger - Gravitational Waves - Continuous'), - ('MMA_M_G_B', 'Messenger - Gravitational Waves - Burst'), - ('MMA_M_C', 'Messenger - Cosmic Rays'), - ('MMA_M_N', 'Messenger - Neutrinos'), - ('MMA_O_BI', 'Objects - Binaries'), - ('MMA_O_BI_BBH', 'Objects - Binaries - Binary Black Holes'), - ('MMA_O_BI_BNS', 'Objects - Binaries - Binary Neutron Stars'), - ('MMA_O_BI_C', 'Objects - Binaries - Cataclysmic Variables'), - ('MMA_O_BI_N', 'Objects - Binaries - Neutron Star-Black Hole'), - ('MMA_O_BI_B', 'Objects - Binaries - Binary Pulsars'), - ('MMA_O_BI_W', 'Objects - Binaries - White Dwarf Binaries'), - ('MMA_O_BH', 'Objects - Black Holes'), - ('MMA_O_BH_AGN', 'Objects - Black Holes - Active Galactic Nuclei'), - ('MMA_O_BH_IM', 'Objects - Black Holes - Intermediate mass'), - ('MMA_O_BH_STM', 'Objects - Black Holes - Stellar mass'), - ('MMA_O_BH_SUM', 'Objects - Black Holes - Supermassive'), - ('MMA_O_E', 'Objects - Exoplanets'), - ('MMA_O_N', 'Objects - Neutron Stars'), - ('MMA_O_N_M', 'Objects - Neutron Stars - Magnetars'), - ('MMA_O_N_P', 'Objects - Neutron Stars - Pulsars'), - ('MMA_O_N_PWN', 'Objects - Neutron Stars - Pulsar Wind Nebula'), - ('MMA_O_S', 'Objects - Supernova Remnants'), - ('MMA_S_F', 'Signals - Fast Radio Bursts'), - ('MMA_S_G', 'Signals - Gamma-ray Bursts'), - ('MMA_S_K', 'Signals - Kilonovae'), - ('MMA_S_N', 'Signals - Novae'), - ('MMA_S_P', 'Signals - Pevatrons'), - ('MMA_S_ST', 'Signals - Stellar flares'), - ('MMA_S_SU', 'Signals - Supernovae'), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), ] - tdamm_tag_manual = ArrayField( + # Define TDAMM fields but make them optional + @property + def tdamm_tag_manual(self): + if hasattr(self, "_tdamm_tag_manual") and self.is_tdamm: + return self._tdamm_tag_manual + return None + + @tdamm_tag_manual.setter + def tdamm_tag_manual(self, value): + if self.is_tdamm: + self._tdamm_tag_manual = value + + @property + def tdamm_tag_ml(self): + if hasattr(self, "_tdamm_tag_ml") and self.is_tdamm: + return self._tdamm_tag_ml + return None + + @tdamm_tag_ml.setter + def tdamm_tag_ml(self, value): + if self.is_tdamm: + self._tdamm_tag_ml = value + + _tdamm_tag_manual = ArrayField( models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), blank=True, null=True, - verbose_name="TDAMM Manual Tags" + verbose_name="TDAMM Manual Tags", + db_column="tdamm_tag_manual", ) - tdamm_tag_ml = ArrayField( + + _tdamm_tag_ml = ArrayField( models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), blank=True, null=True, - verbose_name="TDAMM ML Tags" + verbose_name="TDAMM ML Tags", + db_column="tdamm_tag_ml", ) - tdamm_tag = PairedFieldDescriptor('tdamm_tag') + + tdamm_tag = PairedFieldDescriptor("tdamm_tag") class Meta: """Meta definition for Candidate URL.""" @@ -144,6 +173,7 @@ class Meta: verbose_name = "Candidate URL" verbose_name_plural = "Candidate URLs" ordering = ["url"] + db_table = "sde_collections_candidateurl" @property def fileext(self) -> str: diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py index b7bb3b25..29d86c31 100644 --- a/sde_collections/serializers.py +++ b/sde_collections/serializers.py @@ -112,18 +112,21 @@ class CandidateURLAPISerializer(serializers.ModelSerializer): title = serializers.SerializerMethodField() file_extension = serializers.SerializerMethodField() tree_root = serializers.SerializerMethodField() + tdamm_tag = serializers.SerializerMethodField() class Meta: model = CandidateURL - fields = ( - "url", - "title", - "document_type", - "hash", - "file_extension", - "tree_root", - "tdamm_tag" - ) + fields = ("url", "title", "document_type", "hash", "file_extension", "tree_root", "is_tdamm", "tdamm_tag") + + def to_representation(self, instance): + """Remove tdamm_tag field if is_tdamm is False""" + representation = super().to_representation(instance) + if not instance.is_tdamm: + representation.pop("tdamm_tag", None) + return representation + + def get_tdamm_tag(self, obj): + return obj.tdamm_tag def get_document_type(self, obj): if obj.document_type is not None: diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py index e07d41dc..9ac0c4e3 100644 --- a/sde_collections/utils/paired_field_descriptor.py +++ b/sde_collections/utils/paired_field_descriptor.py @@ -1,6 +1,3 @@ -from django.db import models - - class PairedFieldDescriptor: def __init__(self, field_name): self.manual_field_name = f"{field_name}_manual" From 7e888e8457f02bbe8417f6e66ecc1d52be9608c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:49:23 +0000 Subject: [PATCH 05/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- sde_collections/admin.py | 2 +- sde_collections/models/candidate_url.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 0860d0e5..bf97cf02 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -1,5 +1,6 @@ import csv +from django import forms from django.contrib import admin, messages from django.http import HttpResponse @@ -7,7 +8,6 @@ from .models.collection import Collection, WorkflowHistory from .models.pattern import DivisionPattern, IncludePattern, TitlePattern from .tasks import import_candidate_urls_from_api -from django import forms @admin.action(description="Generate deployment message") diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index 41c1072f..8d2776dd 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -2,13 +2,13 @@ import os from urllib.parse import urlparse +from django.contrib.postgres.fields import ArrayField from django.db import models +from ..utils.paired_field_descriptor import PairedFieldDescriptor from .collection import Collection from .collection_choice_fields import Divisions, DocumentTypes from .pattern import ExcludePattern, TitlePattern -from ..utils.paired_field_descriptor import PairedFieldDescriptor -from django.contrib.postgres.fields import ArrayField class CandidateURLQuerySet(models.QuerySet): From 1b9d8006e6120f449c719bfc9a9fde6e68f8b1b5 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 13 Nov 2024 02:13:53 -0600 Subject: [PATCH 06/21] modify architecture to define one single field to generate two --- ...candidateurl_tdamm_tag_manual_and_more.py} | 25 +-- ..._candidateurl_tdamm_tag_manual_and_more.py | 151 ------------------ sde_collections/models/candidate_url.py | 46 +----- .../utils/paired_field_descriptor.py | 88 ++++++++-- 4 files changed, 98 insertions(+), 212 deletions(-) rename sde_collections/migrations/{0059_candidateurl_tdamm_tag_manual_and_more.py => 0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py} (93%) delete mode 100644 sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py diff --git a/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py similarity index 93% rename from sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py rename to sde_collections/migrations/0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py index 057f1ed6..c635c833 100644 --- a/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py +++ b/sde_collections/migrations/0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.9 on 2024-11-02 04:36 +# Generated by Django 4.2.9 on 2024-11-13 08:01 import django.contrib.postgres.fields from django.db import migrations, models @@ -11,6 +11,13 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name="candidateurl", + name="is_tdamm", + field=models.BooleanField( + default=False, help_text="Enable TDAMM tagging for this URL", verbose_name="Is TDAMM?" + ), + ), migrations.AddField( model_name="candidateurl", name="tdamm_tag_manual", @@ -61,9 +68,9 @@ class Migration(migrations.Migration): max_length=255, ), blank=True, + db_column="tdamm_tag_manual", null=True, size=None, - verbose_name="TDAMM Manual Tags", ), ), migrations.AddField( @@ -116,19 +123,13 @@ class Migration(migrations.Migration): max_length=255, ), blank=True, + db_column="tdamm_tag_ml", null=True, size=None, - verbose_name="TDAMM ML Tags", ), ), - migrations.AddField( - model_name="collection", - name="tdamm_tag_manual", - field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM Manual Tag"), - ), - migrations.AddField( - model_name="collection", - name="tdamm_tag_ml", - field=models.CharField(blank=True, max_length=255, null=True, verbose_name="TDAMM ML Tag"), + migrations.AlterModelTable( + name="candidateurl", + table="sde_collections_candidateurl", ), ] diff --git a/sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py deleted file mode 100644 index d8a0a4a7..00000000 --- a/sde_collections/migrations/0060_alter_candidateurl_tdamm_tag_manual_and_more.py +++ /dev/null @@ -1,151 +0,0 @@ -# Generated by Django 4.2.9 on 2024-11-04 06:33 - -import django.contrib.postgres.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("sde_collections", "0059_candidateurl_tdamm_tag_manual_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="candidateurl", - name="tdamm_tag_manual", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[ - ("MMA_M_EM", "Messenger - EM Radiation"), - ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), - ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), - ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), - ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), - ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), - ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), - ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), - ("MMA_M_G", "Messenger - Gravitational Waves"), - ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), - ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), - ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), - ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), - ("MMA_M_C", "Messenger - Cosmic Rays"), - ("MMA_M_N", "Messenger - Neutrinos"), - ("MMA_O_BI", "Objects - Binaries"), - ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), - ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), - ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), - ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), - ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), - ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), - ("MMA_O_BH", "Objects - Black Holes"), - ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), - ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), - ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), - ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), - ("MMA_O_E", "Objects - Exoplanets"), - ("MMA_O_N", "Objects - Neutron Stars"), - ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), - ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), - ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), - ("MMA_O_S", "Objects - Supernova Remnants"), - ("MMA_S_F", "Signals - Fast Radio Bursts"), - ("MMA_S_G", "Signals - Gamma-ray Bursts"), - ("MMA_S_K", "Signals - Kilonovae"), - ("MMA_S_N", "Signals - Novae"), - ("MMA_S_P", "Signals - Pevatrons"), - ("MMA_S_ST", "Signals - Stellar flares"), - ("MMA_S_SU", "Signals - Supernovae"), - ], - max_length=255, - ), - blank=True, - db_column="tdamm_tag_manual", - null=True, - size=None, - verbose_name="TDAMM Manual Tags", - ), - ), - migrations.RenameField( - model_name="candidateurl", - old_name="tdamm_tag_manual", - new_name="_tdamm_tag_manual", - ), - migrations.AlterField( - model_name="candidateurl", - name="tdamm_tag_ml", - field=django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - choices=[ - ("MMA_M_EM", "Messenger - EM Radiation"), - ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), - ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), - ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), - ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), - ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), - ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), - ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), - ("MMA_M_G", "Messenger - Gravitational Waves"), - ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), - ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), - ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), - ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), - ("MMA_M_C", "Messenger - Cosmic Rays"), - ("MMA_M_N", "Messenger - Neutrinos"), - ("MMA_O_BI", "Objects - Binaries"), - ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), - ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), - ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), - ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), - ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), - ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), - ("MMA_O_BH", "Objects - Black Holes"), - ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), - ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), - ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), - ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), - ("MMA_O_E", "Objects - Exoplanets"), - ("MMA_O_N", "Objects - Neutron Stars"), - ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), - ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), - ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), - ("MMA_O_S", "Objects - Supernova Remnants"), - ("MMA_S_F", "Signals - Fast Radio Bursts"), - ("MMA_S_G", "Signals - Gamma-ray Bursts"), - ("MMA_S_K", "Signals - Kilonovae"), - ("MMA_S_N", "Signals - Novae"), - ("MMA_S_P", "Signals - Pevatrons"), - ("MMA_S_ST", "Signals - Stellar flares"), - ("MMA_S_SU", "Signals - Supernovae"), - ], - max_length=255, - ), - blank=True, - db_column="tdamm_tag_ml", - null=True, - size=None, - verbose_name="TDAMM ML Tags", - ), - ), - migrations.RenameField( - model_name="candidateurl", - old_name="tdamm_tag_ml", - new_name="_tdamm_tag_ml", - ), - migrations.RemoveField( - model_name="collection", - name="tdamm_tag_manual", - ), - migrations.RemoveField( - model_name="collection", - name="tdamm_tag_ml", - ), - migrations.AddField( - model_name="candidateurl", - name="is_tdamm", - field=models.BooleanField( - default=False, help_text="Enable TDAMM tagging for this URL", verbose_name="Is TDAMM" - ), - ), - ] diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index 8d2776dd..ff37b597 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -126,47 +126,13 @@ class CandidateURL(models.Model): ("MMA_S_SU", "Signals - Supernovae"), ] - # Define TDAMM fields but make them optional - @property - def tdamm_tag_manual(self): - if hasattr(self, "_tdamm_tag_manual") and self.is_tdamm: - return self._tdamm_tag_manual - return None - - @tdamm_tag_manual.setter - def tdamm_tag_manual(self, value): - if self.is_tdamm: - self._tdamm_tag_manual = value - - @property - def tdamm_tag_ml(self): - if hasattr(self, "_tdamm_tag_ml") and self.is_tdamm: - return self._tdamm_tag_ml - return None - - @tdamm_tag_ml.setter - def tdamm_tag_ml(self, value): - if self.is_tdamm: - self._tdamm_tag_ml = value - - _tdamm_tag_manual = ArrayField( - models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), - blank=True, - null=True, - verbose_name="TDAMM Manual Tags", - db_column="tdamm_tag_manual", + tdamm_tag = PairedFieldDescriptor( + field_name="tdamm_tag", + field_type=ArrayField(models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), blank=True, null=True), + switch="is_tdamm", + verbose_name="TDAMM Tags", ) - - _tdamm_tag_ml = ArrayField( - models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), - blank=True, - null=True, - verbose_name="TDAMM ML Tags", - db_column="tdamm_tag_ml", - ) - - tdamm_tag = PairedFieldDescriptor("tdamm_tag") - + class Meta: """Meta definition for Candidate URL.""" diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py index 9ac0c4e3..6ed29b60 100644 --- a/sde_collections/utils/paired_field_descriptor.py +++ b/sde_collections/utils/paired_field_descriptor.py @@ -1,21 +1,91 @@ +from django.db import models +from django.contrib.postgres.fields import ArrayField + class PairedFieldDescriptor: - def __init__(self, field_name): + """ + A descriptor that manages paired manual/ML fields where: + - Setting the main field only affects the manual field + - ML field must be set explicitly + - Getting the main field returns manual if present, otherwise ML + """ + + def __init__(self, field_name, field_type, switch, verbose_name=""): + self.field_name = field_name self.manual_field_name = f"{field_name}_manual" self.ml_field_name = f"{field_name}_ml" + self.field_type = field_type + self.verbose_name = verbose_name or field_name.replace('_', ' ').title() + self.switch = switch + + def contribute_to_class(self, cls, name): + """Called by Django when the descriptor is added to the model class.""" + # Create manual field + manual_field = self._create_field( + verbose_name=f"{self.verbose_name} Manual", + db_column=self.manual_field_name + ) + + # Create ML field + ml_field = self._create_field( + verbose_name=f"{self.verbose_name} ML", + db_column=self.ml_field_name + ) + + # Add fields to the model's _meta + cls.add_to_class(self.manual_field_name, manual_field) + cls.add_to_class(self.ml_field_name, ml_field) + + # Store the descriptor + setattr(cls, name, self) + + def _create_field(self, verbose_name, db_column): + """Helper method to create a new field instance with the right configuration""" + if isinstance(self.field_type, type): + # If field_type is a class, instantiate it + field = self.field_type() + else: + # If field_type is already an instance, clone it + field = self.field_type.clone() + + field.verbose_name = verbose_name + field.db_column = db_column + + return field def __get__(self, instance, owner): + """ + Get the value of the main field: + - Returns manual tags if they exist + - Otherwise returns ML tags + - Returns None if switch is False + """ if instance is None: return self - # Return manual tag if available, otherwise ML tag + + if not getattr(instance, self.switch , False): + return None + manual_value = getattr(instance, self.manual_field_name, None) - machine_learning_value = getattr(instance, self.ml_field_name, None) - return manual_value if manual_value is not None else machine_learning_value + ml_value = getattr(instance, self.ml_field_name, None) + + # Return manual if it exists, otherwise ML + return manual_value if manual_value is not None else ml_value def __set__(self, instance, value): - # Set the value of the manual field - setattr(instance, self.manual_field_name, value) + """ + Set only the manual field when setting the field. + ML field must be set explicitly. + """ + if getattr(instance, self.switch, False): + # Only set the manual field + setattr(instance, self.manual_field_name, value) def __delete__(self, instance): - # Delete both manual and ML fields - delattr(instance, self.manual_field_name) - delattr(instance, self.ml_field_name) + """Delete both manual and ML fields""" + setattr(instance, self.manual_field_name, None) + setattr(instance, self.ml_field_name, None) + + def set_ml(self, instance, value): + """Explicit method to set ML tags""" + if getattr(instance, self.switch, False): + setattr(instance, self.ml_field_name, value) \ No newline at end of file From 74b5b7687cd1a3cf81296bba0a0c4d23ea169508 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Nov 2024 08:14:43 +0000 Subject: [PATCH 07/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- sde_collections/models/candidate_url.py | 2 +- .../utils/paired_field_descriptor.py | 31 ++++++++----------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index ff37b597..f85c1388 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -132,7 +132,7 @@ class CandidateURL(models.Model): switch="is_tdamm", verbose_name="TDAMM Tags", ) - + class Meta: """Meta definition for Candidate URL.""" diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py index 6ed29b60..493434fd 100644 --- a/sde_collections/utils/paired_field_descriptor.py +++ b/sde_collections/utils/paired_field_descriptor.py @@ -1,5 +1,6 @@ -from django.db import models from django.contrib.postgres.fields import ArrayField +from django.db import models + class PairedFieldDescriptor: """ @@ -8,28 +9,22 @@ class PairedFieldDescriptor: - ML field must be set explicitly - Getting the main field returns manual if present, otherwise ML """ - + def __init__(self, field_name, field_type, switch, verbose_name=""): self.field_name = field_name self.manual_field_name = f"{field_name}_manual" self.ml_field_name = f"{field_name}_ml" self.field_type = field_type - self.verbose_name = verbose_name or field_name.replace('_', ' ').title() + self.verbose_name = verbose_name or field_name.replace("_", " ").title() self.switch = switch def contribute_to_class(self, cls, name): """Called by Django when the descriptor is added to the model class.""" # Create manual field - manual_field = self._create_field( - verbose_name=f"{self.verbose_name} Manual", - db_column=self.manual_field_name - ) - + manual_field = self._create_field(verbose_name=f"{self.verbose_name} Manual", db_column=self.manual_field_name) + # Create ML field - ml_field = self._create_field( - verbose_name=f"{self.verbose_name} ML", - db_column=self.ml_field_name - ) + ml_field = self._create_field(verbose_name=f"{self.verbose_name} ML", db_column=self.ml_field_name) # Add fields to the model's _meta cls.add_to_class(self.manual_field_name, manual_field) @@ -46,10 +41,10 @@ def _create_field(self, verbose_name, db_column): else: # If field_type is already an instance, clone it field = self.field_type.clone() - + field.verbose_name = verbose_name field.db_column = db_column - + return field def __get__(self, instance, owner): @@ -61,13 +56,13 @@ def __get__(self, instance, owner): """ if instance is None: return self - - if not getattr(instance, self.switch , False): + + if not getattr(instance, self.switch, False): return None manual_value = getattr(instance, self.manual_field_name, None) ml_value = getattr(instance, self.ml_field_name, None) - + # Return manual if it exists, otherwise ML return manual_value if manual_value is not None else ml_value @@ -88,4 +83,4 @@ def __delete__(self, instance): def set_ml(self, instance, value): """Explicit method to set ML tags""" if getattr(instance, self.switch, False): - setattr(instance, self.ml_field_name, value) \ No newline at end of file + setattr(instance, self.ml_field_name, value) From 3ee8845012e78c2a0aee93262ec27ebb1aef6401 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 13 Nov 2024 02:34:24 -0600 Subject: [PATCH 08/21] fix flake8 issues --- sde_collections/utils/paired_field_descriptor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py index 493434fd..1c84e7a3 100644 --- a/sde_collections/utils/paired_field_descriptor.py +++ b/sde_collections/utils/paired_field_descriptor.py @@ -1,7 +1,3 @@ -from django.contrib.postgres.fields import ArrayField -from django.db import models - - class PairedFieldDescriptor: """ A descriptor that manages paired manual/ML fields where: From 13649aee948826b62cdac63cdde06690499a6a0c Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 13 Nov 2024 14:21:34 -0600 Subject: [PATCH 09/21] refactor code for admin panel handling TDAMM --- sde_collections/admin.py | 93 ++----------------- sde_collections/serializers.py | 3 +- .../utils/paired_field_descriptor.py | 6 +- 3 files changed, 15 insertions(+), 87 deletions(-) diff --git a/sde_collections/admin.py b/sde_collections/admin.py index bf97cf02..466bc875 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -265,17 +265,18 @@ def exclude_and_delete_children(modeladmin, request, queryset): class CandidateURLForm(forms.ModelForm): - tdamm_tag_ml = forms.MultipleChoiceField( + # Define the fields as MultipleChoiceFields with checkboxes + tdamm_tag_manual = forms.MultipleChoiceField( choices=CandidateURL.TDAMM_TAG_CHOICES, required=False, - label="TDAMM ML Tags", + label="TDAMM Manual Tags", widget=forms.CheckboxSelectMultiple, ) - tdamm_tag_manual = forms.MultipleChoiceField( + tdamm_tag_ml = forms.MultipleChoiceField( choices=CandidateURL.TDAMM_TAG_CHOICES, required=False, - label="TDAMM Manual Tags", + label="TDAMM ML Tags", widget=forms.CheckboxSelectMultiple, ) @@ -283,83 +284,14 @@ class Meta: model = CandidateURL fields = "__all__" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - instance = kwargs.get("instance") - - # Only show TDAMM fields if is_tdamm is True - if not instance or not instance.is_tdamm: - if "tdamm_tag_ml" in self.fields: - del self.fields["tdamm_tag_ml"] - if "tdamm_tag_manual" in self.fields: - del self.fields["tdamm_tag_manual"] - else: - # Initialize tdamm fields only if is_tdamm is True - if hasattr(self.instance, "tdamm_tag_ml"): - self.fields["tdamm_tag_ml"].initial = self.instance.tdamm_tag_ml or [] - - if hasattr(self.instance, "tdamm_tag_manual"): - self.fields["tdamm_tag_manual"].initial = self.instance.tdamm_tag_manual or [] - - def clean(self): - cleaned_data = super().clean() - return cleaned_data - - def save(self, commit=True): - instance = super().save(commit=False) - - # Handle TDAMM fields if is_tdamm is True - if instance.is_tdamm: - # Get values from the form - tdamm_tag_ml = self.cleaned_data.get("tdamm_tag_ml", []) - tdamm_tag_manual = self.cleaned_data.get("tdamm_tag_manual", []) - - # Set the values directly on the instance - instance.tdamm_tag_ml = tdamm_tag_ml or None - instance.tdamm_tag_manual = tdamm_tag_manual or None - else: - # Clear TDAMM fields if is_tdamm is False - instance.tdamm_tag_ml = None - instance.tdamm_tag_manual = None - - if commit: - instance.save() - - return instance - class CandidateURLAdmin(admin.ModelAdmin): - """Admin View for CandidateURL""" + """Admin view for CandidateURL""" form = CandidateURLForm - - def get_list_display(self, request): - list_display = [ - "url", - "scraped_title", - "collection", - "is_tdamm", - ] - # Add TDAMM-related fields only if any TDAMM-enabled URLs exist - if CandidateURL.objects.filter(is_tdamm=True).exists(): - list_display.extend(["tdamm_tag_ml_display", "tdamm_tag_manual_display"]) - return list_display - - list_filter = ("collection", "is_tdamm") - - @admin.display(description="TDAMM ML Tags") - def tdamm_tag_ml_display(self, obj): - if obj.is_tdamm and obj.tdamm_tag_ml: - readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag_ml] - return ", ".join(readable_tags) - return "" - - @admin.display(description="TDAMM Manual Tags") - def tdamm_tag_manual_display(self, obj): - if obj.is_tdamm and obj.tdamm_tag_manual: - readable_tags = [dict(CandidateURL.TDAMM_TAG_CHOICES).get(tag, tag) for tag in obj.tdamm_tag_manual] - return ", ".join(readable_tags) - return "" + list_display = ["url", "collection", "is_tdamm", "tdamm_tag_manual", "tdamm_tag_ml"] + list_filter = ["collection", "is_tdamm"] + search_fields = ("url", "collection__name") def get_fieldsets(self, request, obj=None): """Dynamically adjust fieldsets based on is_tdamm""" @@ -406,13 +338,6 @@ def get_fieldsets(self, request, obj=None): return fieldsets - def save_model(self, request, obj, form, change): - """Ensure proper saving of the model""" - if not obj.is_tdamm: - obj.tdamm_tag_ml = None - obj.tdamm_tag_manual = None - super().save_model(request, obj, form, change) - class TitlePatternAdmin(admin.ModelAdmin): """Admin View for TitlePattern""" diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py index 29d86c31..06227b31 100644 --- a/sde_collections/serializers.py +++ b/sde_collections/serializers.py @@ -126,7 +126,8 @@ def to_representation(self, instance): return representation def get_tdamm_tag(self, obj): - return obj.tdamm_tag + tags = obj.tdamm_tag + return tags if tags is not None else [] def get_document_type(self, obj): if obj.document_type is not None: diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py index 1c84e7a3..8fdb2a2f 100644 --- a/sde_collections/utils/paired_field_descriptor.py +++ b/sde_collections/utils/paired_field_descriptor.py @@ -59,8 +59,10 @@ def __get__(self, instance, owner): manual_value = getattr(instance, self.manual_field_name, None) ml_value = getattr(instance, self.ml_field_name, None) - # Return manual if it exists, otherwise ML - return manual_value if manual_value is not None else ml_value + # Return manual value only if it exists and is not empty + if manual_value and len(manual_value) > 0: + return manual_value + return ml_value def __set__(self, instance, value): """ From a5a408c13285e04411890854f4bf27d23c1924d8 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Tue, 19 Nov 2024 23:59:48 -0600 Subject: [PATCH 10/21] add TDAMM_TAG_CHOICES to collection_choice_fields --- sde_collections/admin.py | 5 +- sde_collections/models/candidate_url.py | 48 +---------------- .../models/collection_choice_fields.py | 53 +++++++++++++++++++ 3 files changed, 58 insertions(+), 48 deletions(-) diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 466bc875..ba0b477b 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -8,6 +8,7 @@ from .models.collection import Collection, WorkflowHistory from .models.pattern import DivisionPattern, IncludePattern, TitlePattern from .tasks import import_candidate_urls_from_api +from .models.collection_choice_fields import TDAMMTags @admin.action(description="Generate deployment message") @@ -267,14 +268,14 @@ def exclude_and_delete_children(modeladmin, request, queryset): class CandidateURLForm(forms.ModelForm): # Define the fields as MultipleChoiceFields with checkboxes tdamm_tag_manual = forms.MultipleChoiceField( - choices=CandidateURL.TDAMM_TAG_CHOICES, + choices=TDAMMTags.choices, required=False, label="TDAMM Manual Tags", widget=forms.CheckboxSelectMultiple, ) tdamm_tag_ml = forms.MultipleChoiceField( - choices=CandidateURL.TDAMM_TAG_CHOICES, + choices=TDAMMTags.choices, required=False, label="TDAMM ML Tags", widget=forms.CheckboxSelectMultiple, diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index f85c1388..87bfbbaf 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -7,7 +7,7 @@ from ..utils.paired_field_descriptor import PairedFieldDescriptor from .collection import Collection -from .collection_choice_fields import Divisions, DocumentTypes +from .collection_choice_fields import Divisions, DocumentTypes, TDAMMTags from .pattern import ExcludePattern, TitlePattern @@ -82,53 +82,9 @@ class CandidateURL(models.Model): help_text="Helps keep track if the Current URL is present in production or not", ) is_tdamm = models.BooleanField("Is TDAMM?", default=False, help_text="Enable TDAMM tagging for this URL") - - TDAMM_TAG_CHOICES = [ - ("MMA_M_EM", "Messenger - EM Radiation"), - ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), - ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), - ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), - ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), - ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), - ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), - ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), - ("MMA_M_G", "Messenger - Gravitational Waves"), - ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), - ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), - ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), - ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), - ("MMA_M_C", "Messenger - Cosmic Rays"), - ("MMA_M_N", "Messenger - Neutrinos"), - ("MMA_O_BI", "Objects - Binaries"), - ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), - ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), - ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), - ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), - ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), - ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), - ("MMA_O_BH", "Objects - Black Holes"), - ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), - ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), - ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), - ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), - ("MMA_O_E", "Objects - Exoplanets"), - ("MMA_O_N", "Objects - Neutron Stars"), - ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), - ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), - ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), - ("MMA_O_S", "Objects - Supernova Remnants"), - ("MMA_S_F", "Signals - Fast Radio Bursts"), - ("MMA_S_G", "Signals - Gamma-ray Bursts"), - ("MMA_S_K", "Signals - Kilonovae"), - ("MMA_S_N", "Signals - Novae"), - ("MMA_S_P", "Signals - Pevatrons"), - ("MMA_S_ST", "Signals - Stellar flares"), - ("MMA_S_SU", "Signals - Supernovae"), - ] - tdamm_tag = PairedFieldDescriptor( field_name="tdamm_tag", - field_type=ArrayField(models.CharField(max_length=255, choices=TDAMM_TAG_CHOICES), blank=True, null=True), + field_type=ArrayField(models.CharField(max_length=255, choices=TDAMMTags.choices), blank=True, null=True), switch="is_tdamm", verbose_name="TDAMM Tags", ) diff --git a/sde_collections/models/collection_choice_fields.py b/sde_collections/models/collection_choice_fields.py index 3a9a3664..60567abd 100644 --- a/sde_collections/models/collection_choice_fields.py +++ b/sde_collections/models/collection_choice_fields.py @@ -97,3 +97,56 @@ class WorkflowStatusChoices(models.IntegerChoices): PROD_MAJOR = 16, "Prod: Major Issues" MERGE_PENDING = 17, "Code Merge Pending" NEEDS_DELETE = 19, "Delete from Prod" + + +class TDAMMTags(models.TextChoices): + """TDAMM (Tagged Data for Multi-Messenger Astronomy) tag choices.""" + + MMA_M_EM = "MMA_M_EM", "Messenger - EM Radiation" + MMA_M_EM_G = "MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays" + MMA_M_EM_X = "MMA_M_EM_X", "Messenger - EM Radiation - X-rays" + MMA_M_EM_U = "MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet" + MMA_M_EM_O = "MMA_M_EM_O", "Messenger - EM Radiation - Optical" + MMA_M_EM_I = "MMA_M_EM_I", "Messenger - EM Radiation - Infrared" + MMA_M_EM_M = "MMA_M_EM_M", "Messenger - EM Radiation - Microwave" + MMA_M_EM_R = "MMA_M_EM_R", "Messenger - EM Radiation - Radio" + MMA_M_G = "MMA_M_G", "Messenger - Gravitational Waves" + MMA_M_G_CBI = "MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral" + MMA_M_G_S = "MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic" + MMA_M_G_CON = "MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous" + MMA_M_G_B = "MMA_M_G_B", "Messenger - Gravitational Waves - Burst" + MMA_M_C = "MMA_M_C", "Messenger - Cosmic Rays" + MMA_M_N = "MMA_M_N", "Messenger - Neutrinos" + MMA_O_BI = "MMA_O_BI", "Objects - Binaries" + MMA_O_BI_BBH = "MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes" + MMA_O_BI_BNS = "MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars" + MMA_O_BI_C = "MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables" + MMA_O_BI_N = "MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole" + MMA_O_BI_B = "MMA_O_BI_B", "Objects - Binaries - Binary Pulsars" + MMA_O_BI_W = "MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries" + MMA_O_BH = "MMA_O_BH", "Objects - Black Holes" + MMA_O_BH_AGN = "MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei" + MMA_O_BH_IM = "MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass" + MMA_O_BH_STM = "MMA_O_BH_STM", "Objects - Black Holes - Stellar mass" + MMA_O_BH_SUM = "MMA_O_BH_SUM", "Objects - Black Holes - Supermassive" + MMA_O_E = "MMA_O_E", "Objects - Exoplanets" + MMA_O_N = "MMA_O_N", "Objects - Neutron Stars" + MMA_O_N_M = "MMA_O_N_M", "Objects - Neutron Stars - Magnetars" + MMA_O_N_P = "MMA_O_N_P", "Objects - Neutron Stars - Pulsars" + MMA_O_N_PWN = "MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula" + MMA_O_S = "MMA_O_S", "Objects - Supernova Remnants" + MMA_S_F = "MMA_S_F", "Signals - Fast Radio Bursts" + MMA_S_G = "MMA_S_G", "Signals - Gamma-ray Bursts" + MMA_S_K = "MMA_S_K", "Signals - Kilonovae" + MMA_S_N = "MMA_S_N", "Signals - Novae" + MMA_S_P = "MMA_S_P", "Signals - Pevatrons" + MMA_S_ST = "MMA_S_ST", "Signals - Stellar flares" + MMA_S_SU = "MMA_S_SU", "Signals - Supernovae" + + @classmethod + def lookup_by_text(cls, text: str) -> str | None: + """Look up a TDAMM tag by its display text.""" + for choice in cls.choices: + if choice[1].lower() == text.lower(): + return choice[0] + return None From e8b0cf034ecb11d0550c9aee1262b01762d4dba9 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 20 Nov 2024 00:43:45 -0600 Subject: [PATCH 11/21] delete is_tdamm switch functionality --- sde_collections/admin.py | 31 +++++++------------ ...candidateurl_tdamm_tag_manual_and_more.py} | 9 +----- sde_collections/models/candidate_url.py | 3 +- sde_collections/serializers.py | 9 +----- .../utils/paired_field_descriptor.py | 17 ++-------- 5 files changed, 18 insertions(+), 51 deletions(-) rename sde_collections/migrations/{0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py => 0059_candidateurl_tdamm_tag_manual_and_more.py} (95%) diff --git a/sde_collections/admin.py b/sde_collections/admin.py index ba0b477b..02d78434 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -290,12 +290,11 @@ class CandidateURLAdmin(admin.ModelAdmin): """Admin view for CandidateURL""" form = CandidateURLForm - list_display = ["url", "collection", "is_tdamm", "tdamm_tag_manual", "tdamm_tag_ml"] - list_filter = ["collection", "is_tdamm"] + list_display = ["url", "collection", "tdamm_tag_manual", "tdamm_tag_ml"] + list_filter = ["collection"] search_fields = ("url", "collection__name") def get_fieldsets(self, request, obj=None): - """Dynamically adjust fieldsets based on is_tdamm""" fieldsets = [ ( "Essential Information", @@ -316,27 +315,21 @@ def get_fieldsets(self, request, obj=None): "is_pdf", "present_on_test", "present_on_prod", - "is_tdamm", ) }, ), + ( + "TDAMM Tags", + { + "fields": ( + "tdamm_tag_ml", + "tdamm_tag_manual", + ), + "classes": ("collapse",), + }, + ), ] - # Add TDAMM fields only if is_tdamm is True - if obj and obj.is_tdamm: - fieldsets.append( - ( - "TDAMM Tags", - { - "fields": ( - "tdamm_tag_ml", - "tdamm_tag_manual", - ), - "classes": ("collapse",), - }, - ) - ) - return fieldsets diff --git a/sde_collections/migrations/0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py similarity index 95% rename from sde_collections/migrations/0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py rename to sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py index c635c833..16cf4219 100644 --- a/sde_collections/migrations/0059_candidateurl_is_tdamm_candidateurl_tdamm_tag_manual_and_more.py +++ b/sde_collections/migrations/0059_candidateurl_tdamm_tag_manual_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.9 on 2024-11-13 08:01 +# Generated by Django 4.2.9 on 2024-11-20 06:39 import django.contrib.postgres.fields from django.db import migrations, models @@ -11,13 +11,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name="candidateurl", - name="is_tdamm", - field=models.BooleanField( - default=False, help_text="Enable TDAMM tagging for this URL", verbose_name="Is TDAMM?" - ), - ), migrations.AddField( model_name="candidateurl", name="tdamm_tag_manual", diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index 87bfbbaf..f274e839 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -81,11 +81,10 @@ class CandidateURL(models.Model): default=False, help_text="Helps keep track if the Current URL is present in production or not", ) - is_tdamm = models.BooleanField("Is TDAMM?", default=False, help_text="Enable TDAMM tagging for this URL") + # is_tdamm = models.BooleanField("Is TDAMM?", default=False, help_text="Enable TDAMM tagging for this URL") tdamm_tag = PairedFieldDescriptor( field_name="tdamm_tag", field_type=ArrayField(models.CharField(max_length=255, choices=TDAMMTags.choices), blank=True, null=True), - switch="is_tdamm", verbose_name="TDAMM Tags", ) diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py index 06227b31..4aec4233 100644 --- a/sde_collections/serializers.py +++ b/sde_collections/serializers.py @@ -116,14 +116,7 @@ class CandidateURLAPISerializer(serializers.ModelSerializer): class Meta: model = CandidateURL - fields = ("url", "title", "document_type", "hash", "file_extension", "tree_root", "is_tdamm", "tdamm_tag") - - def to_representation(self, instance): - """Remove tdamm_tag field if is_tdamm is False""" - representation = super().to_representation(instance) - if not instance.is_tdamm: - representation.pop("tdamm_tag", None) - return representation + fields = ("url", "title", "document_type", "hash", "file_extension", "tree_root", "tdamm_tag") def get_tdamm_tag(self, obj): tags = obj.tdamm_tag diff --git a/sde_collections/utils/paired_field_descriptor.py b/sde_collections/utils/paired_field_descriptor.py index 8fdb2a2f..afebc35a 100644 --- a/sde_collections/utils/paired_field_descriptor.py +++ b/sde_collections/utils/paired_field_descriptor.py @@ -6,13 +6,12 @@ class PairedFieldDescriptor: - Getting the main field returns manual if present, otherwise ML """ - def __init__(self, field_name, field_type, switch, verbose_name=""): + def __init__(self, field_name, field_type, verbose_name=""): self.field_name = field_name self.manual_field_name = f"{field_name}_manual" self.ml_field_name = f"{field_name}_ml" self.field_type = field_type self.verbose_name = verbose_name or field_name.replace("_", " ").title() - self.switch = switch def contribute_to_class(self, cls, name): """Called by Django when the descriptor is added to the model class.""" @@ -48,14 +47,10 @@ def __get__(self, instance, owner): Get the value of the main field: - Returns manual tags if they exist - Otherwise returns ML tags - - Returns None if switch is False """ if instance is None: return self - if not getattr(instance, self.switch, False): - return None - manual_value = getattr(instance, self.manual_field_name, None) ml_value = getattr(instance, self.ml_field_name, None) @@ -69,16 +64,10 @@ def __set__(self, instance, value): Set only the manual field when setting the field. ML field must be set explicitly. """ - if getattr(instance, self.switch, False): - # Only set the manual field - setattr(instance, self.manual_field_name, value) + + setattr(instance, self.manual_field_name, value) def __delete__(self, instance): """Delete both manual and ML fields""" setattr(instance, self.manual_field_name, None) setattr(instance, self.ml_field_name, None) - - def set_ml(self, instance, value): - """Explicit method to set ML tags""" - if getattr(instance, self.switch, False): - setattr(instance, self.ml_field_name, value) From b65c6931cb280b8e2219aacd9d49b01464c1ff16 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Fri, 15 Nov 2024 16:20:34 -0600 Subject: [PATCH 12/21] add test cases for two column tags functionality --- sde_collections/tests/factories.py | 74 +++++++++++++++ sde_collections/tests/test_tdamm_tags.py | 110 +++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 sde_collections/tests/factories.py create mode 100644 sde_collections/tests/test_tdamm_tags.py diff --git a/sde_collections/tests/factories.py b/sde_collections/tests/factories.py new file mode 100644 index 00000000..cc9b3e61 --- /dev/null +++ b/sde_collections/tests/factories.py @@ -0,0 +1,74 @@ +import factory +from django.contrib.auth import get_user_model +from django.utils import timezone + +from sde_collections.models.collection import Collection +from sde_collections.models.candidate_url import CandidateURL +from sde_collections.models.collection_choice_fields import ( + ConnectorChoices, + Divisions, + DocumentTypes, + UpdateFrequencies, + WorkflowStatusChoices, +) + +User = get_user_model() + + +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = User + + username = factory.Sequence(lambda n: f"user{n}") + email = factory.LazyAttribute(lambda obj: f"{obj.username}@example.com") + + +class CollectionFactory(factory.django.DjangoModelFactory): + class Meta: + model = Collection + + name = factory.Faker("company") + config_folder = factory.Sequence( + lambda n: f"config_folder_{n}" + ) # might need to update this to be calculated based on name? + url = factory.Faker("url") + division = Divisions.ASTROPHYSICS + connector = ConnectorChoices.CRAWLER2 + update_frequency = UpdateFrequencies.WEEKLY + document_type = DocumentTypes.DOCUMENTATION + delete = False + is_multi_division = False + + github_issue_number = factory.Sequence(lambda n: n) + notes = factory.Faker("paragraph") + updated_at = factory.LazyFunction(timezone.now) + new_collection = False + + workflow_status = WorkflowStatusChoices.RESEARCH_IN_PROGRESS + tracker = factory.Maybe("workflow_status") + + # ForeignKey to User for `curated_by` + curated_by = factory.SubFactory(UserFactory) + curation_started = factory.LazyFunction(timezone.now) + + +class CandidateURLFactory(factory.django.DjangoModelFactory): + class Meta: + model = CandidateURL + + collection = factory.SubFactory(CollectionFactory) + url = factory.Faker("url") + hash = factory.LazyFunction(lambda: "1") + scraped_title = factory.Faker("sentence") + generated_title = factory.Faker("sentence") + test_title = "" + production_title = "" + level = 0 + visited = False + document_type = DocumentTypes.DOCUMENTATION + division = Divisions.ASTROPHYSICS + inferenced_by = "" + is_pdf = False + present_on_test = False + present_on_prod = False + is_tdamm = False diff --git a/sde_collections/tests/test_tdamm_tags.py b/sde_collections/tests/test_tdamm_tags.py new file mode 100644 index 00000000..6db17e50 --- /dev/null +++ b/sde_collections/tests/test_tdamm_tags.py @@ -0,0 +1,110 @@ +# docker-compose -f local.yml run --rm django pytest -s sde_collections/tests/test_tdamm_tags.py + +import pytest +from sde_collections.tests.factories import CandidateURLFactory + + +@pytest.mark.django_db +class TestTDAMMFields: + def test_tdamm_switch_behavior(self): + """Test that TDAMM fields only work when switch is enabled""" + # Create URL with TDAMM disabled + url = CandidateURLFactory(is_tdamm=False) + url.tdamm_tag = ["MMA_M_EM"] + assert url.tdamm_tag is None + assert url.tdamm_tag_manual is None + assert url.tdamm_tag_ml is None + + # Enable TDAMM + url.is_tdamm = True + url.save() + url.tdamm_tag = ["MMA_M_EM"] + assert url.tdamm_tag == ["MMA_M_EM"] + assert url.tdamm_tag_manual == ["MMA_M_EM"] + + def test_manual_and_ml_field_behavior(self): + """Test the relationship between manual and ML fields""" + url = CandidateURLFactory(is_tdamm=True) + + # Setting tdamm_tag affects only manual field + url.tdamm_tag = ["MMA_M_EM", "MMA_M_G"] + assert url.tdamm_tag_manual == ["MMA_M_EM", "MMA_M_G"] + assert url.tdamm_tag_ml is None + + # ML field must be set explicitly + url.tdamm_tag_ml = ["MMA_M_N"] + assert url.tdamm_tag_ml == ["MMA_M_N"] + assert url.tdamm_tag_manual == ["MMA_M_EM", "MMA_M_G"] + + def test_field_priority(self): + """Test that manual field takes priority over ML field""" + url = CandidateURLFactory(is_tdamm=True) + + # Set ML tags first + url.tdamm_tag_ml = ["MMA_M_EM"] + assert url.tdamm_tag == ["MMA_M_EM"] + + # Set manual tags - should take priority + url.tdamm_tag = ["MMA_M_G"] + assert url.tdamm_tag == ["MMA_M_G"] + + # Clear manual tags - should fall back to ML tags + url.tdamm_tag_manual = None + assert url.tdamm_tag == ["MMA_M_EM"] + + def test_empty_array_behavior(self): + """Test handling of empty arrays vs None""" + url = CandidateURLFactory(is_tdamm=True) + + # Set ML tags + url.tdamm_tag_ml = ["MMA_M_EM"] + assert url.tdamm_tag == ["MMA_M_EM"] + + # Empty manual array should not override ML tags + url.tdamm_tag = [] + assert url.tdamm_tag == ["MMA_M_EM"] + + # None manual value should not override ML tags + url.tdamm_tag = None + assert url.tdamm_tag == ["MMA_M_EM"] + + def test_field_deletion(self): + """Test deletion of fields""" + url = CandidateURLFactory(is_tdamm=True) + + # Set both manual and ML tags + url.tdamm_tag = ["MMA_M_EM"] + url.tdamm_tag_ml = ["MMA_M_G"] + + # Delete tdamm_tag + del url.tdamm_tag + assert url.tdamm_tag_manual is None + assert url.tdamm_tag_ml is None + + def test_multiple_tags(self): + """Test handling of multiple tags""" + url = CandidateURLFactory(is_tdamm=True) + + # Test multiple manual tags + manual_tags = ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + url.tdamm_tag = manual_tags + assert url.tdamm_tag_manual == manual_tags + + # Test multiple ML tags + ml_tags = ["MMA_O_BH", "MMA_O_N"] + url.tdamm_tag_ml = ml_tags + assert url.tdamm_tag_ml == ml_tags + + def test_persistence(self): + """Test that values persist after save""" + url = CandidateURLFactory(is_tdamm=True) + + # Set values + url.tdamm_tag = ["MMA_M_EM"] + url.tdamm_tag_ml = ["MMA_M_G"] + url.save() + + # Refresh from database + url.refresh_from_db() + assert url.tdamm_tag_manual == ["MMA_M_EM"] + assert url.tdamm_tag_ml == ["MMA_M_G"] From 070750085dc8860cf0df661a20bc80644c5f3ceb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 22:28:46 +0000 Subject: [PATCH 13/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- sde_collections/tests/factories.py | 2 +- sde_collections/tests/test_tdamm_tags.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sde_collections/tests/factories.py b/sde_collections/tests/factories.py index cc9b3e61..47427993 100644 --- a/sde_collections/tests/factories.py +++ b/sde_collections/tests/factories.py @@ -2,8 +2,8 @@ from django.contrib.auth import get_user_model from django.utils import timezone -from sde_collections.models.collection import Collection from sde_collections.models.candidate_url import CandidateURL +from sde_collections.models.collection import Collection from sde_collections.models.collection_choice_fields import ( ConnectorChoices, Divisions, diff --git a/sde_collections/tests/test_tdamm_tags.py b/sde_collections/tests/test_tdamm_tags.py index 6db17e50..72ebfef2 100644 --- a/sde_collections/tests/test_tdamm_tags.py +++ b/sde_collections/tests/test_tdamm_tags.py @@ -1,6 +1,7 @@ # docker-compose -f local.yml run --rm django pytest -s sde_collections/tests/test_tdamm_tags.py import pytest + from sde_collections.tests.factories import CandidateURLFactory From e45eeeba8007a7f0bb3915b79cab721d72683ef2 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 20 Nov 2024 19:49:46 -0600 Subject: [PATCH 14/21] refactor code for DeltaUrl model --- sde_collections/admin.py | 78 ++-- .../migrations/0066_merge_20241120_0158.py | 13 + ..._candidateurl_tdamm_tag_manual_and_more.py | 21 ++ ...manual_curatedurl_tdamm_tag_ml_and_more.py | 344 ++++++++++++++++++ ..._candidateurl_tdamm_tag_manual_and_more.py | 124 +++++++ sde_collections/models/delta_url.py | 10 +- sde_collections/tests/test_tdamm_tags.py | 125 +++++-- 7 files changed, 663 insertions(+), 52 deletions(-) create mode 100644 sde_collections/migrations/0066_merge_20241120_0158.py create mode 100644 sde_collections/migrations/0067_remove_candidateurl_tdamm_tag_manual_and_more.py create mode 100644 sde_collections/migrations/0068_curatedurl_tdamm_tag_manual_curatedurl_tdamm_tag_ml_and_more.py create mode 100644 sde_collections/migrations/0069_candidateurl_tdamm_tag_manual_and_more.py diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 7706e9ec..28c96396 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -285,8 +285,9 @@ def exclude_and_delete_children(modeladmin, request, queryset): candidate_url.get_children().delete() -class CandidateURLForm(forms.ModelForm): - # Define the fields as MultipleChoiceFields with checkboxes +class TDAMMFormMixin(forms.ModelForm): + """Mixin for forms that need TDAMM tag fields""" + tdamm_tag_manual = forms.MultipleChoiceField( choices=TDAMMTags.choices, required=False, @@ -301,40 +302,28 @@ class CandidateURLForm(forms.ModelForm): widget=forms.CheckboxSelectMultiple, ) - class Meta: - model = CandidateURL - fields = "__all__" +class TDAMMAdminMixin: + """Mixin for admin classes that handle TDAMM tags""" -class CandidateURLAdmin(admin.ModelAdmin): - """Admin view for CandidateURL""" - - form = CandidateURLForm - list_display = ["url", "collection", "tdamm_tag_manual", "tdamm_tag_ml"] + list_display = ("url", "scraped_title", "generated_title", "collection") list_filter = ["collection"] search_fields = ("url", "collection__name") def get_fieldsets(self, request, obj=None): fieldsets = [ ( - "Essential Information", + "Overall Information", { "fields": ( "collection", "url", - "hash", "scraped_title", + "scraped_text", "generated_title", - "test_title", - "production_title", - "level", "visited", "document_type", "division", - "inferenced_by", - "is_pdf", - "present_on_test", - "present_on_prod", ) }, ), @@ -349,10 +338,39 @@ def get_fieldsets(self, request, obj=None): }, ), ] - return fieldsets +class CandidateURLForm(TDAMMFormMixin): + class Meta: + model = CandidateURL + fields = "__all__" + + +class DumpURLForm(TDAMMFormMixin, forms.ModelForm): + class Meta: + model = DumpUrl + fields = "__all__" + + +class DeltaURLForm(TDAMMFormMixin, forms.ModelForm): + class Meta: + model = DeltaUrl + fields = "__all__" + + +class CuratedURLForm(TDAMMFormMixin, forms.ModelForm): + class Meta: + model = CuratedUrl + fields = "__all__" + + +class CandidateURLAdmin(TDAMMAdminMixin, admin.ModelAdmin): + """Admin view for CandidateURL""" + + form = CandidateURLForm + + class TitlePatternAdmin(admin.ModelAdmin): """Admin View for TitlePattern""" @@ -408,25 +426,27 @@ class DeltaDivisionPatternAdmin(admin.ModelAdmin): search_fields = ("match_pattern", "division") -class DumpUrlAdmin(admin.ModelAdmin): +class DumpUrlAdmin(TDAMMAdminMixin, admin.ModelAdmin): """Admin View for DumpUrl""" - list_display = ("url", "scraped_title", "collection") - list_filter = ("collection",) + form = DumpURLForm -class DeltaUrlAdmin(admin.ModelAdmin): +class DeltaUrlAdmin(TDAMMAdminMixin, admin.ModelAdmin): """Admin View for DeltaUrl""" - list_display = ("url", "scraped_title", "generated_title", "collection") - list_filter = ("collection",) + form = DeltaURLForm + def get_fieldsets(self, request, obj=None): + fieldsets = super().get_fieldsets(request, obj) + fieldsets[0][1]["fields"] += ("to_delete",) + return fieldsets -class CuratedUrlAdmin(admin.ModelAdmin): + +class CuratedUrlAdmin(TDAMMAdminMixin, admin.ModelAdmin): """Admin View for CuratedUrl""" - list_display = ("url", "scraped_title", "generated_title", "collection") - list_filter = ("collection",) + form = CuratedURLForm admin.site.register(WorkflowHistory, WorkflowHistoryAdmin) diff --git a/sde_collections/migrations/0066_merge_20241120_0158.py b/sde_collections/migrations/0066_merge_20241120_0158.py new file mode 100644 index 00000000..ccd58b61 --- /dev/null +++ b/sde_collections/migrations/0066_merge_20241120_0158.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.9 on 2024-11-20 07:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0059_candidateurl_tdamm_tag_manual_and_more"), + ("sde_collections", "0065_rename_delete_deltaurl_to_delete_and_more"), + ] + + operations = [] diff --git a/sde_collections/migrations/0067_remove_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0067_remove_candidateurl_tdamm_tag_manual_and_more.py new file mode 100644 index 00000000..2391e18c --- /dev/null +++ b/sde_collections/migrations/0067_remove_candidateurl_tdamm_tag_manual_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.9 on 2024-11-20 16:12 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0066_merge_20241120_0158"), + ] + + operations = [ + migrations.RemoveField( + model_name="candidateurl", + name="tdamm_tag_manual", + ), + migrations.RemoveField( + model_name="candidateurl", + name="tdamm_tag_ml", + ), + ] diff --git a/sde_collections/migrations/0068_curatedurl_tdamm_tag_manual_curatedurl_tdamm_tag_ml_and_more.py b/sde_collections/migrations/0068_curatedurl_tdamm_tag_manual_curatedurl_tdamm_tag_ml_and_more.py new file mode 100644 index 00000000..adae0e2b --- /dev/null +++ b/sde_collections/migrations/0068_curatedurl_tdamm_tag_manual_curatedurl_tdamm_tag_ml_and_more.py @@ -0,0 +1,344 @@ +# Generated by Django 4.2.9 on 2024-11-20 16:23 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0067_remove_candidateurl_tdamm_tag_manual_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="curatedurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="curatedurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="deltaurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="deltaurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="dumpurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="dumpurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + ] diff --git a/sde_collections/migrations/0069_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0069_candidateurl_tdamm_tag_manual_and_more.py new file mode 100644 index 00000000..d45e8108 --- /dev/null +++ b/sde_collections/migrations/0069_candidateurl_tdamm_tag_manual_and_more.py @@ -0,0 +1,124 @@ +# Generated by Django 4.2.9 on 2024-11-20 23:42 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0068_curatedurl_tdamm_tag_manual_curatedurl_tdamm_tag_ml_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="candidateurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AddField( + model_name="candidateurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + ] diff --git a/sde_collections/models/delta_url.py b/sde_collections/models/delta_url.py index 3f1212e0..3f5effc6 100644 --- a/sde_collections/models/delta_url.py +++ b/sde_collections/models/delta_url.py @@ -3,8 +3,10 @@ from django.db import models -from .collection_choice_fields import Divisions, DocumentTypes +from .collection_choice_fields import Divisions, DocumentTypes, TDAMMTags from .delta_patterns import DeltaExcludePattern, DeltaTitlePattern +from ..utils.paired_field_descriptor import PairedFieldDescriptor +from django.contrib.postgres.fields import ArrayField class DeltaUrlQuerySet(models.QuerySet): @@ -62,6 +64,12 @@ class BaseUrl(models.Model): document_type = models.IntegerField(choices=DocumentTypes.choices, null=True) division = models.IntegerField(choices=Divisions.choices, null=True) + tdamm_tag = PairedFieldDescriptor( + field_name="tdamm_tag", + field_type=ArrayField(models.CharField(max_length=255, choices=TDAMMTags.choices), blank=True, null=True), + verbose_name="TDAMM Tags", + ) + class Meta: abstract = True ordering = ["url"] diff --git a/sde_collections/tests/test_tdamm_tags.py b/sde_collections/tests/test_tdamm_tags.py index 72ebfef2..8811bd68 100644 --- a/sde_collections/tests/test_tdamm_tags.py +++ b/sde_collections/tests/test_tdamm_tags.py @@ -2,30 +2,17 @@ import pytest -from sde_collections.tests.factories import CandidateURLFactory +from ..models.delta_url import CuratedUrl, DeltaUrl, DumpUrl +from sde_collections.tests.factories import CollectionFactory, DeltaUrlFactory, DumpUrlFactory @pytest.mark.django_db class TestTDAMMFields: - def test_tdamm_switch_behavior(self): - """Test that TDAMM fields only work when switch is enabled""" - # Create URL with TDAMM disabled - url = CandidateURLFactory(is_tdamm=False) - url.tdamm_tag = ["MMA_M_EM"] - assert url.tdamm_tag is None - assert url.tdamm_tag_manual is None - assert url.tdamm_tag_ml is None - - # Enable TDAMM - url.is_tdamm = True - url.save() - url.tdamm_tag = ["MMA_M_EM"] - assert url.tdamm_tag == ["MMA_M_EM"] - assert url.tdamm_tag_manual == ["MMA_M_EM"] + """Test core TDAMM tags functionality with DeltaUrl""" def test_manual_and_ml_field_behavior(self): """Test the relationship between manual and ML fields""" - url = CandidateURLFactory(is_tdamm=True) + url = DeltaUrlFactory() # Setting tdamm_tag affects only manual field url.tdamm_tag = ["MMA_M_EM", "MMA_M_G"] @@ -39,7 +26,7 @@ def test_manual_and_ml_field_behavior(self): def test_field_priority(self): """Test that manual field takes priority over ML field""" - url = CandidateURLFactory(is_tdamm=True) + url = DeltaUrlFactory() # Set ML tags first url.tdamm_tag_ml = ["MMA_M_EM"] @@ -55,7 +42,7 @@ def test_field_priority(self): def test_empty_array_behavior(self): """Test handling of empty arrays vs None""" - url = CandidateURLFactory(is_tdamm=True) + url = DeltaUrlFactory() # Set ML tags url.tdamm_tag_ml = ["MMA_M_EM"] @@ -71,7 +58,7 @@ def test_empty_array_behavior(self): def test_field_deletion(self): """Test deletion of fields""" - url = CandidateURLFactory(is_tdamm=True) + url = DeltaUrlFactory() # Set both manual and ML tags url.tdamm_tag = ["MMA_M_EM"] @@ -84,7 +71,7 @@ def test_field_deletion(self): def test_multiple_tags(self): """Test handling of multiple tags""" - url = CandidateURLFactory(is_tdamm=True) + url = DeltaUrlFactory() # Test multiple manual tags manual_tags = ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] @@ -98,7 +85,7 @@ def test_multiple_tags(self): def test_persistence(self): """Test that values persist after save""" - url = CandidateURLFactory(is_tdamm=True) + url = DeltaUrlFactory() # Set values url.tdamm_tag = ["MMA_M_EM"] @@ -109,3 +96,97 @@ def test_persistence(self): url.refresh_from_db() assert url.tdamm_tag_manual == ["MMA_M_EM"] assert url.tdamm_tag_ml == ["MMA_M_G"] + + +@pytest.mark.django_db +class TestTDAMMTagMigration: + """Test TDAMM tag behavior during the migration process""" + + @pytest.fixture + def collection(self): + return CollectionFactory() + + def test_tdamm_tags_preserved_in_migration(self, collection): + """Test that TDAMM tags are preserved when promoting from Dump to Delta""" + dump_url = DumpUrlFactory(collection=collection, url="https://example.com") + dump_url.tdamm_tag = ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + dump_url.tdamm_tag_ml = ["MMA_O_BH", "MMA_O_N"] + dump_url.save() + + # Migrate to delta + collection.migrate_dump_to_delta() + + # Verify tags in the migrated DeltaUrl + delta_url = DeltaUrl.objects.get(url="https://example.com") + assert delta_url.tdamm_tag == ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + assert delta_url.tdamm_tag_manual == ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + assert delta_url.tdamm_tag_ml == ["MMA_O_BH", "MMA_O_N"] + + def test_tdamm_tags_updated_in_migration(self, collection): + """Test that TDAMM tags are updated during re-migration""" + # Initial migration + dump_url = DumpUrlFactory(collection=collection, url="https://example.com") + dump_url.tdamm_tag = ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + dump_url.tdamm_tag_ml = ["MMA_O_BH", "MMA_O_N"] + dump_url.save() + + # Migrate to delta + collection.migrate_dump_to_delta() + + # Create new DumpUrl with updated tags + updated_dump_url = DumpUrlFactory(collection=collection, url="https://example.com") + updated_dump_url.tdamm_tag = ["MMA_M_G"] + updated_dump_url.save() + collection.migrate_dump_to_delta() + + # Verify tags were updated + delta_url = DeltaUrl.objects.get(url="https://example.com") + assert delta_url.tdamm_tag == ["MMA_M_G"] + assert delta_url.tdamm_tag_manual == ["MMA_M_G"] + + +@pytest.mark.django_db +class TestTDAMMTagPromotion: + """Test TDAMM tag behavior during the promotion process""" + + @pytest.fixture + def collection(self): + return CollectionFactory() + + def test_tdamm_tags_preserved_in_promotion(self, collection): + """Test that TDAMM tags are preserved when promoting from Delta to Curated""" + delta_url = DeltaUrlFactory(collection=collection, url="https://example.com") + delta_url.tdamm_tag = ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + delta_url.tdamm_tag_ml = ["MMA_O_BH", "MMA_O_N"] + delta_url.save() + + # Promote to curated + collection.promote_to_curated() + + # Verify tags in the promoted CuratedUrl + curated_url = CuratedUrl.objects.get(url="https://example.com") + assert curated_url.tdamm_tag == ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + assert curated_url.tdamm_tag_manual == ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + assert curated_url.tdamm_tag_ml == ["MMA_O_BH", "MMA_O_N"] + + def test_tdamm_tags_updated_in_promotion(self, collection): + """Test that TDAMM tags are updated during re-promotion""" + # Initial promotion + delta_url = DeltaUrlFactory(collection=collection, url="https://example.com") + delta_url.tdamm_tag = ["MMA_M_EM", "MMA_M_G", "MMA_M_N"] + delta_url.tdamm_tag_ml = ["MMA_O_BH", "MMA_O_N"] + delta_url.save() + + # Promote to curated + collection.promote_to_curated() + + # Create new DeltaUrl with updated tags + updated_delta_url = DeltaUrlFactory(collection=collection, url="https://example.com") + updated_delta_url.tdamm_tag = ["MMA_M_G"] + updated_delta_url.save() + collection.promote_to_curated() + + # Verify tags were updated + curated_url = CuratedUrl.objects.get(url="https://example.com") + assert curated_url.tdamm_tag == ["MMA_M_G"] + assert curated_url.tdamm_tag_manual == ["MMA_M_G"] From ff1484ba4f6f60389b563458a58ec7a97df6e774 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Wed, 20 Nov 2024 22:17:29 -0600 Subject: [PATCH 15/21] add tdamm_tag field to new serializers --- sde_collections/admin.py | 1 - sde_collections/serializers.py | 23 +++++++++++++++++++++++ sde_collections/tests/test_tdamm_tags.py | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 28c96396..23a9b12c 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -13,7 +13,6 @@ from .models.collection import Collection, WorkflowHistory from .models.delta_url import CuratedUrl, DeltaResolvedTitle, DeltaUrl, DumpUrl from .models.pattern import DivisionPattern, IncludePattern, TitlePattern -from .tasks import import_candidate_urls_from_api from .models.collection_choice_fields import TDAMMTags from .tasks import fetch_and_replace_full_text, import_candidate_urls_from_api diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py index f77c28ab..e895cb28 100644 --- a/sde_collections/serializers.py +++ b/sde_collections/serializers.py @@ -62,6 +62,11 @@ class DeltaURLSerializer(serializers.ModelSerializer): generated_title_id = serializers.SerializerMethodField(read_only=True) match_pattern_type = serializers.SerializerMethodField(read_only=True) delta_urls_count = serializers.SerializerMethodField(read_only=True) + tdamm_tag = serializers.SerializerMethodField() + + def get_tdamm_tag(self, obj): + tags = obj.tdamm_tag + return tags if tags is not None else [] def get_delta_urls_count(self, obj): titlepattern = obj.deltatitlepattern_delta_urls.last() @@ -91,6 +96,7 @@ class Meta: "division", "division_display", "visited", + "tdamm_tag", ) @@ -102,6 +108,11 @@ class CuratedURLSerializer(serializers.ModelSerializer): generated_title_id = serializers.SerializerMethodField(read_only=True) match_pattern_type = serializers.SerializerMethodField(read_only=True) curated_urls_count = serializers.SerializerMethodField(read_only=True) + tdamm_tag = serializers.SerializerMethodField() + + def get_tdamm_tag(self, obj): + tags = obj.tdamm_tag + return tags if tags is not None else [] def get_curated_urls_count(self, obj): titlepattern = obj.deltatitlepattern_curated_urls.last() @@ -131,6 +142,7 @@ class Meta: "division", "division_display", "visited", + "tdamm_tag", ) @@ -148,6 +160,7 @@ class DeltaURLAPISerializer(serializers.ModelSerializer): title = serializers.SerializerMethodField() file_extension = serializers.SerializerMethodField() tree_root = serializers.SerializerMethodField() + tdamm_tag = serializers.SerializerMethodField() class Meta: model = DeltaUrl @@ -157,8 +170,13 @@ class Meta: "document_type", "file_extension", "tree_root", + "tdamm_tag", ) + def get_tdamm_tag(self, obj): + tags = obj.tdamm_tag + return tags if tags is not None else [] + def get_document_type(self, obj): if obj.document_type is not None: return obj.get_document_type_display() @@ -198,8 +216,13 @@ class Meta: "document_type", "file_extension", "tree_root", + "tdamm_tag", ) + def get_tdamm_tag(self, obj): + tags = obj.tdamm_tag + return tags if tags is not None else [] + def get_document_type(self, obj): if obj.document_type is not None: return obj.get_document_type_display() diff --git a/sde_collections/tests/test_tdamm_tags.py b/sde_collections/tests/test_tdamm_tags.py index 8811bd68..81b12f11 100644 --- a/sde_collections/tests/test_tdamm_tags.py +++ b/sde_collections/tests/test_tdamm_tags.py @@ -2,7 +2,7 @@ import pytest -from ..models.delta_url import CuratedUrl, DeltaUrl, DumpUrl +from ..models.delta_url import CuratedUrl, DeltaUrl from sde_collections.tests.factories import CollectionFactory, DeltaUrlFactory, DumpUrlFactory From 70ccb2ca234306c2800378c66ae12a106652ae80 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:36:53 +0000 Subject: [PATCH 16/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- sde_collections/admin.py | 2 +- sde_collections/models/delta_url.py | 4 ++-- sde_collections/tests/test_tdamm_tags.py | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sde_collections/admin.py b/sde_collections/admin.py index 23a9b12c..92ca41c3 100644 --- a/sde_collections/admin.py +++ b/sde_collections/admin.py @@ -11,9 +11,9 @@ from .models.candidate_url import CandidateURL, ResolvedTitle from .models.collection import Collection, WorkflowHistory +from .models.collection_choice_fields import TDAMMTags from .models.delta_url import CuratedUrl, DeltaResolvedTitle, DeltaUrl, DumpUrl from .models.pattern import DivisionPattern, IncludePattern, TitlePattern -from .models.collection_choice_fields import TDAMMTags from .tasks import fetch_and_replace_full_text, import_candidate_urls_from_api diff --git a/sde_collections/models/delta_url.py b/sde_collections/models/delta_url.py index 3f5effc6..e8d00483 100644 --- a/sde_collections/models/delta_url.py +++ b/sde_collections/models/delta_url.py @@ -1,12 +1,12 @@ import os from urllib.parse import urlparse +from django.contrib.postgres.fields import ArrayField from django.db import models +from ..utils.paired_field_descriptor import PairedFieldDescriptor from .collection_choice_fields import Divisions, DocumentTypes, TDAMMTags from .delta_patterns import DeltaExcludePattern, DeltaTitlePattern -from ..utils.paired_field_descriptor import PairedFieldDescriptor -from django.contrib.postgres.fields import ArrayField class DeltaUrlQuerySet(models.QuerySet): diff --git a/sde_collections/tests/test_tdamm_tags.py b/sde_collections/tests/test_tdamm_tags.py index 81b12f11..f520b63b 100644 --- a/sde_collections/tests/test_tdamm_tags.py +++ b/sde_collections/tests/test_tdamm_tags.py @@ -2,8 +2,13 @@ import pytest +from sde_collections.tests.factories import ( + CollectionFactory, + DeltaUrlFactory, + DumpUrlFactory, +) + from ..models.delta_url import CuratedUrl, DeltaUrl -from sde_collections.tests.factories import CollectionFactory, DeltaUrlFactory, DumpUrlFactory @pytest.mark.django_db From 1b70b52f9fa6d2580c88f7ce762ec9399d008578 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Thu, 5 Dec 2024 14:33:00 -0600 Subject: [PATCH 17/21] Fix incorrect import --- sde_collections/models/delta_url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sde_collections/models/delta_url.py b/sde_collections/models/delta_url.py index c7f1f4c8..88df502b 100644 --- a/sde_collections/models/delta_url.py +++ b/sde_collections/models/delta_url.py @@ -6,7 +6,7 @@ from ..utils.paired_field_descriptor import PairedFieldDescriptor from .collection_choice_fields import Divisions, DocumentTypes, TDAMMTags -from .delta_patterns import DeltaExcludePattern, DeltaTitlePattern +from .delta_patterns import DeltaExcludePattern, DeltaIncludePattern class DeltaUrlQuerySet(models.QuerySet): From e0ac9e429a7c9a04774cd6767077fc6df447e8c9 Mon Sep 17 00:00:00 2001 From: Kiran Dawadi Date: Thu, 5 Dec 2024 14:48:53 -0600 Subject: [PATCH 18/21] merge migrations conflict --- .../migrations/0070_merge_20241205_1437.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 sde_collections/migrations/0070_merge_20241205_1437.py diff --git a/sde_collections/migrations/0070_merge_20241205_1437.py b/sde_collections/migrations/0070_merge_20241205_1437.py new file mode 100644 index 00000000..8d904006 --- /dev/null +++ b/sde_collections/migrations/0070_merge_20241205_1437.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.9 on 2024-12-05 20:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0068_alter_deltadivisionpattern_collection_and_more"), + ("sde_collections", "0069_candidateurl_tdamm_tag_manual_and_more"), + ] + + operations = [] From 8f87aefe94a99802598705a7f5eb0bb1793b686c Mon Sep 17 00:00:00 2001 From: Carson Davis Date: Thu, 5 Dec 2024 17:16:16 -0600 Subject: [PATCH 19/21] Update sde_collections/models/candidate_url.py --- sde_collections/models/candidate_url.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sde_collections/models/candidate_url.py b/sde_collections/models/candidate_url.py index 1ba80c93..2fe3dd82 100644 --- a/sde_collections/models/candidate_url.py +++ b/sde_collections/models/candidate_url.py @@ -101,7 +101,6 @@ class Meta: verbose_name = "Candidate URL" verbose_name_plural = "Candidate URLs" ordering = ["url"] - db_table = "sde_collections_candidateurl" @property def fileext(self) -> str: From de20d47a5d5020f763392cfebf7f74555e4ab186 Mon Sep 17 00:00:00 2001 From: Carson Davis Date: Thu, 5 Dec 2024 17:37:35 -0600 Subject: [PATCH 20/21] add a TDAMM field Not TDAMM --- ..._candidateurl_tdamm_tag_manual_and_more.py | 466 ++++++++++++++++++ .../models/collection_choice_fields.py | 1 + 2 files changed, 467 insertions(+) create mode 100644 sde_collections/migrations/0071_alter_candidateurl_tdamm_tag_manual_and_more.py diff --git a/sde_collections/migrations/0071_alter_candidateurl_tdamm_tag_manual_and_more.py b/sde_collections/migrations/0071_alter_candidateurl_tdamm_tag_manual_and_more.py new file mode 100644 index 00000000..12b7ae8e --- /dev/null +++ b/sde_collections/migrations/0071_alter_candidateurl_tdamm_tag_manual_and_more.py @@ -0,0 +1,466 @@ +# Generated by Django 4.2.9 on 2024-12-05 23:36 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0070_merge_20241205_1437"), + ] + + operations = [ + migrations.AlterField( + model_name="candidateurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="candidateurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="curatedurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="curatedurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="deltaurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="deltaurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="dumpurl", + name="tdamm_tag_manual", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_manual", + null=True, + size=None, + ), + ), + migrations.AlterField( + model_name="dumpurl", + name="tdamm_tag_ml", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("Not TDAMM", "Not TDAMM"), + ("MMA_M_EM", "Messenger - EM Radiation"), + ("MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays"), + ("MMA_M_EM_X", "Messenger - EM Radiation - X-rays"), + ("MMA_M_EM_U", "Messenger - EM Radiation - Ultraviolet"), + ("MMA_M_EM_O", "Messenger - EM Radiation - Optical"), + ("MMA_M_EM_I", "Messenger - EM Radiation - Infrared"), + ("MMA_M_EM_M", "Messenger - EM Radiation - Microwave"), + ("MMA_M_EM_R", "Messenger - EM Radiation - Radio"), + ("MMA_M_G", "Messenger - Gravitational Waves"), + ("MMA_M_G_CBI", "Messenger - Gravitational Waves - Compact Binary Inspiral"), + ("MMA_M_G_S", "Messenger - Gravitational Waves - Stochastic"), + ("MMA_M_G_CON", "Messenger - Gravitational Waves - Continuous"), + ("MMA_M_G_B", "Messenger - Gravitational Waves - Burst"), + ("MMA_M_C", "Messenger - Cosmic Rays"), + ("MMA_M_N", "Messenger - Neutrinos"), + ("MMA_O_BI", "Objects - Binaries"), + ("MMA_O_BI_BBH", "Objects - Binaries - Binary Black Holes"), + ("MMA_O_BI_BNS", "Objects - Binaries - Binary Neutron Stars"), + ("MMA_O_BI_C", "Objects - Binaries - Cataclysmic Variables"), + ("MMA_O_BI_N", "Objects - Binaries - Neutron Star-Black Hole"), + ("MMA_O_BI_B", "Objects - Binaries - Binary Pulsars"), + ("MMA_O_BI_W", "Objects - Binaries - White Dwarf Binaries"), + ("MMA_O_BH", "Objects - Black Holes"), + ("MMA_O_BH_AGN", "Objects - Black Holes - Active Galactic Nuclei"), + ("MMA_O_BH_IM", "Objects - Black Holes - Intermediate mass"), + ("MMA_O_BH_STM", "Objects - Black Holes - Stellar mass"), + ("MMA_O_BH_SUM", "Objects - Black Holes - Supermassive"), + ("MMA_O_E", "Objects - Exoplanets"), + ("MMA_O_N", "Objects - Neutron Stars"), + ("MMA_O_N_M", "Objects - Neutron Stars - Magnetars"), + ("MMA_O_N_P", "Objects - Neutron Stars - Pulsars"), + ("MMA_O_N_PWN", "Objects - Neutron Stars - Pulsar Wind Nebula"), + ("MMA_O_S", "Objects - Supernova Remnants"), + ("MMA_S_F", "Signals - Fast Radio Bursts"), + ("MMA_S_G", "Signals - Gamma-ray Bursts"), + ("MMA_S_K", "Signals - Kilonovae"), + ("MMA_S_N", "Signals - Novae"), + ("MMA_S_P", "Signals - Pevatrons"), + ("MMA_S_ST", "Signals - Stellar flares"), + ("MMA_S_SU", "Signals - Supernovae"), + ], + max_length=255, + ), + blank=True, + db_column="tdamm_tag_ml", + null=True, + size=None, + ), + ), + migrations.AlterModelTable( + name="candidateurl", + table=None, + ), + ] diff --git a/sde_collections/models/collection_choice_fields.py b/sde_collections/models/collection_choice_fields.py index 60567abd..14793bef 100644 --- a/sde_collections/models/collection_choice_fields.py +++ b/sde_collections/models/collection_choice_fields.py @@ -102,6 +102,7 @@ class WorkflowStatusChoices(models.IntegerChoices): class TDAMMTags(models.TextChoices): """TDAMM (Tagged Data for Multi-Messenger Astronomy) tag choices.""" + NOT_TDAMM = "Not TDAMM", "Not TDAMM" MMA_M_EM = "MMA_M_EM", "Messenger - EM Radiation" MMA_M_EM_G = "MMA_M_EM_G", "Messenger - EM Radiation - Gamma rays" MMA_M_EM_X = "MMA_M_EM_X", "Messenger - EM Radiation - X-rays" From 547401c4c11260d3f8dd23a917f1d702a5ce92e9 Mon Sep 17 00:00:00 2001 From: Carson Davis Date: Thu, 5 Dec 2024 17:42:50 -0600 Subject: [PATCH 21/21] add tdamm tags to the api serializer tests --- sde_collections/tests/test_apis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sde_collections/tests/test_apis.py b/sde_collections/tests/test_apis.py index 0df84834..1c842e8a 100644 --- a/sde_collections/tests/test_apis.py +++ b/sde_collections/tests/test_apis.py @@ -58,7 +58,7 @@ def test_delta_url_api_serializer_fields(self, client): assert response.status_code == status.HTTP_200_OK data = response.json()["results"][0] - expected_fields = {"url", "title", "document_type", "file_extension", "tree_root"} + expected_fields = {"url", "title", "document_type", "file_extension", "tree_root", "tdamm_tag"} assert set(data.keys()) == expected_fields def test_delta_url_api_pagination(self, client): @@ -122,7 +122,7 @@ def test_curated_url_api_serializer_fields(self, client): assert response.status_code == status.HTTP_200_OK data = response.json()["results"][0] - expected_fields = {"url", "title", "document_type", "file_extension", "tree_root"} + expected_fields = {"url", "title", "document_type", "file_extension", "tree_root", "tdamm_tag"} assert set(data.keys()) == expected_fields def test_candidate_url_api_alias(self, client): @@ -217,7 +217,7 @@ def test_candidate_url_api_serializer_fields(self, client): assert response.status_code == status.HTTP_200_OK data = response.json()["results"][0] - expected_fields = {"url", "title", "document_type", "file_extension", "tree_root"} + expected_fields = {"url", "title", "document_type", "file_extension", "tree_root", "tdamm_tag"} assert set(data.keys()) == expected_fields def test_candidate_url_api_alias(self, client):