1051 backend model changes on cosmos to hold new incoming urls

sde_collections/admin.py

@@ -6,6 +6,7 @@
 from .models.candidate_url import CandidateURL, ResolvedTitle
 from .models.collection import Collection, WorkflowHistory
 from .models.pattern import DivisionPattern, IncludePattern, TitlePattern
+from .models.url import CuratedUrl, DeltaUrl, DumpUrl, Url
 from .tasks import import_candidate_urls_from_api
 
 
@@ -299,9 +300,41 @@ class DivisionPatternAdmin(admin.ModelAdmin):
     search_fields = ("match_pattern", "division")
 
 
+class CuratedUrlAdmin(admin.ModelAdmin):
+    """Admin View for CuratedUrl"""
+
+    list_display = ("url", "scraped_title", "collection")
+    list_filter = ("collection",)
+
+
+class DeltaUrlAdmin(admin.ModelAdmin):
+    """Admin View for DeltaUrl"""
+
+    list_display = ("url", "scraped_title", "collection")
+    list_filter = ("collection",)
+
+
+class DumpUrlAdmin(admin.ModelAdmin):
+    """Admin View for DumpUrl"""
+
+    list_display = ("url", "scraped_title", "generated_title", "collection")
+    list_filter = ("collection",)
+
+
+class UrlAdmin(admin.ModelAdmin):
+    """Admin View for Url"""
+
+    list_display = ("url", "scraped_title", "collection")
+    list_filter = ("collection",)
+
+
 admin.site.register(WorkflowHistory, WorkflowHistoryAdmin)
 admin.site.register(CandidateURL, CandidateURLAdmin)
 admin.site.register(TitlePattern, TitlePatternAdmin)
 admin.site.register(IncludePattern)
 admin.site.register(ResolvedTitle, ResolvedTitleAdmin)
 admin.site.register(DivisionPattern, DivisionPatternAdmin)
+admin.site.register(DeltaUrl, DeltaUrlAdmin)
+admin.site.register(CuratedUrl, CuratedUrlAdmin)
+admin.site.register(DumpUrl, DumpUrlAdmin)
+admin.site.register(Url, UrlAdmin)

sde_collections/management/commands/migrate_urls.py

@@ -0,0 +1,59 @@
+from django.core.management.base import BaseCommand
+
+from sde_collections.models.candidate_url import CandidateURL
+from sde_collections.models.collection import Collection
+from sde_collections.models.collection_choice_fields import WorkflowStatusChoices
+from sde_collections.models.curated_url import CuratedUrl
+from sde_collections.models.delta_url import DeltaUrl
+
+
+class Command(BaseCommand):
+    help = "Migrate CandidateURLs to CuratedUrl or DeltaUrl based on collection workflow status"
+
+    def handle(self, *args, **kwargs):
+        # Migrate CandidateURLs for collections with CURATED or higher workflow status to CuratedUrl
+        collections_for_curated = Collection.objects.filter(workflow_status__gte=WorkflowStatusChoices.CURATED)
+        self.stdout.write(
+            f"Migrating URLs for {collections_for_curated.count()} collections with CURATED or higher status..."
+        )
+
+        for collection in collections_for_curated:
+            candidate_urls = CandidateURL.objects.filter(collection=collection)
+            for candidate_url in candidate_urls:
+                CuratedUrl.objects.create(
+                    collection=candidate_url.collection,
+                    url=candidate_url.url,
+                    scraped_title=candidate_url.scraped_title,
+                    generated_title=candidate_url.generated_title,
+                    visited=candidate_url.visited,
+                    document_type=candidate_url.document_type,
+                    division=candidate_url.division,
+                )
+            self.stdout.write(
+                f"Migrated {candidate_urls.count()} URLs from collection '{collection.name}' to CuratedUrl."
+            )
+
+        # Migrate CandidateURLs for collections with a status lower than CURATED to DeltaUrl
+        collections_for_delta = Collection.objects.filter(workflow_status__lt=WorkflowStatusChoices.CURATED)
+        self.stdout.write(
+            f"Migrating URLs for {collections_for_delta.count()} collections with status lower than CURATED..."
+        )
+
+        for collection in collections_for_delta:
+            candidate_urls = CandidateURL.objects.filter(collection=collection)
+            for candidate_url in candidate_urls:
+                DeltaUrl.objects.create(
+                    collection=candidate_url.collection,
+                    url=candidate_url.url,
+                    scraped_title=candidate_url.scraped_title,
+                    generated_title=candidate_url.generated_title,
+                    visited=candidate_url.visited,
+                    document_type=candidate_url.document_type,
+                    division=candidate_url.division,
+                    delete=False,
+                )
+            self.stdout.write(
+                f"Migrated {candidate_urls.count()} URLs from collection '{collection.name}' to DeltaUrl."
+            )
+
+        self.stdout.write(self.style.SUCCESS("Migration complete."))

sde_collections/migrations/0059_url_curatedurl_deltaurl_dumpurl.py

@@ -0,0 +1,146 @@
+# Generated by Django 4.2.9 on 2024-10-10 03:01
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0058_candidateurl_division_collection_is_multi_division_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Url",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("url", models.CharField(max_length=4096, verbose_name="URL")),
+                (
+                    "scraped_title",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        help_text="This is the original title scraped by Sinequa",
+                        max_length=1024,
+                        verbose_name="Scraped Title",
+                    ),
+                ),
+                (
+                    "generated_title",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        help_text="This is the title generated based on a Title Pattern",
+                        max_length=1024,
+                        verbose_name="Generated Title",
+                    ),
+                ),
+                ("visited", models.BooleanField(default=False)),
+                (
+                    "document_type",
+                    models.IntegerField(
+                        choices=[
+                            (1, "Images"),
+                            (2, "Data"),
+                            (3, "Documentation"),
+                            (4, "Software and Tools"),
+                            (5, "Missions and Instruments"),
+                        ],
+                        null=True,
+                    ),
+                ),
+                (
+                    "division",
+                    models.IntegerField(
+                        choices=[
+                            (1, "Astrophysics"),
+                            (2, "Biological and Physical Sciences"),
+                            (3, "Earth Science"),
+                            (4, "Heliophysics"),
+                            (5, "Planetary Science"),
+                            (6, "General"),
+                        ],
+                        null=True,
+                    ),
+                ),
+                (
+                    "collection",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="urls",
+                        to="sde_collections.collection",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "URL",
+                "verbose_name_plural": "URLs",
+                "ordering": ["url"],
+            },
+        ),
+        migrations.CreateModel(
+            name="CuratedUrl",
+            fields=[
+                (
+                    "url_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="sde_collections.url",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Curated URL",
+                "verbose_name_plural": "Curated URLs",
+            },
+            bases=("sde_collections.url",),
+        ),
+        migrations.CreateModel(
+            name="DeltaUrl",
+            fields=[
+                (
+                    "url_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="sde_collections.url",
+                    ),
+                ),
+                ("delete", models.BooleanField(default=False)),
+            ],
+            options={
+                "verbose_name": "Delta URL",
+                "verbose_name_plural": "Delta URLs",
+            },
+            bases=("sde_collections.url",),
+        ),
+        migrations.CreateModel(
+            name="DumpUrl",
+            fields=[
+                (
+                    "url_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="sde_collections.url",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Dump URL",
+                "verbose_name_plural": "Dump URLs",
+            },
+            bases=("sde_collections.url",),
+        ),
+    ]

sde_collections/migrations/0060_delete_dumpurl.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.2.9 on 2024-10-14 16:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0059_url_curatedurl_deltaurl_dumpurl"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="DumpUrl",
+        ),
+    ]

sde_collections/migrations/0061_dumpurl.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.2.9 on 2024-10-23 19:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0060_delete_dumpurl"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DumpUrl",
+            fields=[
+                (
+                    "url_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="sde_collections.url",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Dump URL",
+                "verbose_name_plural": "Dump URLs",
+            },
+            bases=("sde_collections.url",),
+        ),
+    ]