Merge pull request #417 from NASA-IMPACT/410-adding-additional-status…

…-on-the-curation-app

Adding additional status options for the curation webapp

.gitignore

@@ -293,3 +293,6 @@ config_generation/config.py
 
 #model's inference files
 Document_Classifier_inference/model.pt
+
+#Database Backup
+backup.json

compose/production/postgres/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:16
+FROM postgres:15
 
 COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance
 RUN chmod +x /usr/local/bin/maintenance/*

production.yml

@@ -10,8 +10,9 @@ services:
     build:
       context: .
       dockerfile: ./compose/production/django/Dockerfile
-
     image: sde_indexing_helper_production_django
+    volumes:
+      - ./backups:/app/backups
     depends_on:
       - postgres
       - redis

sde_collections/admin.py

@@ -151,7 +151,7 @@ class CollectionAdmin(admin.ModelAdmin, ExportCsvMixin, UpdateConfigMixin):
         "division",
         "new_collection",
     )
-    list_filter = ("division", "curation_status", "turned_on")
+    list_filter = ("division", "curation_status", "workflow_status", "turned_on")
     search_fields = ("name", "url")
     actions = [
         "export_as_csv",

sde_collections/management/commands/push_to_github.py

@@ -1,6 +1,7 @@
 from django.core.management.base import BaseCommand
 
 from sde_collections.models.collection import Collection
+from sde_collections.models.collection_choice_fields import WorkflowStatusChoices
 from sde_collections.utils.github_helper import GitHubHandler
 
 
@@ -12,33 +13,34 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument("config_folders", nargs="*", type=str, default=[])
 
-    def handle(self, *args, **options):
-        config_folders = options["config_folders"]
-
-        # curation status 5 is Curated
-        collections = Collection.objects.filter(
-            config_folder__in=config_folders
-        ).filter(curation_status=5)
+    @staticmethod
+    def _get_names(collections):
+        return list(collections.values_list("name", flat=True))
 
-        cant_push = Collection.objects.filter(config_folder__in=config_folders).exclude(
-            curation_status=5
+    def handle(self, *args, **options):
+        selected_collections = Collection.objects.filter(
+            config_folders=options["config_folders"]
+        )
+        curated_collections = selected_collections.filter(
+            workflow_status=WorkflowStatusChoices.CURATED
+        )
+        uncurated_collections = selected_collections.exclude(
+            workflow_status=WorkflowStatusChoices.CURATED
         )
-        cant_push = list(cant_push.values_list("name", flat=True))
 
-        gh = GitHubHandler(collections)
+        gh = GitHubHandler(curated_collections)
         gh.push_to_github()
 
         self.stdout.write(
             self.style.SUCCESS(
-                "Successfully pushed: %s"
-                % list(collections.values_list("name", flat=True))
+                "Successfully pushed: %s" % self._get_names(curated_collections)
             )
         )
 
-        if cant_push:
+        if uncurated_collections:
             self.stdout.write(
                 self.style.ERROR(
-                    "Can't push since status is not Curated (choice_id:5) %s"
-                    % cant_push
+                    "The following collections could not be pushed because the workflow status was not Curated %s"
+                    % self._get_names(uncurated_collections)
                 )
             )

sde_collections/migrations/0032_collection_workflow_status.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.2.3 on 2023-10-06 15:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("sde_collections", "0031_candidateurl_is_pdf"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="collection",
+            name="workflow_status",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Research in Progress"),
+                    (2, "Config Created"),
+                    (3, "Engineering in Progress"),
+                    (4, "Engineering Completed"),
+                    (5, "URLs Generated and Ready to Curate"),
+                    (6, "Curation in Progress"),
+                    (7, "Curated"),
+                    (8, "Curation Code Reviewed"),
+                    (9, "Test Deployment Completed"),
+                    (10, "Test Indexing Completed"),
+                    (11, "Test Quality Checks in Progress"),
+                    (12, "Test Quality Checks Completed"),
+                    (13, "Production Deployment Completed"),
+                    (14, "Production Indexing Completed"),
+                    (15, "Production Quality Checks in Progress"),
+                    (16, "Production Quality Checks Completed"),
+                ],
+                default=1,
+            ),
+        ),
+    ]

sde_collections/models/collection.py

@@ -12,6 +12,7 @@
     DocumentTypes,
     SourceChoices,
     UpdateFrequencies,
+    WorkflowStatusChoices,
 )
 
 User = get_user_model()
@@ -25,7 +26,9 @@ class Collection(models.Model):
     url = models.URLField("URL", max_length=2048, blank=True)
     division = models.IntegerField(choices=Divisions.choices)
     turned_on = models.BooleanField("Turned On", default=True)
-    connector = models.IntegerField(choices=ConnectorChoices.choices, default=1)
+    connector = models.IntegerField(
+        choices=ConnectorChoices.choices, default=ConnectorChoices.CRAWLER2
+    )
 
     source = models.IntegerField(choices=SourceChoices.choices)
     update_frequency = models.IntegerField(
@@ -69,7 +72,12 @@ class Collection(models.Model):
     cleaning_order = models.IntegerField(default=0, blank=True)
 
     curation_status = models.IntegerField(
-        choices=CurationStatusChoices.choices, default=1
+        choices=CurationStatusChoices.choices,
+        default=CurationStatusChoices.NEEDS_SCRAPING,
+    )
+    workflow_status = models.IntegerField(
+        choices=WorkflowStatusChoices.choices,
+        default=WorkflowStatusChoices.RESEARCH_IN_PROGRESS,
     )
     curated_by = models.ForeignKey(
         User, on_delete=models.DO_NOTHING, null=True, blank=True
@@ -97,6 +105,28 @@ def curation_status_button_color(self) -> str:
         }
         return color_choices[self.curation_status]
 
+    @property
+    def workflow_status_button_color(self) -> str:
+        color_choices = {
+            1: "btn-light",
+            2: "btn-danger",
+            3: "btn-warning",
+            4: "btn-info",
+            5: "btn-success",
+            6: "btn-primary",
+            7: "btn-info",
+            8: "btn-secondary",
+            9: "btn-light",
+            10: "btn-danger",
+            11: "btn-warning",
+            12: "btn-info",
+            13: "btn-success",
+            14: "btn-primary",
+            15: "btn-info",
+            16: "btn-secondary",
+        }
+        return color_choices[self.workflow_status]
+
     def _process_exclude_list(self):
         """Process the exclude list."""
         return [

sde_collections/models/collection_choice_fields.py

@@ -61,3 +61,32 @@ def get_status_string(cls, value):
             if choice[0] == value:
                 return choice[1]
         return "N/A"
+
+
+class WorkflowStatusChoices(models.IntegerChoices):
+    # someone has started making a new collection, but they aren't done
+    RESEARCH_IN_PROGRESS = 1, "Research in Progress"
+    # someone finished inputing the collection details. finishing move caused a config to be created and sent to github
+    CONFIG_CREATED = 2, "Config Created"
+    # someone has claimed this collection for dev work on a server, and are working on it
+    ENGINEERING_IN_PROGRESS = 3, "Engineering in Progress"
+    # finished config code has been merged into the dev branch via PR, but urls may not be finished scraping
+    ENGINEERING_COMPLETED = 4, "Engineering Completed"
+    # config is on dev branch, and urls are fully scraped and imported into the webapp
+    URLS_GENERATED = 5, "URLs Generated and Ready to Curate"
+    # someone has claimed the collection and is in the process of curating it
+    CURATION_IN_PROGRESS = 6, "Curation in Progress"
+    # curation is finished, and this action sent a pr to github
+    CURATED = 7, "Curated"
+    # code made by webapp was reviewed and merged to test branch. status exists bc of the lag between review and deploy
+    CURATION_CODE_REVIEWED = 8, "Curation Code Reviewed"
+    # test branch code is deployed to test server and indexing has been kicked off
+    TEST_DEPLOYMENT_COMPLETED = 9, "Test Deployment Completed"
+    TEST_INDEXING_COMPLETED = 10, "Test Indexing Completed"
+    TEST_QUALITY_IN_PROGRESS = 11, "Test Quality Checks in Progress"
+    TEST_QUALITY_COMPLETE = 12, "Test Quality Checks Completed"
+    # test branch code is deployed to test server and indexing has been kicked off
+    PROD_DEPLOYMENT_COMPLETED = 13, "Production Deployment Completed"
+    PROD_INDEXING_COMPLETED = 14, "Production Indexing Completed"
+    PROD_QUALITY_IN_PROGRESS = 15, "Production Quality Checks in Progress"
+    PROD_QUALITY_COMPLETE = 16, "Production Quality Checks Completed"

sde_collections/serializers.py

@@ -10,13 +10,18 @@ class CollectionSerializer(serializers.ModelSerializer):
     curation_status_display = serializers.CharField(
         source="get_curation_status_display", read_only=True
     )
+    workflow_status_display = serializers.CharField(
+        source="get_workflow_status_display", read_only=True
+    )
 
     class Meta:
         model = Collection
         fields = (
             "id",
             "curation_status",
+            "workflow_status",
             "curation_status_display",
+            "workflow_status_display",
             "curated_by",
         )
         # extra_kwargs = {

sde_collections/tasks.py

@@ -17,6 +17,7 @@
 
 from .models.candidate_url import CandidateURL
 from .models.collection import Collection
+from .models.collection_choice_fields import WorkflowStatusChoices
 from .sinequa_api import Api
 from .utils.github_helper import GitHubHandler
 
@@ -86,6 +87,9 @@ def import_candidate_urls_task(collection_ids=[], config_folder_names=[]):
         subprocess.run(f'python manage.py loaddata "{urls_file}"', shell=True)
         collection.apply_all_patterns()
         collection.curation_status = 2  # ready to curate
+        collection.workflow_status = (
+            WorkflowStatusChoices.URLS_GENERATED
+        )  # ready to curate
         collection.save()
     shutil.rmtree(TEMP_FOLDER_NAME)
 

sde_collections/utils/github_helper.py

@@ -84,8 +84,8 @@ def create_pull_request(self) -> None:
             print("PR exists")
 
     def push_to_github(self) -> None:
-        if not self.branch_exists(self.github_update_branch):
-            self.create_branch(self.github_update_branch)
+        if not self.branch_exists(self.github_branch):
+            self.create_branch(self.github_branch)
         for collection in self.collections:
             print(f"Pushing {collection.name} to GitHub.")
             self._update_file_contents(collection)