From cf9635788318026cdff79262da7b54df8e10414d Mon Sep 17 00:00:00 2001 From: Carson Davis Date: Tue, 18 Jun 2024 10:54:22 -0500 Subject: [PATCH 1/2] change statuses to include delete and minor quality --- .../add_perfect_to_prod_query.py | 2 +- .../change_statuses_on_webapp.py | 56 ++++++----- ...ter_collection_workflow_status_and_more.py | 97 +++++++++++++++++++ sde_collections/models/collection.py | 10 +- .../models/collection_choice_fields.py | 12 ++- sde_collections/utils/slack_utils.py | 15 ++- 6 files changed, 153 insertions(+), 39 deletions(-) create mode 100644 sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py diff --git a/scripts/quality_and_indexing/add_perfect_to_prod_query.py b/scripts/quality_and_indexing/add_perfect_to_prod_query.py index 013be440..2b7d4333 100644 --- a/scripts/quality_and_indexing/add_perfect_to_prod_query.py +++ b/scripts/quality_and_indexing/add_perfect_to_prod_query.py @@ -6,6 +6,6 @@ from sde_collections.models.collection import Collection from sde_collections.models.collection_choice_fields import WorkflowStatusChoices -for collection in Collection.objects.filter(workflow_status=WorkflowStatusChoices.READY_FOR_PUBLIC_PROD): +for collection in Collection.objects.filter(workflow_status=WorkflowStatusChoices.QUALITY_CHECK_PERFECT): print(collection.config_folder) collection.add_to_public_query() diff --git a/scripts/quality_and_indexing/change_statuses_on_webapp.py b/scripts/quality_and_indexing/change_statuses_on_webapp.py index cb9abb30..9fbbbc6f 100644 --- a/scripts/quality_and_indexing/change_statuses_on_webapp.py +++ b/scripts/quality_and_indexing/change_statuses_on_webapp.py @@ -17,50 +17,54 @@ READY_FOR_LRM_QUALITY_CHECK = 10, "Ready for LRM Quality Check" READY_FOR_FINAL_QUALITY_CHECK = 11, "Ready for Quality Check" QUALITY_CHECK_FAILED = 12, "Quality Check Failed" -READY_FOR_PUBLIC_PROD = 13, "Ready for Public Production" -PERFECT_ON_PROD = 14, "Perfect and on Production" -LOW_PRIORITY_PROBLEMS_ON_PROD = 15, "Low Priority Problems on Production" -HIGH_PRIORITY_PROBLEMS_ON_PROD = 16, "High Priority Problems on Production, only for old sources" +QUALITY_CHECK_PERFECT = 13, "Ready for Public Production" +PROD_PERFECT = 14, "Perfect and on Production" +PROD_MINOR = 15, "Low Priority Problems on Production" +PROD_MAJOR = 16, "High Priority Problems on Production, only for old sources" MERGE_PENDING = 17, "Code Merge Pending" perfect = [ - # "WIND_Spacecraft", - # "gamma_ray_data_tools_core_package", - # "land_processes_distributed_active_archive_center", - # "mdscc_deep_space_network", - # "HelioAnalytics", - # "nasa_infrared_telescope_facility_irtf", - # "gmao_fluid", - # "starchild_a_learning_center_for_young_astronomers", - # "voyager_Cosmic_Ray_Subsystem", - "ldas_land_data_assimilatin_system", - "ppi_node", + "Van_Allen_Probes", + "gamma_ray_data_tools_github", + "hawc_observatory", + "activate_aerosol_cloud_meteorology_interactions_over_the_western_atlantic_experiment", + "nasa_visible_earth", + "global_sulfur_dioxide_monitoring", + "Voyager_Cosmic_Ray_Subsystem", + "stereo_at_gsfc", + "nasa_applied_sciences", + "cosmic_data_stories", + "solar_terrestrial_probes_program", + "atmospheric_imaging_assembly", + "treasure_map", + "incus_investigation_of_convective_updrafts", + "airmoss_airborne_microwave_observatory_of_subcanopy_and_subsurface_at_jpl", + "cii_hosted_payload_opportunity_online_database", + "act_america_atmospheric_carbon_and_transport_america", + "astropy", + "pds_website", + "astrophysics_source_code_library", ] low_priority = [ - "nasa_applied_sciences", - "parker_solar_probe", - "virtual_wave_observatory", - "explorer_program_acquisition", - "lisa_consortium", - "astropy", - "fermi_at_gsfc", - "microobservatory_robotic_telescope_network", + "nasa_arcgis_online", + "physics_of_the_cosmos", + "dscovr_epic_earth_polychromatic_imaging_camera", ] for config in perfect: print(config) collection = Collection.objects.get(config_folder=config) - collection.workflow_status = WorkflowStatusChoices.PERFECT_ON_PROD + collection.workflow_status = WorkflowStatusChoices.PROD_PERFECT collection.save() for config in low_priority: print(config) collection = Collection.objects.get(config_folder=config) - collection.workflow_status = WorkflowStatusChoices.LOW_PRIORITY_PROBLEMS_ON_PROD + collection.workflow_status = WorkflowStatusChoices.PROD_MINOR collection.save() # for config in perfect: # collection = Collection.objects.get(config_folder=config) -# collection.workflow_status = WorkflowStatusChoices.PERFECT_ON_PROD +# collection.workflow_status = WorkflowStatusChoices.PROD_PERFECT # collection.save() diff --git a/sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py b/sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py new file mode 100644 index 00000000..b827a139 --- /dev/null +++ b/sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py @@ -0,0 +1,97 @@ +# Generated by Django 4.2.9 on 2024-06-18 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0055_alter_workflowhistory_old_status_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="collection", + name="workflow_status", + field=models.IntegerField( + choices=[ + (1, "Research in Progress"), + (2, "Ready for Engineering"), + (3, "Engineering in Progress"), + (4, "Ready for Curation"), + (5, "Curation in Progress"), + (6, "Curated"), + (7, "Quality Fixed"), + (8, "Secret Deployment Started"), + (9, "Secret Deployment Failed"), + (10, "Ready for LRM Quality Check"), + (11, "Ready for Quality Check"), + (12, "QC: Failed"), + (18, "QC: Minor Issues"), + (13, "QC: Perfect"), + (14, "Prod: Perfect"), + (15, "Prod: Minor Issues"), + (16, "Prod: Major Issues"), + (17, "Code Merge Pending"), + (19, "Delete from Prod"), + ], + default=1, + ), + ), + migrations.AlterField( + model_name="workflowhistory", + name="old_status", + field=models.IntegerField( + choices=[ + (1, "Research in Progress"), + (2, "Ready for Engineering"), + (3, "Engineering in Progress"), + (4, "Ready for Curation"), + (5, "Curation in Progress"), + (6, "Curated"), + (7, "Quality Fixed"), + (8, "Secret Deployment Started"), + (9, "Secret Deployment Failed"), + (10, "Ready for LRM Quality Check"), + (11, "Ready for Quality Check"), + (12, "QC: Failed"), + (18, "QC: Minor Issues"), + (13, "QC: Perfect"), + (14, "Prod: Perfect"), + (15, "Prod: Minor Issues"), + (16, "Prod: Major Issues"), + (17, "Code Merge Pending"), + (19, "Delete from Prod"), + ], + null=True, + ), + ), + migrations.AlterField( + model_name="workflowhistory", + name="workflow_status", + field=models.IntegerField( + choices=[ + (1, "Research in Progress"), + (2, "Ready for Engineering"), + (3, "Engineering in Progress"), + (4, "Ready for Curation"), + (5, "Curation in Progress"), + (6, "Curated"), + (7, "Quality Fixed"), + (8, "Secret Deployment Started"), + (9, "Secret Deployment Failed"), + (10, "Ready for LRM Quality Check"), + (11, "Ready for Quality Check"), + (12, "QC: Failed"), + (18, "QC: Minor Issues"), + (13, "QC: Perfect"), + (14, "Prod: Perfect"), + (15, "Prod: Minor Issues"), + (16, "Prod: Major Issues"), + (17, "Code Merge Pending"), + (19, "Delete from Prod"), + ], + default=1, + ), + ), + ] diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index 5f082a81..7ec8259e 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -84,7 +84,10 @@ class Meta: def add_to_public_query(self): """Add the collection to the public query.""" - if self.workflow_status != WorkflowStatusChoices.READY_FOR_PUBLIC_PROD: + if self.workflow_status not in [ + WorkflowStatusChoices.QUALITY_CHECK_PERFECT, + WorkflowStatusChoices.QUALITY_CHECK_MINOR, + ]: raise ValueError(f"{self.config_folder} is not ready for public prod, you can't add it to the public query") gh = GitHubHandler() @@ -562,5 +565,8 @@ def create_configs_on_status_change(sender, instance, created, **kwargs): elif instance.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING: instance.create_scraper_config(overwrite=False) instance.create_indexer_config(overwrite=False) - elif instance.workflow_status == WorkflowStatusChoices.READY_FOR_PUBLIC_PROD: + elif instance.workflow_status in [ + WorkflowStatusChoices.QUALITY_CHECK_PERFECT, + WorkflowStatusChoices.QUALITY_CHECK_MINOR, + ]: instance.add_to_public_query() diff --git a/sde_collections/models/collection_choice_fields.py b/sde_collections/models/collection_choice_fields.py index 37ac9412..ea627ce5 100644 --- a/sde_collections/models/collection_choice_fields.py +++ b/sde_collections/models/collection_choice_fields.py @@ -90,9 +90,11 @@ class WorkflowStatusChoices(models.IntegerChoices): SECRET_DEPLOYMENT_FAILED = 9, "Secret Deployment Failed" READY_FOR_LRM_QUALITY_CHECK = 10, "Ready for LRM Quality Check" READY_FOR_FINAL_QUALITY_CHECK = 11, "Ready for Quality Check" - QUALITY_CHECK_FAILED = 12, "Quality Check Failed" - READY_FOR_PUBLIC_PROD = 13, "Ready for Public Production" - PERFECT_ON_PROD = 14, "Perfect and on Production" - LOW_PRIORITY_PROBLEMS_ON_PROD = 15, "Low Priority Problems on Production" - HIGH_PRIORITY_PROBLEMS_ON_PROD = 16, "High Priority Problems on Production, only for old sources" + QUALITY_CHECK_FAILED = 12, "QC: Failed" + QUALITY_CHECK_MINOR = 18, "QC: Minor Issues" + QUALITY_CHECK_PERFECT = 13, "QC: Perfect" + PROD_PERFECT = 14, "Prod: Perfect" + PROD_MINOR = 15, "Prod: Minor Issues" + PROD_MAJOR = 16, "Prod: Major Issues" MERGE_PENDING = 17, "Code Merge Pending" + NEEDS_DELETE = 19, "Delete from Prod" diff --git a/sde_collections/utils/slack_utils.py b/sde_collections/utils/slack_utils.py index 796d9b48..c4cfd78b 100644 --- a/sde_collections/utils/slack_utils.py +++ b/sde_collections/utils/slack_utils.py @@ -58,24 +58,29 @@ SLACK_ID_MAPPING["Xiang Li"], SLACK_ID_MAPPING["Shravan Vishwanathan"], SLACK_ID_MAPPING["Advait Yogaonkar"], + ], + }, + (WorkflowStatusChoices.READY_FOR_FINAL_QUALITY_CHECK, WorkflowStatusChoices.QUALITY_CHECK_PERFECT): { + "message": "{name} has passed all quality checks and is ready for public production! :white_check_mark:", + "tags": [ SLACK_ID_MAPPING["Carson Davis"], SLACK_ID_MAPPING["Bishwas Praveen"], SLACK_ID_MAPPING["Ashish Acharya"], ], }, - (WorkflowStatusChoices.READY_FOR_FINAL_QUALITY_CHECK, WorkflowStatusChoices.READY_FOR_PUBLIC_PROD): { - "message": "{name} has passed all quality checks and is ready for public production! :trophy:", + (WorkflowStatusChoices.READY_FOR_FINAL_QUALITY_CHECK, WorkflowStatusChoices.QUALITY_CHECK_MINOR): { + "message": "{name} has passed all quality checks and is ready for public production! :white_check_mark:", "tags": [ SLACK_ID_MAPPING["Carson Davis"], SLACK_ID_MAPPING["Bishwas Praveen"], SLACK_ID_MAPPING["Ashish Acharya"], ], }, - (WorkflowStatusChoices.READY_FOR_FINAL_QUALITY_CHECK, WorkflowStatusChoices.LOW_PRIORITY_PROBLEMS_ON_PROD): { - "message": "{name} is now on Public Prod and is almost perfect, with minor issues noted. Please review! :memo:", + (WorkflowStatusChoices.QUALITY_CHECK_PERFECT, WorkflowStatusChoices.PROD_PERFECT): { + "message": "{name} is now live on Public Prod! Congrats team! :sparkles:", "tags": [SLACK_ID_MAPPING["channel"]], }, - (WorkflowStatusChoices.READY_FOR_PUBLIC_PROD, WorkflowStatusChoices.PERFECT_ON_PROD): { + (WorkflowStatusChoices.QUALITY_CHECK_MINOR, WorkflowStatusChoices.PROD_MINOR): { "message": "{name} is now live on Public Prod! Congrats team! :sparkles:", "tags": [SLACK_ID_MAPPING["channel"]], }, From d0e94f9a000db26be2f20e4a3a2c8968fc1a5ef0 Mon Sep 17 00:00:00 2001 From: Bishwas Praveen Date: Wed, 31 Jul 2024 09:34:48 -0500 Subject: [PATCH 2/2] added a new migration file --- ...e.py => 0057_alter_collection_workflow_status_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename sde_collections/migrations/{0056_alter_collection_workflow_status_and_more.py => 0057_alter_collection_workflow_status_and_more.py} (96%) diff --git a/sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py b/sde_collections/migrations/0057_alter_collection_workflow_status_and_more.py similarity index 96% rename from sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py rename to sde_collections/migrations/0057_alter_collection_workflow_status_and_more.py index b827a139..ddaa340f 100644 --- a/sde_collections/migrations/0056_alter_collection_workflow_status_and_more.py +++ b/sde_collections/migrations/0057_alter_collection_workflow_status_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.9 on 2024-06-18 15:53 +# Generated by Django 4.2.9 on 2024-07-31 14:23 from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("sde_collections", "0055_alter_workflowhistory_old_status_and_more"), + ("sde_collections", "0056_alter_candidateurl_document_type_and_more"), ] operations = [