From ece1344a0e5419c5813ce7326293d6ae562987ff Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Thu, 21 Dec 2023 23:15:57 +0545
Subject: [PATCH 1/2] Mock DeepL in assisted_tagging test

- Fix analysis entries cluster ordering issue
---
 apps/analysis/dataloaders.py                  |   2 +-
 apps/assisted_tagging/tasks.py                |   4 +-
 .../tests/snapshots/snap_test_query.py        | 392 ++++++++++++++++--
 apps/assisted_tagging/tests/test_query.py     |  16 +-
 4 files changed, 361 insertions(+), 53 deletions(-)

diff --git a/apps/analysis/dataloaders.py b/apps/analysis/dataloaders.py
index 56f9f3ee51..76d6eb8da2 100644
--- a/apps/analysis/dataloaders.py
+++ b/apps/analysis/dataloaders.py
@@ -138,7 +138,7 @@ class AnalysisTopicModelClusterEntryLoader(DataLoaderWithContext):
     def batch_load_fn(self, keys):
         qs = TopicModelCluster.entries.through.objects.filter(
             topicmodelcluster__in=keys,
-        ).select_related('entry').order_by('topicmodelcluster', 'entry')
+        ).select_related('entry').order_by('topicmodelcluster', 'entry_id')
         _map = defaultdict(list)
         for cluster_entry in qs:
             _map[cluster_entry.topicmodelcluster_id].append(cluster_entry.entry)
diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py
index 3a7ca7fa46..c809c30bd4 100644
--- a/apps/assisted_tagging/tasks.py
+++ b/apps/assisted_tagging/tasks.py
@@ -41,7 +41,7 @@ def _get_existing_tags_by_tagid():
         assisted_tag.group = tag_meta.get('group')
         assisted_tag.is_category = tag_meta['is_category']
         assisted_tag.hide_in_analysis_framework_mapping = tag_meta['hide_in_analysis_framework_mapping']
-        if assisted_tag.id:
+        if assisted_tag.pk:
             updated_tags.append(assisted_tag)
         else:
             assisted_tag.tag_id = tag_id
@@ -54,8 +54,8 @@ def _get_existing_tags_by_tagid():
             updated_tags,
             fields=(
                 'name',
-                'is_category',
                 'group',
+                'is_category',
                 'hide_in_analysis_framework_mapping',
             )
         )
diff --git a/apps/assisted_tagging/tests/snapshots/snap_test_query.py b/apps/assisted_tagging/tests/snapshots/snap_test_query.py
index c20c3ca0c4..e2c256558d 100644
--- a/apps/assisted_tagging/tests/snapshots/snap_test_query.py
+++ b/apps/assisted_tagging/tests/snapshots/snap_test_query.py
@@ -22,231 +22,531 @@
             'name': 'all_tags_model'
         }
     ],
-    'tag_count': 45,
+    'tag_count': 105,
     'tags': [
         {
             'is_deprecated': False,
-            'name': '1',
+            'name': 'sectors',
             'tag_id': '1'
         },
         {
             'is_deprecated': False,
-            'name': '101',
+            'name': 'reliability',
+            'tag_id': '10'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Completely reliable',
+            'tag_id': '1001'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Usually reliable',
+            'tag_id': '1002'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Fairly Reliable',
+            'tag_id': '1003'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Unreliable',
+            'tag_id': '1004'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Agriculture',
             'tag_id': '101'
         },
         {
             'is_deprecated': False,
-            'name': '102',
+            'name': 'Cross',
             'tag_id': '102'
         },
         {
             'is_deprecated': False,
-            'name': '103',
+            'name': 'Education',
             'tag_id': '103'
         },
         {
             'is_deprecated': False,
-            'name': '104',
+            'name': 'Food Security',
             'tag_id': '104'
         },
         {
             'is_deprecated': False,
-            'name': '2',
+            'name': 'subpillars_1d',
             'tag_id': '2'
         },
         {
             'is_deprecated': False,
-            'name': '204',
+            'name': 'Environment',
+            'tag_id': '201'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Socio Cultural',
+            'tag_id': '202'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Economy',
+            'tag_id': '203'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Demography',
             'tag_id': '204'
         },
         {
             'is_deprecated': False,
-            'name': '209',
+            'name': 'Legal & Policy',
+            'tag_id': '205'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Security & Stability',
+            'tag_id': '206'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Politics',
+            'tag_id': '207'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Type And Characteristics',
+            'tag_id': '208'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Underlying/Aggravating Factors',
             'tag_id': '209'
         },
         {
             'is_deprecated': False,
-            'name': '214',
+            'name': 'Hazard & Threats',
+            'tag_id': '210'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Type/Numbers/Movements',
+            'tag_id': '212'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Push Factors',
+            'tag_id': '213'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Pull Factors',
             'tag_id': '214'
         },
         {
             'is_deprecated': False,
-            'name': '216',
+            'name': 'Intentions',
+            'tag_id': '215'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Local Integration',
             'tag_id': '216'
         },
         {
             'is_deprecated': False,
-            'name': '217',
+            'name': 'Injured',
             'tag_id': '217'
         },
         {
             'is_deprecated': False,
-            'name': '218',
+            'name': 'Missing',
             'tag_id': '218'
         },
         {
             'is_deprecated': False,
-            'name': '219',
+            'name': 'Dead',
             'tag_id': '219'
         },
         {
             'is_deprecated': False,
-            'name': '3',
+            'name': 'Relief To Population',
+            'tag_id': '220'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Population To Relief',
+            'tag_id': '221'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Physical Constraints',
+            'tag_id': '222'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps',
+            'tag_id': '223'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Communication Means And Preferences',
+            'tag_id': '224'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Information Challenges And Barriers',
+            'tag_id': '225'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Knowledge And Info Gaps (Pop)',
+            'tag_id': '226'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Knowledge And Info Gaps (Hum)',
+            'tag_id': '227'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Cases',
+            'tag_id': '228'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Contact Tracing',
+            'tag_id': '229'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Deaths',
+            'tag_id': '230'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Hospitalization & Care',
+            'tag_id': '231'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Restriction Measures',
+            'tag_id': '232'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Testing',
+            'tag_id': '233'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Vaccination',
+            'tag_id': '234'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'subpillars_2d',
             'tag_id': '3'
         },
         {
             'is_deprecated': False,
-            'name': '301',
+            'name': 'Number Of People At Risk',
             'tag_id': '301'
         },
         {
             'is_deprecated': False,
-            'name': '302',
+            'name': 'Risk And Vulnerabilities',
             'tag_id': '302'
         },
         {
             'is_deprecated': False,
-            'name': '303',
+            'name': 'International Response',
             'tag_id': '303'
         },
         {
             'is_deprecated': False,
-            'name': '304',
+            'name': 'Local Response',
             'tag_id': '304'
         },
         {
             'is_deprecated': False,
-            'name': '315',
+            'name': 'National Response',
+            'tag_id': '305'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Number Of People Reached/Response Gaps',
+            'tag_id': '306'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Coping Mechanisms',
+            'tag_id': '307'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Living Standards',
+            'tag_id': '308'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Number Of People In Need',
+            'tag_id': '309'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Physical And Mental Well Being',
+            'tag_id': '310'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Driver/Aggravating Factors',
+            'tag_id': '311'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Impact On People',
+            'tag_id': '312'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Impact On Systems, Services And Networks',
+            'tag_id': '313'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Number Of People Affected',
+            'tag_id': '314'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Expressed By Humanitarian Staff',
             'tag_id': '315'
         },
         {
             'is_deprecated': False,
-            'name': '316',
+            'name': 'Expressed By Population',
             'tag_id': '316'
         },
         {
             'is_deprecated': False,
-            'name': '317',
+            'name': 'Expressed By Humanitarian Staff',
             'tag_id': '317'
         },
         {
             'is_deprecated': False,
-            'name': '318',
+            'name': 'Expressed By Population',
             'tag_id': '318'
         },
         {
             'is_deprecated': False,
-            'name': '4',
+            'name': 'specific_needs_groups',
             'tag_id': '4'
         },
         {
             'is_deprecated': False,
-            'name': '401',
+            'name': 'Child Head of Household',
             'tag_id': '401'
         },
         {
             'is_deprecated': False,
-            'name': '402',
+            'name': 'Chronically Ill',
             'tag_id': '402'
         },
         {
             'is_deprecated': False,
-            'name': '407',
+            'name': 'Elderly Head of Household',
+            'tag_id': '403'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Female Head of Household',
+            'tag_id': '404'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'GBV survivors',
+            'tag_id': '405'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Indigenous people',
+            'tag_id': '406'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'LGBTQI+',
             'tag_id': '407'
         },
         {
             'is_deprecated': False,
-            'name': '408',
+            'name': 'Minorities',
             'tag_id': '408'
         },
         {
             'is_deprecated': False,
-            'name': '412',
+            'name': 'Persons with Disability',
+            'tag_id': '409'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Pregnant or Lactating Women',
+            'tag_id': '410'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Single Women (including Widows)',
+            'tag_id': '411'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Unaccompanied or Separated Children',
             'tag_id': '412'
         },
         {
             'is_deprecated': False,
-            'name': '5',
+            'name': 'gender',
             'tag_id': '5'
         },
         {
             'is_deprecated': False,
-            'name': '501',
+            'name': 'Female',
             'tag_id': '501'
         },
         {
             'is_deprecated': False,
-            'name': '502',
+            'name': 'Male',
             'tag_id': '502'
         },
         {
             'is_deprecated': False,
-            'name': '6',
+            'name': 'age',
             'tag_id': '6'
         },
         {
             'is_deprecated': False,
-            'name': '601',
+            'name': 'Adult (18 to 59 years old)',
             'tag_id': '601'
         },
         {
             'is_deprecated': False,
-            'name': '602',
+            'name': 'Children/Youth (5 to 17 years old)',
             'tag_id': '602'
         },
         {
             'is_deprecated': False,
-            'name': '603',
+            'name': 'Infants/Toddlers (<5 years old)',
             'tag_id': '603'
         },
         {
             'is_deprecated': False,
-            'name': '604',
+            'name': 'Older Persons (60+ years old)',
             'tag_id': '604'
         },
         {
             'is_deprecated': False,
-            'name': '7',
+            'name': 'severity',
             'tag_id': '7'
         },
         {
             'is_deprecated': False,
-            'name': '701',
+            'name': 'Critical',
             'tag_id': '701'
         },
         {
             'is_deprecated': False,
-            'name': '8',
+            'name': 'Major',
+            'tag_id': '702'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Minor Problem',
+            'tag_id': '703'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'No problem',
+            'tag_id': '704'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Of Concern',
+            'tag_id': '705'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'affected_groups',
             'tag_id': '8'
         },
         {
             'is_deprecated': False,
-            'name': '801',
+            'name': 'Asylum Seekers',
             'tag_id': '801'
         },
         {
             'is_deprecated': False,
-            'name': '802',
+            'name': 'Host',
             'tag_id': '802'
         },
         {
             'is_deprecated': False,
-            'name': '9',
+            'name': 'IDP',
+            'tag_id': '803'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Migrants',
+            'tag_id': '804'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Refugees',
+            'tag_id': '805'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Returnees',
+            'tag_id': '806'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'demographic_group',
             'tag_id': '9'
         },
         {
             'is_deprecated': False,
-            'name': '904',
+            'name': 'Infants/Toddlers (<5 years old) ',
+            'tag_id': '901'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Female Children/Youth (5 to 17 years old)',
+            'tag_id': '902'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Male Children/Youth (5 to 17 years old)',
+            'tag_id': '903'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Female Adult (18 to 59 years old)',
             'tag_id': '904'
         },
         {
             'is_deprecated': False,
-            'name': '905',
+            'name': 'Male Adult (18 to 59 years old)',
             'tag_id': '905'
         },
         {
             'is_deprecated': False,
-            'name': '907',
+            'name': 'Female Older Persons (60+ years old)',
+            'tag_id': '906'
+        },
+        {
+            'is_deprecated': False,
+            'name': 'Male Older Persons (60+ years old)',
             'tag_id': '907'
         }
     ]
diff --git a/apps/assisted_tagging/tests/test_query.py b/apps/assisted_tagging/tests/test_query.py
index 119b999bfb..f41e2bd57a 100644
--- a/apps/assisted_tagging/tests/test_query.py
+++ b/apps/assisted_tagging/tests/test_query.py
@@ -1273,6 +1273,17 @@ class AssistedTaggingCallbackApiTest(TestCase, SnapShotTextCase):
         },
     }
 
+    def setUp(self):
+        super().setUp()
+        self.sync_request_mock = patch('assisted_tagging.tasks.requests')
+        mock = self.sync_request_mock.start()
+        mock.get.return_value.status_code = 200
+        mock.get.return_value.json.return_value = self.DEEPL_TAGS_MOCK_RESPONSE
+
+    def tearDown(self):
+        self.sync_request_mock.stop()
+        super().tearDown()
+
     def test_save_draft_entry(self):
         def _check_draft_entry_status(draft_entry, status):
             draft_entry.refresh_from_db()
@@ -1388,8 +1399,7 @@ def _get_current_prediction_stats():
         self.assertMatchSnapshot(current_model_stats, 'final-current-model-stats')
         self.assertMatchSnapshot(current_prediction_stats, 'final-current-prediction-stats')
 
-    @patch('assisted_tagging.tasks.requests')
-    def test_tags_sync(self, sync_request_mock):
+    def test_tags_sync(self):
         def _get_current_tags():
             return list(
                 AssistedTaggingModelPredictionTag.objects.values(
@@ -1404,8 +1414,6 @@ def _get_current_tags():
             )
 
         self.maxDiff = None
-        sync_request_mock.get.return_value.status_code = 200
-        sync_request_mock.get.return_value.json.return_value = self.DEEPL_TAGS_MOCK_RESPONSE
         self.assertEqual(len(_get_current_tags()), 0)
         sync_tags_with_deepl()
         self.assertNotEqual(len(_get_current_tags()), 0)

From 04fda4ea654b7658445e179177ebcf2bdc6d069c Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Thu, 21 Dec 2023 23:40:21 +0545
Subject: [PATCH 2/2] Update auto_entry_extraction_status to FAILED

---
 apps/deepl_integration/handlers.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py
index 846a4626b2..e024a13faa 100644
--- a/apps/deepl_integration/handlers.py
+++ b/apps/deepl_integration/handlers.py
@@ -359,7 +359,7 @@ def auto_trigger_request_to_extractor(cls, lead):
         payload = {
             "documents": [
                 {
-                    "client_id": cls.get_client_id(lead),  # static clientid for mock
+                    "client_id": cls.get_client_id(lead),
                     "text_extraction_id": str(lead_preview.text_extraction_id)
                 }
             ],
@@ -378,6 +378,8 @@ def auto_trigger_request_to_extractor(cls, lead):
 
         except Exception:
             logger.error('Entry Extraction send failed, Exception occurred!!', exc_info=True)
+            lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.FAILED
+            lead.save(update_fields=('auto_entry_extraction_status',))
         _response = locals().get('response')
         logger.error(
             'Entry Extraction send failed!!',
@@ -387,6 +389,7 @@ def auto_trigger_request_to_extractor(cls, lead):
             },
         )
 
+    # --- Callback logics
     @staticmethod
     def _get_or_create_models_version(models_data):
         def get_versions_map():
@@ -442,7 +445,8 @@ def get_tags_map():
         current_tags_map = get_tags_map()
         # Check if new tags needs to be created
         new_tags = [
-            tag for tag in tags
+            tag
+            for tag in tags
             if tag not in current_tags_map
         ]
         if new_tags:
@@ -467,16 +471,13 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode
 
         tags = model_prediction.get('classification', {})  # NLP TagId
         values = model_prediction.get('values', [])  # Raw value
+
         common_attrs = dict(
             model_version=model_version,
             draft_entry_id=draft_entry.id,
         )
         new_predictions = []
         for category_tag, tags in tags.items():
-            common_attrs = dict(
-                model_version=model_version,
-                draft_entry_id=draft_entry.id
-            )
             for tag, prediction_data in tags.items():
                 prediction_value = prediction_data.get('prediction')
                 threshold_value = prediction_data.get('threshold')
@@ -540,8 +541,8 @@ def save_data(cls, lead, data_url):
                 (data['classification_model_info']['name'], data['classification_model_info']['version'])
             ]
             cls._process_model_preds(model_version, current_tags_map, draft, model_preds)
-            lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS
-            lead.save(update_fields=('auto_entry_extraction_status',))
+        lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS
+        lead.save(update_fields=('auto_entry_extraction_status',))
         return lead