From 738a1aedf2d327a51e953f38877bc672d3ce2d46 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Tue, 11 Jul 2023 16:20:13 +0545 Subject: [PATCH 01/24] Switch to new DEEPL text extraction endpoint --- apps/deepl_integration/handlers.py | 13 ++++-- apps/deepl_integration/serializers.py | 45 +++++++++---------- apps/lead/tests/test_apis.py | 5 ++- apps/lead/typings.py | 2 +- apps/unified_connector/tests/test_mutation.py | 5 ++- deep/deepl.py | 4 +- 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index f67721017f..e5991df3e1 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -39,7 +39,7 @@ LeadPreview, LeadPreviewImage, ) -from lead.typings import NlpExtractorUrl +from lead.typings import NlpExtractorDocument from entry.models import Entry from analysis.models import ( TopicModel, @@ -85,6 +85,11 @@ def _make_hash_value(self, instance, timestamp): return str(type(instance)) + str(instance.pk) + str(timestamp) +class NlpRequestType: + SYSTEM = 0 # Note: SYSTEM refers to requests from CONNECTORS. + USER = 1 + + class BaseHandler: REQUEST_HEADERS = { 'Content-Type': 'application/json', @@ -352,14 +357,14 @@ class LeadExtractionHandler(BaseHandler): @classmethod def send_trigger_request_to_extractor( cls, - urls: List[NlpExtractorUrl], + documents: List[NlpExtractorDocument], callback_url: str, high_priority=False, ): payload = { - 'urls': urls, + 'documents': documents, 'callback_url': callback_url, - 'type': 'user' if high_priority else 'system', + 'request_type': NlpRequestType.USER if high_priority else NlpRequestType.SYSTEM, } try: response = requests.post( diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 888ae3092c..6b263d2004 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -47,13 +47,23 @@ def validate(self, data): return data +class DeeplServerBaseCallbackSerializer(BaseCallbackSerializer): + class Status(models.IntegerChoices): + # NOTE: Defined by NLP + # INITIATED = 1, 'Initiated' # Not needed or used by deep + SUCCESS = 2, 'Success' + FAILED = 3, 'Failed' + INPUT_URL_PROCESS_FAILED = 4, 'Input url process failed' + + status = serializers.ChoiceField(choices=Status.choices) + + # -- Lead -class LeadExtractCallbackSerializer(BaseCallbackSerializer): +class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): """ Serialize deepl extractor """ url = serializers.CharField() - extraction_status = serializers.IntegerField() # 0 = Failed, 1 = Success # Data fields images_path = serializers.ListField( child=serializers.CharField(allow_blank=True), @@ -68,21 +78,21 @@ class LeadExtractCallbackSerializer(BaseCallbackSerializer): def validate(self, data): data = super().validate(data) # Additional validation - if data['extraction_status'] == 1 and data.get('text_path') in [None, '']: + if data['status'] == self.Status.SUCCESS and data.get('text_path') in [None, '']: raise serializers.ValidationError({ - 'text_path': 'text_path is required when extraction_status is success/1' + 'text_path': 'text_path is required when extraction status is success' }) - if data['extraction_status'] == 1: + if data['status'] == self.Status.SUCCESS: errors = {} for key in ['text_path', 'total_words_count', 'total_pages']: if key not in data: - errors[key] = f'<{key}> is missing. Required when the extraction is 1 (Success)' + errors[key] = f'<{key}> is missing. Required when the extraction status is Success' if errors: raise serializers.ValidationError(errors) return data def create(self, data): - success = data['extraction_status'] == 1 + success = data['status'] == self.Status.SUCCESS lead = data['object'] # Added from validate if success: lead = self.nlp_handler.save_data( @@ -100,12 +110,11 @@ def create(self, data): # -- Unified Connector -class UnifiedConnectorLeadExtractCallbackSerializer(BaseCallbackSerializer): +class UnifiedConnectorLeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): """ Serialize deepl extractor """ url = serializers.CharField() - extraction_status = serializers.IntegerField() # 0 = Failed, 1 = Success # Data fields images_path = serializers.ListField( child=serializers.CharField(allow_blank=True), @@ -123,17 +132,17 @@ def validate(self, data): raise serializers.ValidationError({ 'url': 'Different url found provided vs original connector lead', }) - if data['extraction_status'] == 1: + if data['status'] == self.Status.SUCCESS: errors = {} for key in ['text_path', 'total_words_count', 'total_pages']: if key not in data: - errors[key] = f'<{key}> is missing. Required when the extraction is 1 (Success)' + errors[key] = f'<{key}> is missing. Required when the extraction is Success' if errors: raise serializers.ValidationError(errors) return data def create(self, data): - success = data['extraction_status'] == 1 + success = data['status'] == self.Status.SUCCESS connector_lead = data['object'] # Added from validate if success: return self.nlp_handler.save_data( @@ -197,17 +206,6 @@ def create(self, validated_data): ) -# -- Analysis -class DeeplServerBaseCallbackSerializer(BaseCallbackSerializer): - class Status(models.IntegerChoices): - # NOTE: Defined by NLP - SUCCESS = 2, 'Success' - FAILED = 3, 'Failed' - INPUT_URL_PROCESS_FAILED = 4, 'Input url process failed' - - status = serializers.ChoiceField(choices=Status.choices) - - class EntriesCollectionBaseCallbackSerializer(DeeplServerBaseCallbackSerializer): model: Type[DeeplTrackBaseModel] presigned_s3_url = serializers.URLField() @@ -222,6 +220,7 @@ def create(self, validated_data): return obj +# -- Analysis class AnalysisTopicModelCallbackSerializer(EntriesCollectionBaseCallbackSerializer): model = TopicModel nlp_handler = AnalysisTopicModelHandler diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 345586d414..13d149c2ac 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -26,6 +26,7 @@ from lead.filter_set import LeadFilterSet from lead.serializers import SimpleLeadGroupSerializer from deepl_integration.handlers import LeadExtractionHandler +from deepl_integration.serializers import DeeplServerBaseCallbackSerializer from entry.models import ( Entry, ProjectEntryLabel, @@ -1800,7 +1801,7 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f 'url': 'http://random.com/pdf_file.pdf', 'total_words_count': 300, 'total_pages': 4, - 'extraction_status': 0, + 'status': DeeplServerBaseCallbackSerializer.Status.FAILED.value, } # After callback [Failure] @@ -1811,7 +1812,7 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f self.assertEqual(LeadPreview.objects.filter(lead=self.lead).count(), 0) self.assertEqual(LeadPreviewImage.objects.filter(lead=self.lead).count(), 0) - data['extraction_status'] = 1 + data['status'] = DeeplServerBaseCallbackSerializer.Status.SUCCESS.value # After callback [Success] with self.captureOnCommitCallbacks(execute=True): response = self.client.post(url, data) diff --git a/apps/lead/typings.py b/apps/lead/typings.py index a69749d232..ea8979e1e2 100644 --- a/apps/lead/typings.py +++ b/apps/lead/typings.py @@ -1,6 +1,6 @@ from typing import TypedDict -class NlpExtractorUrl(TypedDict): +class NlpExtractorDocument(TypedDict): url: str client_id: str diff --git a/apps/unified_connector/tests/test_mutation.py b/apps/unified_connector/tests/test_mutation.py index 3979cac3bf..9ee21837b3 100644 --- a/apps/unified_connector/tests/test_mutation.py +++ b/apps/unified_connector/tests/test_mutation.py @@ -15,6 +15,7 @@ ConnectorLeadPreviewImage, ) from deepl_integration.handlers import UnifiedConnectorLeadHandler +from deepl_integration.serializers import DeeplServerBaseCallbackSerializer from unified_connector.factories import ( ConnectorLeadFactory, ConnectorSourceFactory, @@ -496,7 +497,7 @@ def _check_connector_lead_status(connector_lead, status): text_path='https://example.com/url-where-data-is-fetched-from-mock-response', total_words_count=100, total_pages=10, - extraction_status=0, # Failed + status=DeeplServerBaseCallbackSerializer.Status.FAILED.value, ) response = self.client.post(url, data) @@ -522,7 +523,7 @@ def _check_connector_lead_status(connector_lead, status): text_path='https://example.com/url-where-data-is-fetched-from-mock-response', total_words_count=100, total_pages=10, - extraction_status=1, # Failed + status=DeeplServerBaseCallbackSerializer.Status.SUCCESS.value, ) response = self.client.post(url, data) self.assert_400(response) diff --git a/deep/deepl.py b/deep/deepl.py index fef134e968..25dd329f53 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -8,12 +8,12 @@ class DeeplServiceEndpoint(): # DEEPL Service Endpoints (Existing/Legacy) # NOTE: This will be moved to server endpoints in near future - DOCS_EXTRACTOR_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/extract_docs' ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/vf_tags' ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/model_info' ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/entry_predict' - # DEEPL Server Endpoints (New/Legacy) + # DEEPL Server Endpoints (New) + DOCS_EXTRACTOR_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/text-extraction/' ANALYSIS_TOPIC_MODEL = f'{DEEPL_SERVER_DOMAIN}/api/v1/topicmodel/' ANALYSIS_AUTOMATIC_SUMMARY = f'{DEEPL_SERVER_DOMAIN}/api/v1/summarization/' ANALYSIS_AUTOMATIC_NGRAM = f'{DEEPL_SERVER_DOMAIN}/api/v1/ngrams/' From f0a80781f43a91713f441c7d638f36485f4c7126 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Tue, 11 Jul 2023 16:42:16 +0545 Subject: [PATCH 02/24] Update GH CI actions --- .github/workflows/ci.yml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 953daf3f7e..006fa92c1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: name: ๐Ÿšด Build + Test ๐Ÿšด # Match the name below (8398a7/action-slack). runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@master - name: ๐Ÿณ Prepare Docker id: prep @@ -24,17 +24,8 @@ jobs: id: buildx uses: docker/setup-buildx-action@master - - name: ๐Ÿณ Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.ref }} - restore-keys: | - ${{ runner.os }}-buildx-refs/develop - ${{ runner.os }}-buildx- - - name: ๐Ÿณ Build image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . builder: ${{ steps.buildx.outputs.name }} @@ -43,8 +34,9 @@ jobs: load: true target: worker # this has all the dep tags: ${{ steps.prep.outputs.tagged_image }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new + # Using experimental GH api: https://docs.docker.com/build/ci/github-actions/cache/#cache-backend-api + cache-from: type=gha + cache-to: type=gha,mode=max - name: ๐Ÿ•ฎ Validate latest graphql schema. env: @@ -67,7 +59,7 @@ jobs: } - name: ๐Ÿคž Run Test ๐Ÿงช & Publish coverage to code climate - uses: paambaati/codeclimate-action@v2.7.5 + uses: paambaati/codeclimate-action@v5.0.0 env: CC_TEST_REPORTER_ID: ${{ secrets.CODE_CLIMATE_ID }} DOCKER_IMAGE_SERVER: ${{ steps.prep.outputs.tagged_image }} @@ -77,19 +69,10 @@ jobs: ${{github.workspace}}/coverage/coverage.xml:coverage.py - name: Publish coverage to code cov - uses: codecov/codecov-action@v2 - - # Temp fix - # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#github-cache - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: ๐Ÿณ Move docker cache (๐Ÿง™ Hack fix) - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + uses: codecov/codecov-action@v3 - name: Deploy coverage to GH Pages ๐Ÿš€ - uses: JamesIves/github-pages-deploy-action@4.1.4 + uses: JamesIves/github-pages-deploy-action@v4 if: github.ref == 'refs/heads/develop' && github.event_name == 'push' with: branch: gh-pages From 3185d19717b7866758bbd273ae4e441376d2b829 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Thu, 9 Nov 2023 18:06:17 +0545 Subject: [PATCH 03/24] Add tag in analysis_framework --- apps/analysis_framework/admin.py | 6 + apps/analysis_framework/dataloaders.py | 16 +++ apps/analysis_framework/factories.py | 25 ++++ apps/analysis_framework/filter_set.py | 19 +++ .../migrations/0040_auto_20231109_1208.py | 27 ++++ apps/analysis_framework/models.py | 7 ++ apps/analysis_framework/schema.py | 42 ++++++- .../tests/snapshots/snap_test_schemas.py | 117 ++++++++++++++++++ apps/analysis_framework/tests/test_filters.py | 17 ++- apps/analysis_framework/tests/test_schemas.py | 20 ++- schema.graphql | 25 +++- 11 files changed, 310 insertions(+), 11 deletions(-) create mode 100644 apps/analysis_framework/migrations/0040_auto_20231109_1208.py diff --git a/apps/analysis_framework/admin.py b/apps/analysis_framework/admin.py index 0c23fc8d45..9c50badab6 100644 --- a/apps/analysis_framework/admin.py +++ b/apps/analysis_framework/admin.py @@ -13,6 +13,7 @@ from .models import ( AnalysisFramework, + AnalysisFrameworkTag, AnalysisFrameworkRole, AnalysisFrameworkMembership, Section, @@ -113,3 +114,8 @@ class AnalysisFrameworkRoleAdmin(admin.ModelAdmin): def has_add_permission(self, request, obj=None): return False + + +@admin.register(AnalysisFrameworkTag) +class AnalysisFrameworkTagAdmin(admin.ModelAdmin): + list_display = ('id', 'title',) diff --git a/apps/analysis_framework/dataloaders.py b/apps/analysis_framework/dataloaders.py index 61ce9ccfdb..3e60585cdd 100644 --- a/apps/analysis_framework/dataloaders.py +++ b/apps/analysis_framework/dataloaders.py @@ -12,6 +12,7 @@ Filter, Exportable, AnalysisFrameworkMembership, + AnalysisFramework, ) @@ -90,6 +91,17 @@ def batch_load_fn(self, keys): return Promise.resolve([_map[key] for key in keys]) +class AnalysisFrameworkTagsLoader(DataLoaderWithContext): + def batch_load_fn(self, keys): + qs = AnalysisFramework.tags.through.objects.filter( + analysisframework__in=keys, + ).select_related('analysisframeworktag') + _map = defaultdict(list) + for row in qs: + _map[row.analysisframework_id].append(row.analysisframeworktag) + return Promise.resolve([_map[key] for key in keys]) + + class DataLoaders(WithContextMixin): @cached_property def secondary_widgets(self): @@ -114,3 +126,7 @@ def exportables(self): @cached_property def members(self): return MembershipLoader(context=self.context) + + @cached_property + def af_tags(self): + return AnalysisFrameworkTagsLoader(context=self.context) diff --git a/apps/analysis_framework/factories.py b/apps/analysis_framework/factories.py index 1fc3f4027c..33b8552113 100644 --- a/apps/analysis_framework/factories.py +++ b/apps/analysis_framework/factories.py @@ -1,15 +1,32 @@ import factory from factory import fuzzy from factory.django import DjangoModelFactory +from django.core.files.base import ContentFile from .models import ( AnalysisFramework, + AnalysisFrameworkTag, Section, Widget, Filter, ) +class AnalysisFrameworkTagFactory(DjangoModelFactory): + title = factory.Sequence(lambda n: f'AF-Tag-{n}') + description = factory.Faker('sentence', nb_words=20) + icon = factory.LazyAttribute( + lambda n: ContentFile( + factory.django.ImageField()._make_data( + {'width': 100, 'height': 100} + ), f'example_{n.title}.png' + ) + ) + + class Meta: + model = AnalysisFrameworkTag + + class AnalysisFrameworkFactory(DjangoModelFactory): title = factory.Sequence(lambda n: f'AF-{n}') description = factory.Faker('sentence', nb_words=20) @@ -17,6 +34,14 @@ class AnalysisFrameworkFactory(DjangoModelFactory): class Meta: model = AnalysisFramework + @factory.post_generation + def tags(self, create, extracted, **_): + if not create: + return + if extracted: + for tag in extracted: + self.tags.add(tag) + class SectionFactory(DjangoModelFactory): title = factory.Sequence(lambda n: f'Section-{n}') diff --git a/apps/analysis_framework/filter_set.py b/apps/analysis_framework/filter_set.py index 6f41bea999..8186f1c798 100644 --- a/apps/analysis_framework/filter_set.py +++ b/apps/analysis_framework/filter_set.py @@ -5,8 +5,11 @@ UserResourceFilterSet, UserResourceGqlFilterSet, ) +from utils.graphene.filters import IDListFilter + from .models import ( AnalysisFramework, + AnalysisFrameworkTag, ) from entry.models import Entry from django.utils import timezone @@ -28,7 +31,22 @@ class Meta: }, } + # ----------------------------- Graphql Filters --------------------------------------- +class AnalysisFrameworkTagGqFilterSet(django_filters.FilterSet): + search = django_filters.CharFilter(method='search_filter') + + class Meta: + model = AnalysisFrameworkTag + fields = ['id'] + + def search_filter(self, qs, _, value): + if value: + return qs.filter( + models.Q(title__icontains=value) | + models.Q(description__icontains=value) + ) + return qs class AnalysisFrameworkGqFilterSet(UserResourceGqlFilterSet): @@ -39,6 +57,7 @@ class AnalysisFrameworkGqFilterSet(UserResourceGqlFilterSet): method='filter_recently_used', label='Recently Used', ) + tags = IDListFilter(distinct=True) class Meta: model = AnalysisFramework diff --git a/apps/analysis_framework/migrations/0040_auto_20231109_1208.py b/apps/analysis_framework/migrations/0040_auto_20231109_1208.py new file mode 100644 index 0000000000..1c9924582b --- /dev/null +++ b/apps/analysis_framework/migrations/0040_auto_20231109_1208.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.17 on 2023-11-09 12:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('analysis_framework', '0039_analysisframework_cloned_from'), + ] + + operations = [ + migrations.CreateModel( + name='AnalysisFrameworkTag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('description', models.TextField(blank=True)), + ('icon', models.FileField(max_length=255, upload_to='af-tag-icon/')), + ], + ), + migrations.AddField( + model_name='analysisframework', + name='tags', + field=models.ManyToManyField(blank=True, related_name='_analysis_framework_analysisframework_tags_+', to='analysis_framework.AnalysisFrameworkTag'), + ), + ] diff --git a/apps/analysis_framework/models.py b/apps/analysis_framework/models.py index bd12485d54..7c95fa99ca 100644 --- a/apps/analysis_framework/models.py +++ b/apps/analysis_framework/models.py @@ -11,6 +11,12 @@ from .widgets import store as widgets_store +class AnalysisFrameworkTag(models.Model): + title = models.CharField(max_length=255) + description = models.TextField(blank=True) + icon = models.FileField(upload_to='af-tag-icon/', max_length=255) + + class AnalysisFramework(UserResource): """ Analysis framework defining framework to do analysis @@ -19,6 +25,7 @@ class AnalysisFramework(UserResource): """ title = models.CharField(max_length=255) description = models.TextField(blank=True, null=True) + tags = models.ManyToManyField(AnalysisFrameworkTag, related_name='+', blank=True) is_private = models.BooleanField(default=False) assisted_tagging_enabled = models.BooleanField(default=False) diff --git a/apps/analysis_framework/schema.py b/apps/analysis_framework/schema.py index 54a1e38c0d..33f7a249f3 100644 --- a/apps/analysis_framework/schema.py +++ b/apps/analysis_framework/schema.py @@ -15,6 +15,7 @@ from assisted_tagging.models import PredictionTagAnalysisFrameworkWidgetMapping from .models import ( AnalysisFramework, + AnalysisFrameworkTag, Section, Widget, Filter, @@ -29,7 +30,7 @@ AnalysisFrameworkRoleTypeEnum, ) from .serializers import AnalysisFrameworkPropertiesGqlSerializer -from .filter_set import AnalysisFrameworkGqFilterSet +from .filter_set import AnalysisFrameworkGqFilterSet, AnalysisFrameworkTagGqFilterSet from .public_schema import PublicAnalysisFrameworkListType @@ -84,6 +85,18 @@ def resolve_widgets(root, info): return info.context.dl.analysis_framework.sections_widgets.load(root.id) +class AnalysisFrameworkTagType(DjangoObjectType): + class Meta: + model = AnalysisFrameworkTag + only_fields = ( + 'id', + 'title', + 'description', + ) + + icon = graphene.Field(FileFieldType, required=False) + + # NOTE: We have AnalysisFrameworkDetailType for detailed AF Type. class AnalysisFrameworkType(DjangoObjectType): class Meta: @@ -96,12 +109,19 @@ class Meta: current_user_role = graphene.Field(AnalysisFrameworkRoleTypeEnum) preview_image = graphene.Field(FileFieldType) export = graphene.Field(FileFieldType) + cloned_from = graphene.ID(source='cloned_from_id') allowed_permissions = graphene.List( graphene.NonNull( graphene.Enum.from_enum(AfP.Permission), - ), required=True + ), + required=True, + ) + tags = graphene.List( + graphene.NonNull( + AnalysisFrameworkTagType, + ), + required=True, ) - cloned_from = graphene.ID(source='cloned_from_id') @staticmethod def get_custom_node(_, info, id): @@ -123,6 +143,10 @@ def resolve_allowed_permissions(root, info): is_public=not root.is_private, ) + @staticmethod + def resolve_tags(root, info): + return info.context.dl.analysis_framework.af_tags.load(root.id) + class AnalysisFrameworkRoleType(DjangoObjectType): class Meta: @@ -251,6 +275,12 @@ class Meta: filterset_class = AnalysisFrameworkGqFilterSet +class AnalysisFrameworkTagListType(CustomDjangoListObjectType): + class Meta: + model = AnalysisFrameworkTag + filterset_class = AnalysisFrameworkTagGqFilterSet + + class Query: analysis_framework = DjangoObjectField(AnalysisFrameworkDetailType) analysis_frameworks = DjangoPaginatedListObjectField( @@ -265,6 +295,12 @@ class Query: page_size_query_param='pageSize' ) ) + analysis_framework_tags = DjangoPaginatedListObjectField( + AnalysisFrameworkTagListType, + pagination=PageGraphqlPagination( + page_size_query_param='pageSize' + ) + ) @staticmethod def resolve_analysis_frameworks(root, info, **kwargs) -> QuerySet: diff --git a/apps/analysis_framework/tests/snapshots/snap_test_schemas.py b/apps/analysis_framework/tests/snapshots/snap_test_schemas.py index 4b747b7df3..9a1b0b4cc4 100644 --- a/apps/analysis_framework/tests/snapshots/snap_test_schemas.py +++ b/apps/analysis_framework/tests/snapshots/snap_test_schemas.py @@ -344,3 +344,120 @@ } } } + +snapshots['TestAnalysisFrameworkQuery::test_analysis_framework_list response-01'] = [ + { + 'clonedFrom': None, + 'description': 'Investment on gun young catch management sense technology check civil quite others his other life edge network wall quite boy those seem shoulder future fall citizen.', + 'id': '2', + 'isPrivate': False, + 'tags': [ + { + 'description': 'Future choice whatever from behavior benefit suggest page southern role movie win her need stop peace technology officer relate animal direction eye.', + 'icon': { + 'name': 'af-tag-icon/example_AF-Tag-1.png', + 'url': 'http://testserver/media/af-tag-icon/example_AF-Tag-1.png' + }, + 'id': '2', + 'title': 'AF-Tag-1' + } + ], + 'title': 'AF-1' + }, + { + 'clonedFrom': None, + 'description': 'West then enjoy may condition tree that fear police participant check several.', + 'id': '3', + 'isPrivate': False, + 'tags': [ + ], + 'title': 'AF-2' + } +] + +snapshots['TestAnalysisFrameworkQuery::test_analysis_framework_list response-02'] = [ + { + 'clonedFrom': None, + 'description': 'Investment on gun young catch management sense technology check civil quite others his other life edge network wall quite boy those seem shoulder future fall citizen.', + 'id': '2', + 'isPrivate': False, + 'tags': [ + { + 'description': 'Future choice whatever from behavior benefit suggest page southern role movie win her need stop peace technology officer relate animal direction eye.', + 'icon': { + 'name': 'af-tag-icon/example_AF-Tag-1.png', + 'url': 'http://testserver/media/af-tag-icon/example_AF-Tag-1.png' + }, + 'id': '2', + 'title': 'AF-Tag-1' + } + ], + 'title': 'AF-1' + }, + { + 'clonedFrom': None, + 'description': 'West then enjoy may condition tree that fear police participant check several.', + 'id': '3', + 'isPrivate': False, + 'tags': [ + ], + 'title': 'AF-2' + } +] + +snapshots['TestAnalysisFrameworkQuery::test_analysis_framework_list response-03'] = [ + { + 'clonedFrom': None, + 'description': 'Here writer policy news range successful simply director allow firm environment decision wall then fire pretty how trip learn enter east no enjoy.', + 'id': '1', + 'isPrivate': True, + 'tags': [ + { + 'description': 'Each cause bill scientist nation opportunity all behavior discussion own night respond red information last everything thank serve civil.', + 'icon': { + 'name': 'af-tag-icon/example_AF-Tag-0.png', + 'url': 'http://testserver/media/af-tag-icon/example_AF-Tag-0.png' + }, + 'id': '1', + 'title': 'AF-Tag-0' + }, + { + 'description': 'Future choice whatever from behavior benefit suggest page southern role movie win her need stop peace technology officer relate animal direction eye.', + 'icon': { + 'name': 'af-tag-icon/example_AF-Tag-1.png', + 'url': 'http://testserver/media/af-tag-icon/example_AF-Tag-1.png' + }, + 'id': '2', + 'title': 'AF-Tag-1' + } + ], + 'title': 'AF-0' + }, + { + 'clonedFrom': None, + 'description': 'Investment on gun young catch management sense technology check civil quite others his other life edge network wall quite boy those seem shoulder future fall citizen.', + 'id': '2', + 'isPrivate': False, + 'tags': [ + { + 'description': 'Future choice whatever from behavior benefit suggest page southern role movie win her need stop peace technology officer relate animal direction eye.', + 'icon': { + 'name': 'af-tag-icon/example_AF-Tag-1.png', + 'url': 'http://testserver/media/af-tag-icon/example_AF-Tag-1.png' + }, + 'id': '2', + 'title': 'AF-Tag-1' + } + ], + 'title': 'AF-1' + }, + { + 'clonedFrom': None, + 'description': 'West then enjoy may condition tree that fear police participant check several.', + 'id': '3', + 'isPrivate': False, + 'tags': [ + ], + 'title': 'AF-2' + } +] diff --git a/apps/analysis_framework/tests/test_filters.py b/apps/analysis_framework/tests/test_filters.py index e605bea27f..1e34fc32f1 100644 --- a/apps/analysis_framework/tests/test_filters.py +++ b/apps/analysis_framework/tests/test_filters.py @@ -3,7 +3,7 @@ from utils.graphene.tests import GraphQLTestCase from analysis_framework.filter_set import AnalysisFrameworkGqFilterSet -from analysis_framework.factories import AnalysisFrameworkFactory +from analysis_framework.factories import AnalysisFrameworkFactory, AnalysisFrameworkTagFactory from entry.factories import EntryFactory from lead.factories import LeadFactory @@ -57,3 +57,18 @@ def test_filter_recently_used(self, now_patch): )) expected = set([af1.pk, af2.pk]) self.assertEqual(obtained, expected) + + def test_tags_filter(self): + tag1, tag2, _ = AnalysisFrameworkTagFactory.create_batch(3) + af1 = AnalysisFrameworkFactory.create(title='one', tags=[tag1]) + af2 = AnalysisFrameworkFactory.create(title='two', tags=[tag1, tag2]) + AnalysisFrameworkFactory.create(title='twoo') + for tags, expected in [ + ([tag1, tag2], [af1, af2]), + ([tag1], [af1, af2]), + ([tag2], [af2]), + ]: + obtained = self.filter_class(data=dict( + tags=[tag.id for tag in tags] + )).qs + self.assertQuerySetIdEqual(expected, obtained) diff --git a/apps/analysis_framework/tests/test_schemas.py b/apps/analysis_framework/tests/test_schemas.py index 0e80aeaaa5..21509feee2 100644 --- a/apps/analysis_framework/tests/test_schemas.py +++ b/apps/analysis_framework/tests/test_schemas.py @@ -7,13 +7,14 @@ from lead.factories import LeadFactory from analysis_framework.factories import ( AnalysisFrameworkFactory, + AnalysisFrameworkTagFactory, SectionFactory, WidgetFactory, ) class TestAnalysisFrameworkQuery(GraphQLSnapShotTestCase): - factories_used = [AnalysisFrameworkFactory, SectionFactory, WidgetFactory, ProjectFactory] + factories_used = [AnalysisFrameworkFactory, AnalysisFrameworkTagFactory, SectionFactory, WidgetFactory, ProjectFactory] def test_analysis_framework_list(self): query = ''' @@ -28,14 +29,24 @@ def test_analysis_framework_list(self): description isPrivate clonedFrom + tags { + id + title + description + icon { + url + name + } + } } } } ''' user = UserFactory.create() - private_af = AnalysisFrameworkFactory.create(is_private=True) - normal_af = AnalysisFrameworkFactory.create() + tag1, tag2, _ = AnalysisFrameworkTagFactory.create_batch(3) + private_af = AnalysisFrameworkFactory.create(is_private=True, tags=[tag1, tag2]) + normal_af = AnalysisFrameworkFactory.create(tags=[tag2]) member_af = AnalysisFrameworkFactory.create() member_af.add_member(user) @@ -51,6 +62,7 @@ def test_analysis_framework_list(self): self.assertIdEqual(results[0]['id'], normal_af.id) self.assertIdEqual(results[1]['id'], member_af.id) self.assertNotIn(str(private_af.id), [d['id'] for d in results]) # Can't see private project. + self.assertMatchSnapshot(results, 'response-01') project = ProjectFactory.create(analysis_framework=private_af) # It shouldn't list private AF after adding to a project. @@ -58,6 +70,7 @@ def test_analysis_framework_list(self): results = content['data']['analysisFrameworks']['results'] self.assertEqual(content['data']['analysisFrameworks']['totalCount'], 2) self.assertNotIn(str(private_af.id), [d['id'] for d in results]) # Can't see private project. + self.assertMatchSnapshot(results, 'response-02') project.add_member(user) # It should list private AF after user is member of the project. @@ -65,6 +78,7 @@ def test_analysis_framework_list(self): results = content['data']['analysisFrameworks']['results'] self.assertEqual(content['data']['analysisFrameworks']['totalCount'], 3) self.assertIn(str(private_af.id), [d['id'] for d in results]) # Can see private project now. + self.assertMatchSnapshot(results, 'response-03') def test_public_analysis_framework(self): query = ''' diff --git a/schema.graphql b/schema.graphql index c1175a2a7b..d48a3e7f85 100644 --- a/schema.graphql +++ b/schema.graphql @@ -47,8 +47,9 @@ type AnalysisFrameworkDetailType { currentUserRole: AnalysisFrameworkRoleTypeEnum previewImage: FileFieldType export: FileFieldType - allowedPermissions: [AnalysisFrameworkPermission!]! clonedFrom: ID + allowedPermissions: [AnalysisFrameworkPermission!]! + tags: [AnalysisFrameworkTagType!]! primaryTagging: [SectionType!] secondaryTagging: [WidgetType!] members: [AnalysisFrameworkMembershipType!] @@ -186,6 +187,20 @@ enum AnalysisFrameworkRoleTypeEnum { UNKNOWN } +type AnalysisFrameworkTagListType { + results: [AnalysisFrameworkTagType!] + totalCount: Int + page: Int + pageSize: Int +} + +type AnalysisFrameworkTagType { + id: ID! + title: String! + description: String! + icon: FileFieldType +} + type AnalysisFrameworkType { id: ID! title: String! @@ -200,8 +215,9 @@ type AnalysisFrameworkType { currentUserRole: AnalysisFrameworkRoleTypeEnum previewImage: FileFieldType export: FileFieldType - allowedPermissions: [AnalysisFrameworkPermission!]! clonedFrom: ID + allowedPermissions: [AnalysisFrameworkPermission!]! + tags: [AnalysisFrameworkTagType!]! } type AnalysisFrameworkVisibleProjectType { @@ -3746,8 +3762,9 @@ type Query { user(id: ID!): UserType users(id: Float, search: String, membersExcludeProject: ID, membersExcludeFramework: ID, membersExcludeUsergroup: ID, page: Int = 1, ordering: String, pageSize: Int): UserListType analysisFramework(id: ID!): AnalysisFrameworkDetailType - analysisFrameworks(id: Float, createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, isCurrentUserMember: Boolean, recentlyUsed: Boolean, page: Int = 1, ordering: String, pageSize: Int): AnalysisFrameworkListType - publicAnalysisFrameworks(id: Float, createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, isCurrentUserMember: Boolean, recentlyUsed: Boolean, page: Int = 1, ordering: String, pageSize: Int): PublicAnalysisFrameworkListType + analysisFrameworks(id: Float, createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, isCurrentUserMember: Boolean, recentlyUsed: Boolean, tags: [ID!], page: Int = 1, ordering: String, pageSize: Int): AnalysisFrameworkListType + publicAnalysisFrameworks(id: Float, createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], search: String, isCurrentUserMember: Boolean, recentlyUsed: Boolean, tags: [ID!], page: Int = 1, ordering: String, pageSize: Int): PublicAnalysisFrameworkListType + analysisFrameworkTags(id: Float, search: String, page: Int = 1, ordering: String, pageSize: Int): AnalysisFrameworkTagListType project(id: ID!): ProjectDetailType projects(createdAt: DateTime, createdAtGte: DateTime, createdAtLte: DateTime, modifiedAt: DateTime, modifiedAtGte: DateTime, modifiedAtLte: DateTime, createdBy: [ID!], modifiedBy: [ID!], ids: [ID!], excludeIds: [ID!], status: ProjectStatusEnum, organizations: [ID!], analysisFrameworks: [ID!], regions: [ID!], search: String, isCurrentUserMember: Boolean, hasPermissionAccess: ProjectPermission, ordering: [ProjectOrderingEnum!], isTest: Boolean, page: Int = 1, pageSize: Int): ProjectListType recentProjects: [ProjectDetailType!] From 54332026039dc52bc1d35515b68f5e6e8136d2b9 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Fri, 10 Nov 2023 11:48:48 +0545 Subject: [PATCH 04/24] Config test media location for each pytest worker --- apps/analysis_framework/models.py | 3 +++ apps/tabular/tests/test_unit.py | 3 ++- deep/settings.py | 3 ++- deep/tests/test_case.py | 12 ++++++++++++ utils/graphene/tests.py | 15 ++++----------- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/apps/analysis_framework/models.py b/apps/analysis_framework/models.py index 7c95fa99ca..88a583d120 100644 --- a/apps/analysis_framework/models.py +++ b/apps/analysis_framework/models.py @@ -16,6 +16,9 @@ class AnalysisFrameworkTag(models.Model): description = models.TextField(blank=True) icon = models.FileField(upload_to='af-tag-icon/', max_length=255) + def __str__(self): + return self.title + class AnalysisFramework(UserResource): """ diff --git a/apps/tabular/tests/test_unit.py b/apps/tabular/tests/test_unit.py index 05329dfd7b..9ee2c0a106 100644 --- a/apps/tabular/tests/test_unit.py +++ b/apps/tabular/tests/test_unit.py @@ -3,10 +3,10 @@ from tempfile import NamedTemporaryFile from deep.tests import TestCase, TEST_MEDIA_ROOT +from utils.common import makedirs from gallery.models import File from geo.models import GeoArea, Region, AdminLevel - from project.models import Project from tabular.tasks import auto_detect_and_update_fields @@ -371,6 +371,7 @@ def test_sheet_option_change_data_row_index(self): assert len(field.actual_data) == 9 def initialize_data_and_basic_test(self, csv_data): + makedirs(TEST_MEDIA_ROOT) file = NamedTemporaryFile('w', dir=TEST_MEDIA_ROOT, delete=False) self.files.append(file.name) diff --git a/deep/settings.py b/deep/settings.py index cc13ee4294..a78d952f0a 100644 --- a/deep/settings.py +++ b/deep/settings.py @@ -122,6 +122,7 @@ HTTP_PROTOCOL = env('DEEP_HTTPS') # See if we are inside a test environment (pytest) +PYTEST_XDIST_WORKER = env('PYTEST_XDIST_WORKER') TESTING = any([ arg in sys.argv for arg in [ 'test', @@ -130,7 +131,7 @@ '/usr/local/lib/python3.6/dist-packages/py/test.py', ] # Provided by pytest-xdist -]) or env('PYTEST_XDIST_WORKER') is not None +]) or PYTEST_XDIST_WORKER is not None TEST_RUNNER = 'snapshottest.django.TestRunner' TEST_DIR = os.path.join(BASE_DIR, 'deep/test_files') diff --git a/deep/tests/test_case.py b/deep/tests/test_case.py index 7cae8e650b..f9339948db 100644 --- a/deep/tests/test_case.py +++ b/deep/tests/test_case.py @@ -1,4 +1,5 @@ import os +import shutil import autofixture from rest_framework import ( test, @@ -22,6 +23,9 @@ TEST_MEDIA_ROOT = 'rest-media-temp' +if settings.PYTEST_XDIST_WORKER: + TEST_MEDIA_ROOT = f'rest-media-temp/{settings.PYTEST_XDIST_WORKER}' + TEST_EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' TEST_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' TEST_CACHES = { @@ -48,6 +52,13 @@ TEST_AUTH_PASSWORD_VALIDATORS = [] +def clean_up_test_media_files(path): + try: + shutil.rmtree(path, ignore_errors=True) + except FileNotFoundError: + pass + + @override_settings( DEBUG=True, EMAIL_BACKEND=TEST_EMAIL_BACKEND, @@ -86,6 +97,7 @@ def setUp(self): def tearDown(self): super().tearDown() _set_middleware_current_request(None) + clean_up_test_media_files(os.path.join(settings.BASE_DIR, TEST_MEDIA_ROOT)) for file_path in self.deep_test_files_path: if os.path.isfile(file_path): os.unlink(file_path) diff --git a/utils/graphene/tests.py b/utils/graphene/tests.py index 861ac6ec70..da45c12c2c 100644 --- a/utils/graphene/tests.py +++ b/utils/graphene/tests.py @@ -1,7 +1,6 @@ import os import json import pytz -import shutil import inspect import datetime from enum import Enum @@ -27,6 +26,7 @@ TEST_AUTH_PASSWORD_VALIDATORS, TEST_EMAIL_BACKEND, TEST_FILE_STORAGE, + clean_up_test_media_files, ) from analysis_framework.models import AnalysisFramework, AnalysisFrameworkRole @@ -35,15 +35,8 @@ User = get_user_model() TEST_MEDIA_ROOT = 'media-temp' - - -def clean_up_test_media_files(): - try: - # NOTE: CI will clean itself - if os.environ.get('CI', '').lower() != 'true': - shutil.rmtree(os.path.join(settings.BASE_DIR, TEST_MEDIA_ROOT), ignore_errors=True) - except FileNotFoundError: - pass +if settings.PYTEST_XDIST_WORKER: + TEST_MEDIA_ROOT = f'media-temp/{settings.PYTEST_XDIST_WORKER}' @override_settings( @@ -66,7 +59,7 @@ class GraphQLTestCase(BaseGraphQLTestCase): @classmethod def tearDownClass(cls): # clear the temporary media files - clean_up_test_media_files() + clean_up_test_media_files(os.path.join(settings.BASE_DIR, TEST_MEDIA_ROOT)) super().tearDownClass() def _setup_premailer_patcher(self, mock): From 48bcf20ed4386ed4ba703e18606fecf06863d2eb Mon Sep 17 00:00:00 2001 From: thenav56 Date: Tue, 28 Nov 2023 11:39:47 +0545 Subject: [PATCH 05/24] Add LeadFactory for clean in AF Schema test --- apps/analysis_framework/tests/test_schemas.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/apps/analysis_framework/tests/test_schemas.py b/apps/analysis_framework/tests/test_schemas.py index 21509feee2..fae1c32179 100644 --- a/apps/analysis_framework/tests/test_schemas.py +++ b/apps/analysis_framework/tests/test_schemas.py @@ -14,7 +14,14 @@ class TestAnalysisFrameworkQuery(GraphQLSnapShotTestCase): - factories_used = [AnalysisFrameworkFactory, AnalysisFrameworkTagFactory, SectionFactory, WidgetFactory, ProjectFactory] + factories_used = [ + AnalysisFrameworkFactory, + AnalysisFrameworkTagFactory, + SectionFactory, + WidgetFactory, + ProjectFactory, + LeadFactory, + ] def test_analysis_framework_list(self): query = ''' @@ -260,12 +267,14 @@ def test_recent_analysis_framework(self): ''' # lets create some analysis_framework - analysis_framework1 = AnalysisFrameworkFactory.create() - analysis_framework2 = AnalysisFrameworkFactory.create() - analysis_framework3 = AnalysisFrameworkFactory.create() - analysis_framework4 = AnalysisFrameworkFactory.create() - analysis_framework5 = AnalysisFrameworkFactory.create() - analysis_framework6 = AnalysisFrameworkFactory.create() + ( + analysis_framework1, + analysis_framework2, + analysis_framework3, + analysis_framework4, + analysis_framework5, + analysis_framework6, + ) = AnalysisFrameworkFactory.create_batch(6) project1 = ProjectFactory.create(analysis_framework=analysis_framework1) project2 = ProjectFactory.create(analysis_framework=analysis_framework2) From 400e49c791df072bbfdf41e89e755ef89ee95267 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Mon, 20 Nov 2023 11:11:47 +0545 Subject: [PATCH 06/24] Auto extraction entry mock data added draftentry --- apps/assisted_tagging/admin.py | 31 + apps/assisted_tagging/enums.py | 9 + apps/assisted_tagging/filters.py | 32 +- .../0011_draftentry_draft_entry_type.py | 18 + apps/assisted_tagging/models.py | 5 + apps/assisted_tagging/mutation.py | 18 + apps/assisted_tagging/schema.py | 80 ++- apps/assisted_tagging/serializers.py | 44 +- apps/assisted_tagging/tasks.py | 10 +- apps/deepl_integration/handlers.py | 192 +++++- apps/deepl_integration/serializers.py | 19 + apps/deepl_integration/views.py | 15 + .../migrations/0049_auto_20231121_0926.py | 41 ++ .../migrations/0050_delete_extractedlead.py | 16 + apps/lead/models.py | 8 + deep/deepl.py | 3 + deep/test_files/mock_draftentry.json | 567 ++++++++++++++++++ deep/urls.py | 6 + schema.graphql | 44 ++ 19 files changed, 1152 insertions(+), 6 deletions(-) create mode 100644 apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type.py create mode 100644 apps/lead/migrations/0049_auto_20231121_0926.py create mode 100644 apps/lead/migrations/0050_delete_extractedlead.py create mode 100644 deep/test_files/mock_draftentry.json diff --git a/apps/assisted_tagging/admin.py b/apps/assisted_tagging/admin.py index 846f6b4061..0348a0e19a 100644 --- a/apps/assisted_tagging/admin.py +++ b/apps/assisted_tagging/admin.py @@ -1 +1,32 @@ # Register your models here. +from django.contrib import admin + +from assisted_tagging.models import AssistedTaggingModelPredictionTag, AssistedTaggingPrediction, DraftEntry +from deep.admin import VersionAdmin + + +@admin.register(DraftEntry) +class DraftEntryAdmin(VersionAdmin): + list_display = [ + 'lead', + 'prediction_status', + 'excerpt', + ] + + +@admin.register(AssistedTaggingPrediction) +class AssistedTaggingPredictionAdmin(VersionAdmin): + list_display = [ + "data_type", + "draft_entry", + "value" + ] + + +@admin.register(AssistedTaggingModelPredictionTag) +class AssistedTaggingModelPredictionTagAdmin(VersionAdmin): + list_display = [ + 'name', + 'is_category', + 'tag_id', + ] diff --git a/apps/assisted_tagging/enums.py b/apps/assisted_tagging/enums.py index 996c7629e9..2bd8a1f0c2 100644 --- a/apps/assisted_tagging/enums.py +++ b/apps/assisted_tagging/enums.py @@ -9,17 +9,26 @@ DraftEntry, AssistedTaggingPrediction, ) +from lead.models import Lead DraftEntryPredictionStatusEnum = convert_enum_to_graphene_enum( DraftEntry.PredictionStatus, name='DraftEntryPredictionStatusEnum') AssistedTaggingPredictionDataTypeEnum = convert_enum_to_graphene_enum( AssistedTaggingPrediction.DataType, name='AssistedTaggingPredictionDataTypeEnum') +DraftEntryTypeEnum = convert_enum_to_graphene_enum( + DraftEntry.DraftEntryType, name="DraftEntryTypeEnum" +) +AutoEntryExtractionTypeEnum = convert_enum_to_graphene_enum( + Lead.AutoExtractionStatus, name="AutoEntryExtractionTypeEnum" +) enum_map = { get_enum_name_from_django_field(field): enum for field, enum in ( (DraftEntry.prediction_status, DraftEntryPredictionStatusEnum), (AssistedTaggingPrediction.data_type, AssistedTaggingPredictionDataTypeEnum), + (DraftEntry.draft_entry_type, DraftEntryTypeEnum), + (Lead.auto_entry_extraction_status, AutoEntryExtractionTypeEnum), ) } diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index 792d600548..277fac72ae 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -1 +1,31 @@ -# +import django_filters + +from deep.filter_set import generate_type_for_filter_set +from .models import DraftEntry +from utils.graphene.filters import IDListFilter, MultipleInputFilter +from .enums import ( + DraftEntryTypeEnum +) + + +class DraftEntryFilterSet(django_filters.FilterSet): + lead = IDListFilter() + draft_entry_type = MultipleInputFilter(DraftEntryTypeEnum) + + class Meta: + model = DraftEntry + fields = () + + def filter_lead(self, qs, _, value): + return qs if value is None else qs.filter(lead=value) + + def filter_draft_entry_type(self, qs, _, value): + return qs if value is None else qs.filter(draft_entry_type=value) + + +DraftEntryFilterDataType, DraftEntryFilterDataInputType = generate_type_for_filter_set( + DraftEntryFilterSet, + "project.schema.ProjectListType", + "DraftEntryFilterDataType", + "DraftEntryFilterDataInputType", +) diff --git a/apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type.py b/apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type.py new file mode 100644 index 0000000000..0fec29e96d --- /dev/null +++ b/apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-11-06 10:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assisted_tagging', '0010_draftentry_related_geoareas'), + ] + + operations = [ + migrations.AddField( + model_name='draftentry', + name='draft_entry_type', + field=models.SmallIntegerField(choices=[(0, 'Auto Extraction'), (1, 'Manual Extraction')], default=1), + ), + ] diff --git a/apps/assisted_tagging/models.py b/apps/assisted_tagging/models.py index 2464ab5b0e..fb02a1f931 100644 --- a/apps/assisted_tagging/models.py +++ b/apps/assisted_tagging/models.py @@ -90,6 +90,10 @@ class PredictionStatus(models.IntegerChoices): DONE = 2, 'Done' SEND_FAILED = 3, 'Send Failed' + class DraftEntryType(models.IntegerChoices): + AUTO = 0, 'Auto Extraction' # NLP defiend extraction text + MANUAL = 1, 'Manual Extraction' # manaul defiend extraction text + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='+') lead = models.ForeignKey(Lead, on_delete=models.CASCADE, related_name='+') excerpt = models.TextField() @@ -98,6 +102,7 @@ class PredictionStatus(models.IntegerChoices): prediction_received_at = models.DateTimeField(null=True, blank=True) # Additional attribues related_geoareas = models.ManyToManyField(GeoArea, blank=True) + draft_entry_type = models.SmallIntegerField(choices=DraftEntryType.choices, default=DraftEntryType.MANUAL) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/apps/assisted_tagging/mutation.py b/apps/assisted_tagging/mutation.py index f47c80952b..e6568d34ae 100644 --- a/apps/assisted_tagging/mutation.py +++ b/apps/assisted_tagging/mutation.py @@ -21,6 +21,7 @@ DraftEntryGqlSerializer, WrongPredictionReviewGqlSerializer, MissingPredictionReviewGqlSerializer, + AutoDraftEntryGqlSerializer ) @@ -39,6 +40,11 @@ serializer_class=MissingPredictionReviewGqlSerializer, ) +AutoDraftEntryInputType = generate_input_type_for_serializer( + "AutoDraftEntryInputType", + serializer_class=AutoDraftEntryGqlSerializer +) + class CreateDraftEntry(PsGrapheneMutation): class Arguments: @@ -96,6 +102,17 @@ def filter_queryset(cls, qs, info): created_by=info.context.user, ) +# auto draft_entry_create + + +class CreateAutoDraftEntry(PsGrapheneMutation): + class Arguments: + data = AutoDraftEntryInputType(required=True) + model = DraftEntry + serializer_class = AutoDraftEntryGqlSerializer + result = graphene.Field(DraftEntryType) + permissions = [PP.Permission.CREATE_ENTRY] + class AssistedTaggingMutationType(graphene.ObjectType): draft_entry_create = CreateDraftEntry.Field() @@ -103,3 +120,4 @@ class AssistedTaggingMutationType(graphene.ObjectType): wrong_prediction_review_create = CreateWrongPredictionReview.Field() missing_prediction_review_delete = DeleteMissingPredictionReview.Field() wrong_prediction_review_delete = DeleteWrongPredictionReview.Field() + auto_draft_entry_create = CreateAutoDraftEntry.Field() diff --git a/apps/assisted_tagging/schema.py b/apps/assisted_tagging/schema.py index f6ae0f23b8..fd926dcb4c 100644 --- a/apps/assisted_tagging/schema.py +++ b/apps/assisted_tagging/schema.py @@ -1,7 +1,8 @@ import graphene -from graphene_django import DjangoObjectType +from graphene_django import DjangoObjectType, DjangoListField from graphene_django_extras import DjangoObjectField from django.db.models import Prefetch +from assisted_tagging.filters import DraftEntryFilterDataInputType, DraftEntryFilterSet from utils.graphene.enums import EnumDescription from user_resource.schema import UserResourceMixin @@ -20,9 +21,11 @@ MissingPredictionReview, WrongPredictionReview, ) +from lead.models import Lead from .enums import ( DraftEntryPredictionStatusEnum, AssistedTaggingPredictionDataTypeEnum, + AutoEntryExtractionTypeEnum ) @@ -103,6 +106,13 @@ def get_draft_entry_qs(info): return qs.none() +def get_draft_entry_with_filter_qs(info, filters): + qs = DraftEntry.objects.filter(project=info.context.active_project) + if PP.check_permission(info, PP.Permission.VIEW_ENTRY): + return DraftEntryFilterSet(queryset=qs, data=filters).qs + return qs.none() + + class WrongPredictionReviewType(UserResourceMixin, DjangoObjectType): prediction = graphene.ID(source='prediction_id', required=True) @@ -208,6 +218,74 @@ def resolve_related_geoareas(root, info, **kwargs): return root.related_geoareas.all() # NOTE: Prefetched by DraftEntry +class DraftEntryByLeadType(DjangoObjectType): + prediction_status = graphene.Field(DraftEntryPredictionStatusEnum, required=True) + prediction_status_display = EnumDescription(source='get_prediction_status_display', required=True) + predictions = graphene.List( + graphene.NonNull(AssistedTaggingPredictionType) + ) + missing_prediction_reviews = graphene.List( + graphene.NonNull(MissingPredictionReviewType), + ) + related_geoareas = graphene.List( + graphene.NonNull(ProjectGeoAreaType) + ) + + class Meta: + model = DraftEntry + only_fields = ( + 'id', + 'excerpt', + 'prediction_received_at', + ) + + @staticmethod + def custom_queryset(queryset, info, _filter, **kwargs): + return get_draft_entry_with_filter_qs(info, _filter).prefetch_related( + Prefetch( + 'predictions', + queryset=AssistedTaggingPrediction.objects.order_by('id'), + ), + Prefetch( + 'related_geoareas', + queryset=get_geo_area_queryset_for_project_geo_area_type().order_by('id'), + ), + 'predictions__model_version', + 'predictions__model_version__model', + 'predictions__wrong_prediction_reviews', + 'missing_prediction_reviews', + 'related_geoareas', + ) + + @staticmethod + def resolve_predictions(root, info, **kwargs): + return root.predictions.filter(is_selected=True) # NOTE: Prefetched by DraftEntry + + @staticmethod + def resolve_missing_prediction_reviews(root, info, **kwargs): + return root.missing_prediction_reviews.all() # NOTE: Prefetched by DraftEntry + + @staticmethod + def resolve_related_geoareas(root, info, **kwargs): + return root.related_geoareas.all() + + +class AutoextractionStatusType(graphene.ObjectType): + auto_entry_extraction_status = graphene.Field(AutoEntryExtractionTypeEnum, required=True) + + @staticmethod + def custom_queryset(root, info, lead_id): + return Lead.objects.get(id=lead_id) # This is attached to project type. + + class AssistedTaggingQueryType(graphene.ObjectType): draft_entry = DjangoObjectField(DraftEntryType) + draft_entry_by_leads = DjangoListField(DraftEntryByLeadType, filter=DraftEntryFilterDataInputType()) + extraction_status_by_lead = graphene.Field(AutoextractionStatusType,lead_id=graphene.Int(required=True)) + + def resolve_draft_entry_by_leads(root, info, filter): + return DraftEntryByLeadType.custom_queryset(root, info, filter) + + def resolve_extraction_status_by_lead(root, info, lead_id): + return AutoextractionStatusType.custom_queryset(root, info,lead_id) diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index 3dcdb4d82c..34fe201474 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -1,6 +1,5 @@ from django.db import transaction from rest_framework import serializers - from user_resource.serializers import UserResourceSerializer, UserResourceCreatedMixin from deep.serializers import ProjectPropertySerializerMixin, TempClientIdMixin from analysis_framework.models import Widget @@ -11,7 +10,8 @@ PredictionTagAnalysisFrameworkWidgetMapping, WrongPredictionReview, ) -from .tasks import trigger_request_for_draft_entry_task +from lead.models import LeadPreview +from .tasks import trigger_request_for_draft_entry_task, trigger_request_for_mock_entry_task # ---------- Graphql --------------------------- @@ -127,3 +127,43 @@ def validate(self, data): association='Association is required for this widget.' )) return data + + +class AutoDraftEntryGqlSerializer(ProjectPropertySerializerMixin, UserResourceCreatedMixin, serializers.ModelSerializer): + class Meta: + model = DraftEntry + fields = ( + 'lead', + ) + + def validate_lead(self, lead): + if lead.project != self.project: + raise serializers.ValidationError('Only lead from current project are allowed.') + af = lead.project.analysis_framework + if af is None or not af.assisted_tagging_enabled: + raise serializers.ValidationError('Assisted tagging is disabled for the Framework used by this project.') + return lead + + def validate(self, data): + if self.instance and self.instance.created_by != self.context['request'].user: + raise serializers.ValidationError('Only reviewer can edit this review') + data['project'] = self.project + if self.project.is_private: + raise serializers.ValidationError('Assisted tagging is not available for private projects.') + return data + + def create(self, data): + text = LeadPreview.objects.filter(lead=self.data['lead']).first() + if text.text_extract == '' or None: + raise serializers.DjangoValidationError('Simplifed Text is empty') + # Use already existing draft entry if found + data['draft_entry_type'] = 0 # auto extraction + # Create new one and send trigger to deepl + transaction.on_commit( + lambda: trigger_request_for_mock_entry_task.delay(self.data['lead']) + ) + + return True + + def update(self, *_): + raise Exception('Update not allowed') diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py index e274a47334..ed9afed588 100644 --- a/apps/assisted_tagging/tasks.py +++ b/apps/assisted_tagging/tasks.py @@ -2,10 +2,11 @@ import requests from celery import shared_task +from lead.models import Lead from utils.common import redis_lock from deep.deepl import DeeplServiceEndpoint -from deepl_integration.handlers import AssistedTaggingDraftEntryHandler +from deepl_integration.handlers import AssistedTaggingDraftEntryHandler, AutoAssistedTaggingDraftEntryHandler from .models import ( DraftEntry, @@ -92,6 +93,13 @@ def trigger_request_for_draft_entry_task(draft_entry_id): return AssistedTaggingDraftEntryHandler.send_trigger_request_to_extractor(draft_entry) +@shared_task +@redis_lock('trigger_request_for_mock_entry_task_{0}', 60 * 60 * 0.5) +def trigger_request_for_mock_entry_task(lead): + lead = Lead.objects.get(id=lead) + return AutoAssistedTaggingDraftEntryHandler.auto_trigger_request_to_extractor(lead) + + @shared_task @redis_lock('sync_tags_with_deepl_task', 60 * 60 * 0.5) def sync_tags_with_deepl_task(): diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index e5991df3e1..e4b8b92608 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -203,7 +203,7 @@ def send_trigger_request_to_extractor(cls, draft_entry): }, ) - # --- Callback logics +# --- Callback logics @staticmethod def _get_or_create_models_version(models_data): def get_versions_map(): @@ -348,6 +348,196 @@ def save_data(cls, draft_entry, data): return draft_entry +class AutoAssistedTaggingDraftEntryHandler(BaseHandler): + model = Lead + callback_url_name = 'auto-assisted_tagging_draft_entry_prediction_callback' + + @classmethod + def auto_trigger_request_to_extractor(cls, lead): + payload = { + "documents": [ + { + "client_id": cls.get_client_id(lead), # static clientid for mock + "text_extraction_id": "43545" # static text_extraction id + } + ], + "callback_url": cls.get_callback_url() + } + logger.error(payload) + try: + response = requests.post( + url=DeeplServiceEndpoint.ENTRY_EXTRACTION_CLASSIFICATION, + headers=cls.REQUEST_HEADERS, + json=payload + ) + logger.error(response.status_code) + if response.status_code == 202: + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING + lead.save() + return True + + except Exception: + logger.error('Entry Extraction send failed, Exception occurred!!', exc_info=True) + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.FAILED + lead.save() + _response = locals().get('response') + logger.error( + 'Entry Extraction send failed!!', + extra={ + 'payload': payload, + 'response': _response.content if _response else None + }, + ) + + @staticmethod + def _get_or_create_models_version(models_data): + def get_versions_map(): + return { + (model_version.model.model_id, model_version.version): model_version + for model_version in AssistedTaggingModelVersion.objects.filter( + reduce( + lambda acc, item: acc | item, + [ + models.Q( + model__model_id=model_data['id'], + version=model_data['version'], + ) + for model_data in models_data + ], + ) + ).select_related('model').all() + } + + existing_model_versions = get_versions_map() + new_model_versions = [ + model_data + for model_data in models_data + if (model_data['id'], model_data['version']) not in existing_model_versions + ] + + if new_model_versions: + AssistedTaggingModelVersion.objects.bulk_create([ + AssistedTaggingModelVersion( + model=AssistedTaggingModel.objects.get_or_create( + model_id=model_data['id'], + defaults=dict( + name=model_data['id'], + ), + )[0], + version=model_data['version'], + ) + for model_data in models_data + ]) + existing_model_versions = get_versions_map() + return existing_model_versions + + @classmethod + def _get_or_create_tags_map(cls, tags): + from assisted_tagging.tasks import sync_tags_with_deepl_task + + def get_tags_map(): + return { + tag_id: _id + for _id, tag_id in AssistedTaggingModelPredictionTag.objects.values_list('id', 'tag_id') + } + + current_tags_map = get_tags_map() + # Check if new tags needs to be created + new_tags = [ + tag for tag in tags + if tag not in current_tags_map + ] + if new_tags: + # Create new tags + AssistedTaggingModelPredictionTag.objects.bulk_create([ + AssistedTaggingModelPredictionTag( + name=new_tag, + tag_id=new_tag, + ) + for new_tag in new_tags + ]) + # Refetch + current_tags_map = get_tags_map() + sync_tags_with_deepl_task.delay() + return current_tags_map + + @classmethod + def _process_model_preds(cls, model_version, current_tags_map, draft_entry, model_prediction): + prediction_status = model_prediction['prediction_status'] + if prediction_status == 0: # If 0 no tags are provided + return + + tags = model_prediction.get('tags', {}) # NLP TagId + values = model_prediction.get('values', []) # Raw value + + common_attrs = dict( + model_version=model_version, + draft_entry_id=draft_entry.id, + ) + new_predictions = [] + for category_tag, tags in tags.items(): + for tag, prediction_data in tags.items(): + prediction_value = prediction_data.get('prediction') + threshold_value = prediction_data.get('threshold') + is_selected = prediction_data.get('is_selected', False) + new_predictions.append( + AssistedTaggingPrediction( + **common_attrs, + data_type=AssistedTaggingPrediction.DataType.TAG, + category_id=current_tags_map[category_tag], + tag_id=current_tags_map[tag], + prediction=prediction_value, + threshold=threshold_value, + is_selected=is_selected, + ) + ) + + for value in set(values): + new_predictions.append( + AssistedTaggingPrediction( + **common_attrs, + data_type=AssistedTaggingPrediction.DataType.RAW, + value=value, + is_selected=True, + ) + ) + AssistedTaggingPrediction.objects.bulk_create(new_predictions) + + @classmethod + def save_data(cls, lead, data): + for model_preds in data['blocks']: + classification = model_preds['classification'] + current_tags_map = cls._get_or_create_tags_map([ + tag + for prediction in classification['model_preds'] + for category_tag, tags in prediction.get('tags', {}).items() + for tag in [ + category_tag, + *tags.keys(), + ] + ]) + models_version_map = cls._get_or_create_models_version([ + prediction['model_info'] + for prediction in classification['model_preds'] + ]) + + with transaction.atomic(): + draft = DraftEntry.objects.create( + project=lead.project, + lead=lead, + excerpt=model_preds['text'], + draft_entry_type=0 + ) + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS + lead.save() + draft.save() + for prediction in classification['model_preds']: + model_version = models_version_map[(prediction['model_info']['id'], prediction['model_info']['version'])] + cls._process_model_preds(model_version, current_tags_map, draft, prediction) + + return lead + + class LeadExtractionHandler(BaseHandler): model = Lead callback_url_name = 'lead_extract_callback' diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 6b263d2004..94499ad618 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -12,6 +12,7 @@ AnalysisAutomaticSummaryHandler, AnalyticalStatementNGramHandler, AnalyticalStatementGeoHandler, + AutoAssistedTaggingDraftEntryHandler ) from deduplication.tasks.indexing import index_lead_and_calculate_duplicates @@ -206,6 +207,24 @@ def create(self, validated_data): ) +class AutoAssistedBlockPredicationCallbackSerializer(serializers.Serializer): + class ClassificationInfoCallBackSerializer(serializers.Serializer): + model_preds = AssistedTaggingModelPredictionCallbackSerializer(many=True) + text = serializers.CharField() + classification = ClassificationInfoCallBackSerializer() + + +class AutoAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): + blocks = AutoAssistedBlockPredicationCallbackSerializer(many=True) + nlp_handler = AutoAssistedTaggingDraftEntryHandler + + def create(self, validated_data): + return self.nlp_handler.save_data( + validated_data['object'], + validated_data + ) + + class EntriesCollectionBaseCallbackSerializer(DeeplServerBaseCallbackSerializer): model: Type[DeeplTrackBaseModel] presigned_s3_url = serializers.URLField() diff --git a/apps/deepl_integration/views.py b/apps/deepl_integration/views.py index d6163d0931..a33112d37b 100644 --- a/apps/deepl_integration/views.py +++ b/apps/deepl_integration/views.py @@ -15,8 +15,11 @@ AnalysisAutomaticSummaryCallbackSerializer, AnalyticalStatementNGramCallbackSerializer, AnalyticalStatementGeoCallbackSerializer, + AutoAssistedTaggingDraftEntryCallbackSerializer ) +from utils.request import RequestHelper + class BaseCallbackView(views.APIView): serializer: Type[serializers.Serializer] @@ -33,6 +36,18 @@ class AssistedTaggingDraftEntryPredictionCallbackView(BaseCallbackView): serializer = AssistedTaggingDraftEntryPredictionCallbackSerializer +class AutoTaggingDraftEntryPredictionCallbackView(views.APIView): + serializer = AutoAssistedTaggingDraftEntryCallbackSerializer + permission_classes = [permissions.AllowAny] + + def post(self, request, **_): + data = RequestHelper(url=request.data['entry_extraction_classification_path'], ignore_error=True).json() + serializer = self.serializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save() + return response.Response("Request successfully completed", status=status.HTTP_200_OK) + + class LeadExtractCallbackView(BaseCallbackView): serializer = LeadExtractCallbackSerializer diff --git a/apps/lead/migrations/0049_auto_20231121_0926.py b/apps/lead/migrations/0049_auto_20231121_0926.py new file mode 100644 index 0000000000..5957bbf49b --- /dev/null +++ b/apps/lead/migrations/0049_auto_20231121_0926.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.17 on 2023-11-21 09:26 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('lead', '0048_auto_20230228_0810'), + ] + + operations = [ + migrations.AddField( + model_name='lead', + name='auto_entry_extraction_status', + field=models.SmallIntegerField(choices=[(0, 'Pending'), (1, 'Success'), (2, 'Failed')], default=0), + ), + migrations.CreateModel( + name='ExtractedLead', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('client_id', models.CharField(blank=True, default=None, max_length=128, null=True, unique=True)), + ('entry_extraction_classification', models.TextField()), + ('text_extraction_id', models.CharField(blank=True, max_length=50, null=True)), + ('text_extraction_status', models.IntegerField(choices=[(1, 'Initiated'), (2, 'Success'), (3, 'Failed'), (4, 'Input url process failed')], default=1)), + ('text_classification_status', models.IntegerField(choices=[(1, 'Initiated'), (2, 'Success'), (3, 'Failed'), (4, 'Input url process failed')], default=1)), + ('created_by', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='extractedlead_created', to=settings.AUTH_USER_MODEL)), + ('lead', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='lead.lead')), + ('modified_by', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='extractedlead_modified', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + 'abstract': False, + }, + ), + ] diff --git a/apps/lead/migrations/0050_delete_extractedlead.py b/apps/lead/migrations/0050_delete_extractedlead.py new file mode 100644 index 0000000000..167a9a2851 --- /dev/null +++ b/apps/lead/migrations/0050_delete_extractedlead.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.17 on 2023-11-23 04:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0049_auto_20231121_0926'), + ] + + operations = [ + migrations.DeleteModel( + name='ExtractedLead', + ), + ] diff --git a/apps/lead/models.py b/apps/lead/models.py index c17c8bd918..c241693dde 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -89,6 +89,12 @@ class ExtractionStatus(models.IntegerChoices): SUCCESS = 2, 'Success' FAILED = 3, 'Failed' + class AutoExtractionStatus(models.IntegerChoices): + NONE = 0, "None" + PENDING = 1, 'Pending' + SUCCESS = 2, 'Success' + FAILED = 3, 'Failed' + lead_group = models.ForeignKey( LeadGroup, on_delete=models.SET_NULL, @@ -148,6 +154,8 @@ class ExtractionStatus(models.IntegerChoices): is_indexed = models.BooleanField(default=False) duplicate_leads_count = models.PositiveIntegerField(default=0) indexed_at = models.DateTimeField(null=True, blank=True) + auto_entry_extraction_status = models.SmallIntegerField( + choices=AutoExtractionStatus.choices, default=AutoExtractionStatus.NONE) def __str__(self): return '{}'.format(self.title) diff --git a/deep/deepl.py b/deep/deepl.py index 25dd329f53..7064fb6676 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -18,3 +18,6 @@ class DeeplServiceEndpoint(): ANALYSIS_AUTOMATIC_SUMMARY = f'{DEEPL_SERVER_DOMAIN}/api/v1/summarization/' ANALYSIS_AUTOMATIC_NGRAM = f'{DEEPL_SERVER_DOMAIN}/api/v1/ngrams/' ANALYSIS_GEO = f'{DEEPL_SERVER_DOMAIN}/api/v1/geolocation/' + + # AutoExtraction DeepL endpoint + ENTRY_EXTRACTION_CLASSIFICATION = f'{DEEPL_SERVER_DOMAIN}/api/v1/entry-extraction-classification/' diff --git a/deep/test_files/mock_draftentry.json b/deep/test_files/mock_draftentry.json new file mode 100644 index 0000000000..cde285400d --- /dev/null +++ b/deep/test_files/mock_draftentry.json @@ -0,0 +1,567 @@ +{ + "metadata": { + "total_pages": 10, + "total_words_count": 5876, + "title": "AI in humanitarian domain", + "author": "D Harvey", + "keywords": [ + "health", + "medical treatments" + ], + "format": "PDF 1.4" + }, + "blocks": [ + { + "type": "text", + "page": 1, + "x0": 0, + "y0": 63.453, + "x1": 545.23, + "y1": 106.23, + "text": "The 2021 Gu rainy season performance varied across Somalia with many places recording average to below average rainfall (Maps 1 & 2, and Annex I). The seasonal rains which started in late April lasted for three weeks and came to an early end during the first week of May 2021. During the three weeks of rainfall, some places recorded heavy rains that led to flash floods in the northern parts of the country. The southern regions recorded below normal seasonal rains, leaving many places under water stress. This follows another poor rainfall performance during the 2020 Deyr (October- December) season which led to moderate drought conditions this year that lasted till late April", + "textOrder": 1, + "textCrop": [ + 36, + 135.47, + 321.3, + 295.97 + ], + "relevant": true, + "classification": { + "model_preds": [ + { + "tags": { + "1": { + "101": { + "prediction": 0.0000270270529, + "threshold": 0.14, + "is_selected": false + }, + "102": { + "prediction": 2.791275697595933, + "threshold": 0.17, + "is_selected": true + }, + "103": { + "prediction": 0.000845505346661, + "threshold": 0.1, + "is_selected": false + }, + "104": { + "prediction": 0.001551844096476, + "threshold": 0.14, + "is_selected": false + }, + "105": { + "prediction": 0.000610130882706, + "threshold": 0.18, + "is_selected": false + }, + "106": { + "prediction": 0.021222406732185, + "threshold": 0.14, + "is_selected": false + }, + "107": { + "prediction": 0.000047710691433, + "threshold": 0.1, + "is_selected": false + }, + "108": { + "prediction": 0.000005902628667, + "threshold": 0.12, + "is_selected": false + }, + "109": { + "prediction": 5.845728317896525, + "threshold": 0.15, + "is_selected": true + }, + "110": { + "prediction": 0.000993687879398, + "threshold": 0.18, + "is_selected": false + }, + "111": { + "prediction": 0.000130282686379, + "threshold": 0.14, + "is_selected": false + } + }, + "2": { + "201": { + "prediction": 0.001077209547956, + "threshold": 0.17, + "is_selected": false + }, + "202": { + "prediction": 0.002082403516397, + "threshold": 0.15, + "is_selected": false + }, + "203": { + "prediction": 0.027398668074359, + "threshold": 0.24, + "is_selected": false + }, + "204": { + "prediction": 0.000344154328299, + "threshold": 0.14, + "is_selected": false + }, + "205": { + "prediction": 0.008931630191968, + "threshold": 0.47, + "is_selected": false + }, + "206": { + "prediction": 1.561535708606243, + "threshold": 0.16, + "is_selected": true + }, + "207": { + "prediction": 0.008022336987779, + "threshold": 0.22, + "is_selected": false + }, + "208": { + "prediction": 0.000339151683008, + "threshold": 0.21, + "is_selected": false + }, + "209": { + "prediction": 0.664219930768013, + "threshold": 0.05, + "is_selected": false + }, + "210": { + "prediction": 0.070768408477306, + "threshold": 0.24, + "is_selected": false + }, + "212": { + "prediction": 0.002422974061882, + "threshold": 0.31, + "is_selected": false + }, + "213": { + "prediction": 0.000046979557038, + "threshold": 0.26, + "is_selected": false + }, + "214": { + "prediction": 0.000070475855157, + "threshold": 0.09, + "is_selected": false + }, + "215": { + "prediction": 0.000008547227329, + "threshold": 0.15, + "is_selected": false + }, + "216": { + "prediction": 0.000167800135387, + "threshold": 0.13, + "is_selected": false + }, + "217": { + "prediction": 0.000016116082691, + "threshold": 0.04, + "is_selected": false + }, + "218": { + "prediction": 0.00002616270649, + "threshold": 0.09, + "is_selected": false + }, + "219": { + "prediction": 0.000166147779405, + "threshold": 0.13, + "is_selected": false + }, + "220": { + "prediction": 0.000002293435324, + "threshold": 0.22, + "is_selected": false + }, + "221": { + "prediction": 2.3751352e-7, + "threshold": 0.16, + "is_selected": false + }, + "222": { + "prediction": 0.000008183459954, + "threshold": 0.16, + "is_selected": false + }, + "223": { + "prediction": 0.000764200214892, + "threshold": 0.09, + "is_selected": false + }, + "224": { + "prediction": 7.88740062e-7, + "threshold": 0.21, + "is_selected": false + }, + "225": { + "prediction": 0.000002737477267, + "threshold": 0.04, + "is_selected": false + }, + "226": { + "prediction": 3.95147303e-7, + "threshold": 0.09, + "is_selected": false + }, + "227": { + "prediction": 0.000015625468157, + "threshold": 0.07, + "is_selected": false + }, + "228": { + "prediction": 0.000017889078663, + "threshold": 0.72, + "is_selected": false + }, + "229": { + "prediction": 3.389243e-9, + "threshold": 0.55, + "is_selected": false + }, + "230": { + "prediction": 0.000016569508455, + "threshold": 0.61, + "is_selected": false + }, + "231": { + "prediction": 0.000004143511584, + "threshold": 0.3, + "is_selected": false + }, + "232": { + "prediction": 0.000640358295008, + "threshold": 0.23, + "is_selected": false + }, + "233": { + "prediction": 0.000006404845789, + "threshold": 0.31, + "is_selected": false + }, + "234": { + "prediction": 0.000074884925338, + "threshold": 0.39, + "is_selected": false + } + }, + "3": { + "301": { + "prediction": 0.000083330627376, + "threshold": 0.01, + "is_selected": false + }, + "302": { + "prediction": 0.011204658287831, + "threshold": 0.11, + "is_selected": false + }, + "303": { + "prediction": 0.000861139989483, + "threshold": 0.38, + "is_selected": false + }, + "304": { + "prediction": 0.000009533644629, + "threshold": 0.01, + "is_selected": false + }, + "305": { + "prediction": 0.010194102137843, + "threshold": 0.17, + "is_selected": false + }, + "306": { + "prediction": 0.000047473428519, + "threshold": 0.15, + "is_selected": false + }, + "307": { + "prediction": 0.00275926431641, + "threshold": 0.09, + "is_selected": false + }, + "308": { + "prediction": 0.006035644596872, + "threshold": 0.13, + "is_selected": false + }, + "309": { + "prediction": 0.000018762974768, + "threshold": 0.07, + "is_selected": false + }, + "310": { + "prediction": 0.16048023244366, + "threshold": 0.16, + "is_selected": true + }, + "311": { + "prediction": 0.001379056581451, + "threshold": 0.15, + "is_selected": false + }, + "312": { + "prediction": 0.144955087453127, + "threshold": 0.2, + "is_selected": false + }, + "313": { + "prediction": 0.042628173832782, + "threshold": 0.16, + "is_selected": false + }, + "314": { + "prediction": 0.000043664708755, + "threshold": 0.05, + "is_selected": false + }, + "315": { + "prediction": 0.000097360397275, + "threshold": 0.45, + "is_selected": false + }, + "316": { + "prediction": 0.000012243420618, + "threshold": 0.06, + "is_selected": false + }, + "317": { + "prediction": 0.000005113670909, + "threshold": 0.28, + "is_selected": false + }, + "318": { + "prediction": 0.000393391634973, + "threshold": 0.13, + "is_selected": false + } + }, + "4": { + "401": { + "prediction": 1.17259055e-7, + "threshold": 0.29, + "is_selected": false + }, + "402": { + "prediction": 0.000013744229364, + "threshold": 0.45, + "is_selected": false + }, + "403": { + "prediction": 4.87375621e-7, + "threshold": 0.03, + "is_selected": false + }, + "404": { + "prediction": 1.85885169e-7, + "threshold": 0.34, + "is_selected": false + }, + "405": { + "prediction": 3.05366841e-7, + "threshold": 0.37, + "is_selected": false + }, + "406": { + "prediction": 0.000034889759263, + "threshold": 0.25, + "is_selected": false + }, + "407": { + "prediction": 0.000001803972996, + "threshold": 0.07, + "is_selected": false + }, + "408": { + "prediction": 0.001109504095935, + "threshold": 0.11, + "is_selected": false + }, + "409": { + "prediction": 2.59159425e-7, + "threshold": 0.43, + "is_selected": false + }, + "410": { + "prediction": 1.45469337e-7, + "threshold": 0.23, + "is_selected": false + }, + "411": { + "prediction": 0.000005189525136, + "threshold": 0.06, + "is_selected": false + }, + "412": { + "prediction": 0.000002016342806, + "threshold": 0.36, + "is_selected": false + } + }, + "5": { + "501": { + "prediction": 0.000025967284374, + "threshold": 0.45, + "is_selected": false + }, + "502": { + "prediction": 0.000565126356378, + "threshold": 0.48, + "is_selected": false + } + }, + "6": { + "601": { + "prediction": 0.000177106418657, + "threshold": 0.06, + "is_selected": false + }, + "602": { + "prediction": 0.00055463691145, + "threshold": 0.48, + "is_selected": false + }, + "603": { + "prediction": 0.000022633564774, + "threshold": 0.34, + "is_selected": false + }, + "604": { + "prediction": 0.001842333040258, + "threshold": 0.16, + "is_selected": false + } + }, + "7": { + "701": { + "prediction": 0.000903384244777, + "threshold": 0.27, + "is_selected": false + }, + "702": { + "prediction": 0.001251186238898, + "threshold": 0.11, + "is_selected": false + }, + "703": { + "prediction": 0.000834142483654, + "threshold": 0.05, + "is_selected": false + }, + "704": { + "prediction": 0.000382219089564, + "threshold": 0.24, + "is_selected": false + }, + "705": { + "prediction": 0.001979890172758, + "threshold": 0.12, + "is_selected": true + } + }, + "8": { + "801": { + "prediction": 0.000014654744629, + "threshold": 0.66, + "is_selected": false + }, + "802": { + "prediction": 0.000044733506002, + "threshold": 0.3, + "is_selected": false + }, + "803": { + "prediction": 0.001036487083184, + "threshold": 0.36, + "is_selected": false + }, + "804": { + "prediction": 0.000707629901033, + "threshold": 0.23, + "is_selected": false + }, + "805": { + "prediction": 0.00003381500674, + "threshold": 0.58, + "is_selected": false + }, + "806": { + "prediction": 0.702028969923655, + "threshold": 0.3, + "is_selected": false + } + }, + "9": { + "902": { + "prediction": -1, + "threshold": -1, + "is_selected": false + }, + "903": { + "prediction": -1, + "threshold": -1, + "is_selected": false + }, + "904": { + "prediction": -1, + "threshold": -1, + "is_selected": false + }, + "905": { + "prediction": -1, + "threshold": -1, + "is_selected": false + }, + "906": { + "prediction": -1, + "threshold": -1, + "is_selected": false + }, + "907": { + "prediction": -1, + "threshold": -1, + "is_selected": false + } + } + }, + "prediction_status": "1", + "model_info": { + "id": "all_tags_model", + "version": "1.0.0" + } + }, + { + "model_info": { + "id": "geolocation", + "version": "1.0.0" + }, + "values": [], + "prediction_status": 1 + }, + { + "model_info": { + "id": "reliability", + "version": "1.0.0" + }, + "tags": {}, + "prediction_status": 0 + } + ] + } + } + ], + "client_id": "Mzc3Mg-bxc2ae-ab84e5577710374c9044d5527a813c7f", + "text_extraction_id": "Eei55y-ees2ae-ab77e5589110334c9077d5527a813c7f", + "text_extraction_status": 2 +} diff --git a/deep/urls.py b/deep/urls.py index 5f45439f0d..747ad29892 100644 --- a/deep/urls.py +++ b/deep/urls.py @@ -149,6 +149,7 @@ ) from deepl_integration.views import ( AssistedTaggingDraftEntryPredictionCallbackView, + AutoTaggingDraftEntryPredictionCallbackView, LeadExtractCallbackView, UnifiedConnectorLeadExtractCallbackView, AnalysisTopicModelCallbackView, @@ -566,6 +567,11 @@ def get_api_path(path): AssistedTaggingDraftEntryPredictionCallbackView.as_view(), name='assisted_tagging_draft_entry_prediction_callback', ), + re_path( + get_api_path(r'callback/auto-assisted-tagging-draft-entry-prediction/$'), + AutoTaggingDraftEntryPredictionCallbackView.as_view(), + name='auto-assisted_tagging_draft_entry_prediction_callback', + ), re_path( get_api_path(r'callback/analysis-topic-model/$'), diff --git a/schema.graphql b/schema.graphql index d48a3e7f85..ee70edc956 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1383,6 +1383,7 @@ type AssistedTaggingMutationType { wrongPredictionReviewCreate(data: WrongPredictionReviewInputType!): CreateWrongPredictionReview missingPredictionReviewDelete(id: ID!): DeleteMissingPredictionReview wrongPredictionReviewDelete(id: ID!): DeleteWrongPredictionReview + autoDraftEntryCreate(data: AutoDraftEntryInputType!): CreateAutoDraftEntry } enum AssistedTaggingPredictionDataTypeEnum { @@ -1408,6 +1409,8 @@ type AssistedTaggingPredictionType { type AssistedTaggingQueryType { draftEntry(id: ID!): DraftEntryType + draftEntryByLeads(filter: DraftEntryFilterDataInputType): [DraftEntryByLeadType!] + extractionStatusByLead: AutoextractionStatusType } type AssistedTaggingRootQueryType { @@ -1441,6 +1444,20 @@ type AttributeType { geoSelectedOptions: [ProjectGeoAreaType!] } +input AutoDraftEntryInputType { + lead: ID! +} + +enum AutoEntryExtractionTypeEnum { + PENDING + SUCCESS + FAILED +} + +type AutoextractionStatusType { + autoEntryExtractionStatus: AutoEntryExtractionTypeEnum! +} + enum AutomaticSummaryStatusEnum { PENDING STARTED @@ -1732,6 +1749,12 @@ type CreateAnalysisReportUpload { result: AnalysisReportUploadType } +type CreateAutoDraftEntry { + errors: [GenericScalar!] + ok: Boolean + result: DraftEntryType +} + type CreateDraftEntry { errors: [GenericScalar!] ok: Boolean @@ -1935,6 +1958,22 @@ type DjangoDebugSQL { encoding: String } +type DraftEntryByLeadType { + id: ID! + excerpt: String! + predictionReceivedAt: DateTime + predictionStatus: DraftEntryPredictionStatusEnum! + predictionStatusDisplay: EnumDescription! + predictions: [AssistedTaggingPredictionType!] + missingPredictionReviews: [MissingPredictionReviewType!] + relatedGeoareas: [ProjectGeoAreaType!] +} + +input DraftEntryFilterDataInputType { + lead: [ID!] + draftEntryType: [DraftEntryTypeEnum!] +} + input DraftEntryInputType { lead: ID! excerpt: String! @@ -1958,6 +1997,11 @@ type DraftEntryType { relatedGeoareas: [ProjectGeoAreaType!] } +enum DraftEntryTypeEnum { + AUTO + MANUAL +} + input EMMEntityInputType { name: String! } From c6036b61b7529fa48fefd344b5e5c710a6d7c026 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 29 Nov 2023 10:08:50 +0545 Subject: [PATCH 07/24] serializers updated and mock json updated --- apps/assisted_tagging/schema.py | 4 +- apps/assisted_tagging/tasks.py | 5 +- apps/deepl_integration/handlers.py | 47 ++++++++-------- apps/deepl_integration/serializers.py | 54 +++++++++++++++++-- apps/deepl_integration/views.py | 12 +---- .../migrations/0051_auto_20231128_0958.py | 28 ++++++++++ apps/lead/models.py | 2 + apps/lead/tests/test_apis.py | 22 +++++++- deep/deepl.py | 2 +- schema.graphql | 3 +- 10 files changed, 134 insertions(+), 45 deletions(-) create mode 100644 apps/lead/migrations/0051_auto_20231128_0958.py diff --git a/apps/assisted_tagging/schema.py b/apps/assisted_tagging/schema.py index fd926dcb4c..1338b5d37b 100644 --- a/apps/assisted_tagging/schema.py +++ b/apps/assisted_tagging/schema.py @@ -282,10 +282,10 @@ def custom_queryset(root, info, lead_id): class AssistedTaggingQueryType(graphene.ObjectType): draft_entry = DjangoObjectField(DraftEntryType) draft_entry_by_leads = DjangoListField(DraftEntryByLeadType, filter=DraftEntryFilterDataInputType()) - extraction_status_by_lead = graphene.Field(AutoextractionStatusType,lead_id=graphene.Int(required=True)) + extraction_status_by_lead = graphene.Field(AutoextractionStatusType, lead_id=graphene.ID(required=True)) def resolve_draft_entry_by_leads(root, info, filter): return DraftEntryByLeadType.custom_queryset(root, info, filter) def resolve_extraction_status_by_lead(root, info, lead_id): - return AutoextractionStatusType.custom_queryset(root, info,lead_id) + return AutoextractionStatusType.custom_queryset(root, info, lead_id) diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py index ed9afed588..a0b4fb4f56 100644 --- a/apps/assisted_tagging/tasks.py +++ b/apps/assisted_tagging/tasks.py @@ -1,5 +1,6 @@ import logging import requests +import json from celery import shared_task from lead.models import Lead @@ -25,7 +26,9 @@ def _get_existing_tags_by_tagid(): tag.tag_id: tag # tag_id is from deepl for tag in AssistedTaggingModelPredictionTag.objects.all() } - response = requests.get(DeeplServiceEndpoint.ASSISTED_TAGGING_TAGS_ENDPOINT).json() + headers = {"Authorization": "TOKEN c2e6c102ad3b4e1097242d0730a091c54f112c66"} + response = requests.get(DeeplServiceEndpoint.ASSISTED_TAGGING_TAGS_ENDPOINT, headers=headers) + response = json.loads(response.text) existing_tags_by_tagid = _get_existing_tags_by_tagid() new_tags = [] diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index e4b8b92608..6da6fd45cd 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -399,7 +399,7 @@ def get_versions_map(): lambda acc, item: acc | item, [ models.Q( - model__model_id=model_data['id'], + model__model_id=model_data['name'], version=model_data['version'], ) for model_data in models_data @@ -412,16 +412,16 @@ def get_versions_map(): new_model_versions = [ model_data for model_data in models_data - if (model_data['id'], model_data['version']) not in existing_model_versions + if (model_data['name'], model_data['version']) not in existing_model_versions ] if new_model_versions: AssistedTaggingModelVersion.objects.bulk_create([ AssistedTaggingModelVersion( model=AssistedTaggingModel.objects.get_or_create( - model_id=model_data['id'], + model_id=model_data['name'], defaults=dict( - name=model_data['id'], + name=model_data['name'], ), )[0], version=model_data['version'], @@ -464,12 +464,11 @@ def get_tags_map(): @classmethod def _process_model_preds(cls, model_version, current_tags_map, draft_entry, model_prediction): prediction_status = model_prediction['prediction_status'] - if prediction_status == 0: # If 0 no tags are provided + if not prediction_status: # If False no tags are provided return - tags = model_prediction.get('tags', {}) # NLP TagId + tags = model_prediction.get('classification', {}) # NLP TagId values = model_prediction.get('values', []) # Raw value - common_attrs = dict( model_version=model_version, draft_entry_id=draft_entry.id, @@ -505,35 +504,37 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode @classmethod def save_data(cls, lead, data): + # print("handler data......",data) for model_preds in data['blocks']: classification = model_preds['classification'] current_tags_map = cls._get_or_create_tags_map([ tag - for prediction in classification['model_preds'] - for category_tag, tags in prediction.get('tags', {}).items() + for category_tag, tags in classification.items() for tag in [ category_tag, *tags.keys(), ] ]) models_version_map = cls._get_or_create_models_version([ - prediction['model_info'] - for prediction in classification['model_preds'] + data['classification_model_info'] ]) with transaction.atomic(): - draft = DraftEntry.objects.create( - project=lead.project, - lead=lead, - excerpt=model_preds['text'], - draft_entry_type=0 - ) - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS - lead.save() - draft.save() - for prediction in classification['model_preds']: - model_version = models_version_map[(prediction['model_info']['id'], prediction['model_info']['version'])] - cls._process_model_preds(model_version, current_tags_map, draft, prediction) + if model_preds['relevant']: + draft = DraftEntry.objects.create( + project=lead.project, + lead=lead, + excerpt=model_preds['text'], + draft_entry_type=0 + ) + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS + lead.save() + draft.save() + model_version = models_version_map[ + (data['classification_model_info']['name'], data['classification_model_info']['version']) + ] + for prediction in data['blocks']: + cls._process_model_preds(model_version, current_tags_map, draft, prediction) return lead diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 94499ad618..7fd433a915 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -31,6 +31,8 @@ from .models import DeeplTrackBaseModel +from utils.request import RequestHelper + class BaseCallbackSerializer(serializers.Serializer): nlp_handler: Type[BaseHandler] @@ -192,6 +194,32 @@ class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): prediction_status = serializers.IntegerField() # 0 -> Failure, 1 -> Success +class AutoAssistedTaggingModelPredicationCallBackSerializer(serializers.Serializer): + class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): + predication = serializers.DecimalField( + max_digits=AssistedTaggingPrediction.prediction.field.max_digits, + decimal_places=AssistedTaggingPrediction.prediction.field.decimal_places, + required=False, + ) + threshold = serializers.DecimalField( + # From apps/assisted_tagging/models.py::AssistedTaggingPrediction::threshold + max_digits=AssistedTaggingPrediction.threshold.field.max_digits, + decimal_places=AssistedTaggingPrediction.threshold.field.decimal_places, + required=False, + ) + is_selected = serializers.BooleanField() + values = serializers.ListSerializer( + child=serializers.CharField(), + required=False, + ) + tags = serializers.DictField( + child=serializers.DictField( + child=ModelPredictionCallbackSerializerTagValue(), + ), + required=False, + ) + + class AssistedTaggingDraftEntryPredictionCallbackSerializer(BaseCallbackSerializer): model_preds = AssistedTaggingModelPredictionCallbackSerializer(many=True) @@ -208,22 +236,38 @@ def create(self, validated_data): class AutoAssistedBlockPredicationCallbackSerializer(serializers.Serializer): - class ClassificationInfoCallBackSerializer(serializers.Serializer): - model_preds = AssistedTaggingModelPredictionCallbackSerializer(many=True) text = serializers.CharField() - classification = ClassificationInfoCallBackSerializer() + relevant = serializers.BooleanField() + prediction_status = serializers.BooleanField() + # classification = AutoAssistedTaggingModelPredicationCallBackSerializer() + classification = serializers.DictField(child=serializers.DictField()) class AutoAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): - blocks = AutoAssistedBlockPredicationCallbackSerializer(many=True) + entry_extraction_classification_path = serializers.URLField(required=True) + text_extraction_id = serializers.CharField(required=True) + status = serializers.IntegerField() nlp_handler = AutoAssistedTaggingDraftEntryHandler def create(self, validated_data): + obj = validated_data['object'] + validated_data = RequestHelper(url=validated_data['entry_extraction_classification_path'], ignore_error=True).json() return self.nlp_handler.save_data( - validated_data['object'], + obj, validated_data ) +# class AutoAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): +# blocks = AutoAssistedBlockPredicationCallbackSerializer(many=True) +# classification_model_info = serializers.DictField() +# nlp_handler = AutoAssistedTaggingDraftEntryHandler + +# def create(self, validated_data): +# return self.nlp_handler.save_data( +# validated_data['object'], +# validated_data +# ) + class EntriesCollectionBaseCallbackSerializer(DeeplServerBaseCallbackSerializer): model: Type[DeeplTrackBaseModel] diff --git a/apps/deepl_integration/views.py b/apps/deepl_integration/views.py index a33112d37b..264d0ce6ec 100644 --- a/apps/deepl_integration/views.py +++ b/apps/deepl_integration/views.py @@ -18,8 +18,6 @@ AutoAssistedTaggingDraftEntryCallbackSerializer ) -from utils.request import RequestHelper - class BaseCallbackView(views.APIView): serializer: Type[serializers.Serializer] @@ -36,16 +34,8 @@ class AssistedTaggingDraftEntryPredictionCallbackView(BaseCallbackView): serializer = AssistedTaggingDraftEntryPredictionCallbackSerializer -class AutoTaggingDraftEntryPredictionCallbackView(views.APIView): +class AutoTaggingDraftEntryPredictionCallbackView(BaseCallbackView): serializer = AutoAssistedTaggingDraftEntryCallbackSerializer - permission_classes = [permissions.AllowAny] - - def post(self, request, **_): - data = RequestHelper(url=request.data['entry_extraction_classification_path'], ignore_error=True).json() - serializer = self.serializer(data=data) - serializer.is_valid(raise_exception=True) - serializer.save() - return response.Response("Request successfully completed", status=status.HTTP_200_OK) class LeadExtractCallbackView(BaseCallbackView): diff --git a/apps/lead/migrations/0051_auto_20231128_0958.py b/apps/lead/migrations/0051_auto_20231128_0958.py new file mode 100644 index 0000000000..8f8bb7fb6b --- /dev/null +++ b/apps/lead/migrations/0051_auto_20231128_0958.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.17 on 2023-11-28 09:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0050_delete_extractedlead'), + ] + + operations = [ + migrations.AddField( + model_name='leadpreview', + name='entry_extraction_id', + field=models.CharField(blank=True, max_length=30, null=True), + ), + migrations.AddField( + model_name='leadpreview', + name='text_extraction_id', + field=models.CharField(blank=True, max_length=30, null=True), + ), + migrations.AlterField( + model_name='lead', + name='auto_entry_extraction_status', + field=models.SmallIntegerField(choices=[(0, 'None'), (1, 'Pending'), (2, 'Success'), (3, 'Failed')], default=0), + ), + ] diff --git a/apps/lead/models.py b/apps/lead/models.py index c241693dde..09e05fb14b 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -364,6 +364,8 @@ class ClassificationStatus(models.TextChoices): choices=ClassificationStatus.choices, default=ClassificationStatus.NONE, ) + entry_extraction_id = models.CharField(max_length=30, null=True, blank=True) + text_extraction_id = models.CharField(max_length=30, null=True, blank=True) def __str__(self): return 'Text extracted for {}'.format(self.lead) diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 13d149c2ac..2d16fe9136 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -25,7 +25,7 @@ from organization.serializers import SimpleOrganizationSerializer from lead.filter_set import LeadFilterSet from lead.serializers import SimpleLeadGroupSerializer -from deepl_integration.handlers import LeadExtractionHandler +from deepl_integration.handlers import AutoAssistedTaggingDraftEntryHandler, LeadExtractionHandler from deepl_integration.serializers import DeeplServerBaseCallbackSerializer from entry.models import ( Entry, @@ -1852,3 +1852,23 @@ def test_client_id_generator(self): LeadExtractionHandler.get_object_using_client_id(client_id) else: assert LeadExtractionHandler.get_object_using_client_id(client_id) == lead + + +class AutoEntryExtractionCallback(TestCase): + def setUp(self): + super().setUp() + self.lead = LeadFactory.create() + + @mock.patch('deepl_integration.handlers.RequestHelper.json') + def test_entry_extraction_callback(self, get_json_mock): + url = '/api/v1/callback/auto-assisted-tagging-draft-entry-prediction/' + self.authenticate() + get_json_mock.return_value = "" + data = { + "client_id": AutoAssistedTaggingDraftEntryHandler.get_client_id(self.lead), + 'entry_extraction_classification_path': 'https://server-deepl.dev.datafriendlyspace.org/media/', + 'text_extraction_id': '43545', + 'status': 1 + } + response = self.client.post(url, data) + self.assert_200(response) diff --git a/deep/deepl.py b/deep/deepl.py index 7064fb6676..20fe9b1be8 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -8,7 +8,7 @@ class DeeplServiceEndpoint(): # DEEPL Service Endpoints (Existing/Legacy) # NOTE: This will be moved to server endpoints in near future - ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/vf_tags' + ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/api/v1/nlp-tags/' ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/model_info' ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/entry_predict' diff --git a/schema.graphql b/schema.graphql index ee70edc956..21e2c57595 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1410,7 +1410,7 @@ type AssistedTaggingPredictionType { type AssistedTaggingQueryType { draftEntry(id: ID!): DraftEntryType draftEntryByLeads(filter: DraftEntryFilterDataInputType): [DraftEntryByLeadType!] - extractionStatusByLead: AutoextractionStatusType + extractionStatusByLead(leadId: ID!): AutoextractionStatusType } type AssistedTaggingRootQueryType { @@ -1449,6 +1449,7 @@ input AutoDraftEntryInputType { } enum AutoEntryExtractionTypeEnum { + NONE PENDING SUCCESS FAILED From 9b1e0f989ebf5ab7e14c3b2300dd185db23b3fc9 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 29 Nov 2023 11:33:41 +0545 Subject: [PATCH 08/24] validation on duplicate draftentry of autoextraction --- apps/assisted_tagging/serializers.py | 3 +++ apps/lead/tests/test_apis.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index 34fe201474..814550507c 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -156,6 +156,9 @@ def create(self, data): text = LeadPreview.objects.filter(lead=self.data['lead']).first() if text.text_extract == '' or None: raise serializers.DjangoValidationError('Simplifed Text is empty') + draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], draft_entry_type=0) + if draft_entry: + raise serializers.DjangoValidationError('Draft entry already exit') # Use already existing draft entry if found data['draft_entry_type'] = 0 # auto extraction # Create new one and send trigger to deepl diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 2d16fe9136..757937b3d5 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1841,7 +1841,7 @@ def test_client_id_generator(self): lead1, f'{UidBase64Helper.encode(lead1.pk)}-some-random-id', LeadExtractionHandler.Exception.InvalidOrExpiredToken, - ), + F;L'AMSDF';L (lead1, '11-some-random-id', LeadExtractionHandler.Exception.InvalidTokenValue), (lead1, 'some-random-id', LeadExtractionHandler.Exception.InvalidTokenValue), (lead2, lead2_client_id, LeadExtractionHandler.Exception.ObjectNotFound), From 9c5c45d8459da37a3e40f57949991ab5177ca01f Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 29 Nov 2023 14:46:42 +0545 Subject: [PATCH 09/24] text extraction id added into lead preview --- apps/deepl_integration/handlers.py | 3 +++ apps/deepl_integration/serializers.py | 3 ++- apps/lead/tests/test_apis.py | 11 ++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 6da6fd45cd..897a036244 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -557,6 +557,7 @@ def send_trigger_request_to_extractor( 'callback_url': callback_url, 'request_type': NlpRequestType.USER if high_priority else NlpRequestType.SYSTEM, } + logger.error(payload) try: response = requests.post( DeeplServiceEndpoint.DOCS_EXTRACTOR_ENDPOINT, @@ -616,6 +617,7 @@ def save_data( images_uri: List[str], word_count: int, page_count: int, + text_extraction_id: str ): LeadPreview.objects.filter(lead=lead).delete() LeadPreviewImage.objects.filter(lead=lead).delete() @@ -626,6 +628,7 @@ def save_data( text_extract=RequestHelper(url=text_source_uri, ignore_error=True).get_text(sanitize=True) or '', word_count=word_count, page_count=page_count, + text_extraction_id=text_extraction_id ) # Save extracted images as LeadPreviewImage instances # TODO: The logic is same for unified_connector leads as well. Maybe have a single func? diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 7fd433a915..ccf5170a56 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -75,7 +75,7 @@ class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): text_path = serializers.CharField(required=False) total_words_count = serializers.IntegerField(required=False, default=0) total_pages = serializers.IntegerField(required=False, default=0) - + text_extraction_id = serializers.CharField(required=False) nlp_handler = LeadExtractionHandler def validate(self, data): @@ -104,6 +104,7 @@ def create(self, data): data.get('images_path', [])[:10], # TODO: Support for more images, too much image will error. data.get('total_words_count'), data.get('total_pages'), + data.get('text_extraction_id') ) # Add to deduplication index transaction.on_commit(lambda: index_lead_and_calculate_duplicates.delay(lead.id)) diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 757937b3d5..827039b7f2 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1841,7 +1841,7 @@ def test_client_id_generator(self): lead1, f'{UidBase64Helper.encode(lead1.pk)}-some-random-id', LeadExtractionHandler.Exception.InvalidOrExpiredToken, - F;L'AMSDF';L + ), (lead1, '11-some-random-id', LeadExtractionHandler.Exception.InvalidTokenValue), (lead1, 'some-random-id', LeadExtractionHandler.Exception.InvalidTokenValue), (lead2, lead2_client_id, LeadExtractionHandler.Exception.ObjectNotFound), @@ -1854,16 +1854,18 @@ def test_client_id_generator(self): assert LeadExtractionHandler.get_object_using_client_id(client_id) == lead -class AutoEntryExtractionCallback(TestCase): +class AutoEntryExtractionTestCase(TestCase): def setUp(self): super().setUp() self.lead = LeadFactory.create() + # @mock.patch('assisted_tagging.mutation. @mock.patch('deepl_integration.handlers.RequestHelper.json') def test_entry_extraction_callback(self, get_json_mock): url = '/api/v1/callback/auto-assisted-tagging-draft-entry-prediction/' self.authenticate() - get_json_mock.return_value = "" + get_json_mock.return_value = "testing" + print(get_json_mock) data = { "client_id": AutoAssistedTaggingDraftEntryHandler.get_client_id(self.lead), 'entry_extraction_classification_path': 'https://server-deepl.dev.datafriendlyspace.org/media/', @@ -1872,3 +1874,6 @@ def test_entry_extraction_callback(self, get_json_mock): } response = self.client.post(url, data) self.assert_200(response) + + def test_auto_extraction_mutation(self): + pass From 332048ed0867fe77ac5e7cec11d9e15f651d8f1d Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 30 Nov 2023 11:13:52 +0545 Subject: [PATCH 10/24] nlp extraction status code changed in lead exraction --- apps/assisted_tagging/serializers.py | 4 ++-- apps/deepl_integration/handlers.py | 11 +++++---- apps/deepl_integration/serializers.py | 2 +- .../migrations/0052_auto_20231130_0615.py | 23 +++++++++++++++++++ apps/lead/models.py | 11 +++++---- 5 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 apps/lead/migrations/0052_auto_20231130_0615.py diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index 814550507c..48c0067353 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -153,8 +153,8 @@ def validate(self, data): return data def create(self, data): - text = LeadPreview.objects.filter(lead=self.data['lead']).first() - if text.text_extract == '' or None: + lead_preview = LeadPreview.objects.filter(lead=self.data['lead']).first() + if lead_preview.text_extract == None: raise serializers.DjangoValidationError('Simplifed Text is empty') draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], draft_entry_type=0) if draft_entry: diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 897a036244..5b5d50a7d4 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -354,16 +354,16 @@ class AutoAssistedTaggingDraftEntryHandler(BaseHandler): @classmethod def auto_trigger_request_to_extractor(cls, lead): + lead_preview = LeadPreview.objects.get(lead=lead) payload = { "documents": [ { "client_id": cls.get_client_id(lead), # static clientid for mock - "text_extraction_id": "43545" # static text_extraction id + "text_extraction_id": lead_preview.text_extraction_id # static text_extraction id } ], "callback_url": cls.get_callback_url() } - logger.error(payload) try: response = requests.post( url=DeeplServiceEndpoint.ENTRY_EXTRACTION_CLASSIFICATION, @@ -504,7 +504,9 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode @classmethod def save_data(cls, lead, data): - # print("handler data......",data) + draft_entry = DraftEntry.objects.filter(lead=lead, draft_entry_type=0) + if draft_entry: + raise serializers.ValidationError('Draft entry already exit') for model_preds in data['blocks']: classification = model_preds['classification'] current_tags_map = cls._get_or_create_tags_map([ @@ -557,14 +559,13 @@ def send_trigger_request_to_extractor( 'callback_url': callback_url, 'request_type': NlpRequestType.USER if high_priority else NlpRequestType.SYSTEM, } - logger.error(payload) try: response = requests.post( DeeplServiceEndpoint.DOCS_EXTRACTOR_ENDPOINT, headers=cls.REQUEST_HEADERS, data=json.dumps(payload) ) - if response.status_code == 200: + if response.status_code == 202: return True except Exception: logger.error('Lead Extraction Failed, Exception occurred!!', exc_info=True) diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index ccf5170a56..19707c666c 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -66,7 +66,7 @@ class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): """ Serialize deepl extractor """ - url = serializers.CharField() + url = serializers.CharField(required=False) # Data fields images_path = serializers.ListField( child=serializers.CharField(allow_blank=True), diff --git a/apps/lead/migrations/0052_auto_20231130_0615.py b/apps/lead/migrations/0052_auto_20231130_0615.py new file mode 100644 index 0000000000..848e03b6a6 --- /dev/null +++ b/apps/lead/migrations/0052_auto_20231130_0615.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.17 on 2023-11-30 06:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0051_auto_20231128_0958'), + ] + + operations = [ + migrations.AlterField( + model_name='leadpreview', + name='entry_extraction_id', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='leadpreview', + name='text_extraction_id', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/apps/lead/models.py b/apps/lead/models.py index 09e05fb14b..9276750849 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -91,9 +91,10 @@ class ExtractionStatus(models.IntegerChoices): class AutoExtractionStatus(models.IntegerChoices): NONE = 0, "None" - PENDING = 1, 'Pending' - SUCCESS = 2, 'Success' - FAILED = 3, 'Failed' + STARTED = 1, "Started" + PENDING = 2, 'Pending' + SUCCESS = 3, 'Success' + FAILED = 4, 'Failed' lead_group = models.ForeignKey( LeadGroup, @@ -364,8 +365,8 @@ class ClassificationStatus(models.TextChoices): choices=ClassificationStatus.choices, default=ClassificationStatus.NONE, ) - entry_extraction_id = models.CharField(max_length=30, null=True, blank=True) - text_extraction_id = models.CharField(max_length=30, null=True, blank=True) + entry_extraction_id = models.TextField(blank=True,null=True) + text_extraction_id = models.TextField(blank=True,null=True) def __str__(self): return 'Text extracted for {}'.format(self.lead) From aeb7890265244716205fbd125ffa74887043cd2e Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 1 Dec 2023 00:34:51 +0545 Subject: [PATCH 11/24] Status check of sucess and pending --- apps/assisted_tagging/serializers.py | 11 +++++++---- apps/deepl_integration/handlers.py | 5 +++-- ..._alter_lead_auto_entry_extraction_status.py | 18 ++++++++++++++++++ schema.graphql | 1 + 4 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 apps/lead/migrations/0053_alter_lead_auto_entry_extraction_status.py diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index 48c0067353..1a2e9e50b7 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -10,7 +10,7 @@ PredictionTagAnalysisFrameworkWidgetMapping, WrongPredictionReview, ) -from lead.models import LeadPreview +from lead.models import LeadPreview, Lead from .tasks import trigger_request_for_draft_entry_task, trigger_request_for_mock_entry_task @@ -153,14 +153,17 @@ def validate(self, data): return data def create(self, data): - lead_preview = LeadPreview.objects.filter(lead=self.data['lead']).first() - if lead_preview.text_extract == None: + lead = Lead.objects.get(id=self.data['lead']) + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING + lead.save() + if lead.auto_entry_extraction_status == (Lead.AutoExtractionStatus.SUCCESS): + raise serializers.DjangoValidationError("Already Tiggered") + if lead.leadpreview .text_extract == None: raise serializers.DjangoValidationError('Simplifed Text is empty') draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], draft_entry_type=0) if draft_entry: raise serializers.DjangoValidationError('Draft entry already exit') # Use already existing draft entry if found - data['draft_entry_type'] = 0 # auto extraction # Create new one and send trigger to deepl transaction.on_commit( lambda: trigger_request_for_mock_entry_task.delay(self.data['lead']) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 5b5d50a7d4..f8dc188583 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -378,9 +378,10 @@ def auto_trigger_request_to_extractor(cls, lead): except Exception: logger.error('Entry Extraction send failed, Exception occurred!!', exc_info=True) - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.FAILED - lead.save() + # lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.FAILED + # lead.save() _response = locals().get('response') + logger.error(payload) logger.error( 'Entry Extraction send failed!!', extra={ diff --git a/apps/lead/migrations/0053_alter_lead_auto_entry_extraction_status.py b/apps/lead/migrations/0053_alter_lead_auto_entry_extraction_status.py new file mode 100644 index 0000000000..9f1f1a114c --- /dev/null +++ b/apps/lead/migrations/0053_alter_lead_auto_entry_extraction_status.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-11-30 16:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0052_auto_20231130_0615'), + ] + + operations = [ + migrations.AlterField( + model_name='lead', + name='auto_entry_extraction_status', + field=models.SmallIntegerField(choices=[(0, 'None'), (1, 'Started'), (2, 'Pending'), (3, 'Success'), (4, 'Failed')], default=0), + ), + ] diff --git a/schema.graphql b/schema.graphql index 21e2c57595..1ce8485dae 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1450,6 +1450,7 @@ input AutoDraftEntryInputType { enum AutoEntryExtractionTypeEnum { NONE + STARTED PENDING SUCCESS FAILED From 3eda48b31e439e5f9b838825b219a2c2682f34fb Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 1 Dec 2023 12:42:14 +0545 Subject: [PATCH 12/24] cleanup code and repeated tag save issue fixed --- apps/assisted_tagging/admin.py | 3 ++- apps/assisted_tagging/models.py | 5 ++++- apps/assisted_tagging/serializers.py | 4 ++-- apps/deepl_integration/handlers.py | 9 ++++++--- apps/lead/models.py | 4 ++-- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/assisted_tagging/admin.py b/apps/assisted_tagging/admin.py index 0348a0e19a..155f785a8f 100644 --- a/apps/assisted_tagging/admin.py +++ b/apps/assisted_tagging/admin.py @@ -19,7 +19,8 @@ class AssistedTaggingPredictionAdmin(VersionAdmin): list_display = [ "data_type", "draft_entry", - "value" + "value", + "is_selected" ] diff --git a/apps/assisted_tagging/models.py b/apps/assisted_tagging/models.py index fb02a1f931..81370cdbf0 100644 --- a/apps/assisted_tagging/models.py +++ b/apps/assisted_tagging/models.py @@ -104,6 +104,9 @@ class DraftEntryType(models.IntegerChoices): related_geoareas = models.ManyToManyField(GeoArea, blank=True) draft_entry_type = models.SmallIntegerField(choices=DraftEntryType.choices, default=DraftEntryType.MANUAL) + def __str__(self): + return f'{self.id}' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.predictions: models.QuerySet[AssistedTaggingPrediction] @@ -181,7 +184,7 @@ class DataType(models.IntegerChoices): id: int def __str__(self): - return self.id + return str(self.id) class WrongPredictionReview(UserResource): diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index 1a2e9e50b7..f88de8cf0f 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -10,7 +10,7 @@ PredictionTagAnalysisFrameworkWidgetMapping, WrongPredictionReview, ) -from lead.models import LeadPreview, Lead +from lead.models import Lead from .tasks import trigger_request_for_draft_entry_task, trigger_request_for_mock_entry_task @@ -158,7 +158,7 @@ def create(self, data): lead.save() if lead.auto_entry_extraction_status == (Lead.AutoExtractionStatus.SUCCESS): raise serializers.DjangoValidationError("Already Tiggered") - if lead.leadpreview .text_extract == None: + if not lead.leadpreview.text_extract: raise serializers.DjangoValidationError('Simplifed Text is empty') draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], draft_entry_type=0) if draft_entry: diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index f8dc188583..81ee3926dd 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -359,7 +359,7 @@ def auto_trigger_request_to_extractor(cls, lead): "documents": [ { "client_id": cls.get_client_id(lead), # static clientid for mock - "text_extraction_id": lead_preview.text_extraction_id # static text_extraction id + "text_extraction_id": lead_preview.text_extraction_id } ], "callback_url": cls.get_callback_url() @@ -476,6 +476,10 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode ) new_predictions = [] for category_tag, tags in tags.items(): + common_attrs = dict( + model_version=model_version, + draft_entry_id=draft_entry.id + ) for tag, prediction_data in tags.items(): prediction_value = prediction_data.get('prediction') threshold_value = prediction_data.get('threshold') @@ -536,8 +540,7 @@ def save_data(cls, lead, data): model_version = models_version_map[ (data['classification_model_info']['name'], data['classification_model_info']['version']) ] - for prediction in data['blocks']: - cls._process_model_preds(model_version, current_tags_map, draft, prediction) + cls._process_model_preds(model_version, current_tags_map, draft, model_preds) return lead diff --git a/apps/lead/models.py b/apps/lead/models.py index 9276750849..981074774f 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -365,8 +365,8 @@ class ClassificationStatus(models.TextChoices): choices=ClassificationStatus.choices, default=ClassificationStatus.NONE, ) - entry_extraction_id = models.TextField(blank=True,null=True) - text_extraction_id = models.TextField(blank=True,null=True) + entry_extraction_id = models.TextField(blank=True, null=True) + text_extraction_id = models.TextField(blank=True, null=True) def __str__(self): return 'Text extracted for {}'.format(self.lead) From 41fd45dae66b8b34933dd91e500b13c3b7f387e1 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Mon, 4 Dec 2023 17:32:20 +0545 Subject: [PATCH 13/24] Entry predication json changed --- apps/deepl_integration/handlers.py | 31 +++++++++++++++------------ apps/deepl_integration/serializers.py | 15 +++++++------ deep/deepl.py | 6 +++--- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 81ee3926dd..aa649ed382 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -188,13 +188,14 @@ def send_trigger_request_to_extractor(cls, draft_entry): headers=cls.REQUEST_HEADERS, json=payload ) - if response.status_code == 200: + if response.status_code == 202: return True except Exception: logger.error('Assisted tagging send failed, Exception occurred!!', exc_info=True) draft_entry.prediction_status = DraftEntry.PredictionStatus.SEND_FAILED draft_entry.save(update_fields=('prediction_status',)) _response = locals().get('response') + logger.error(payload) logger.error( 'Assisted tagging send failed!!', extra={ @@ -279,10 +280,10 @@ def get_tags_map(): @classmethod def _process_model_preds(cls, model_version, current_tags_map, draft_entry, model_prediction): prediction_status = model_prediction['prediction_status'] - if prediction_status == 0: # If 0 no tags are provided + if not prediction_status: # If 0 no tags are provided return - tags = model_prediction.get('tags', {}) # NLP TagId + tags = model_prediction.get('model_tags', {}) # NLP TagId values = model_prediction.get('values', []) # Raw value common_attrs = dict( @@ -320,28 +321,30 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode @classmethod def save_data(cls, draft_entry, data): - model_preds = data['model_preds'] + model_preds = data # Save if new tags are provided current_tags_map = cls._get_or_create_tags_map([ tag - for prediction in model_preds - for category_tag, tags in prediction.get('tags', {}).items() + # for prediction in model_preds['model_tags'] + for category_tag, tags in model_preds['model_tags'].items() for tag in [ category_tag, *tags.keys(), ] ]) - models_version_map = cls._get_or_create_models_version([ - prediction['model_info'] - for prediction in model_preds - ]) - + print(current_tags_map) + models_version_map = cls._get_or_create_models_version( + [ + model_preds['model_info'] + ] + ) with transaction.atomic(): draft_entry.clear_data() # Clear old data if exists draft_entry.calculated_at = timezone.now() - for prediction in model_preds: - model_version = models_version_map[(prediction['model_info']['id'], prediction['model_info']['version'])] - cls._process_model_preds(model_version, current_tags_map, draft_entry, prediction) + # for prediction in model_preds: + model_version = models_version_map[(model_preds['model_info']['id'], model_preds['model_info']['version'])] + print(model_preds) + cls._process_model_preds(model_version, current_tags_map, draft_entry, model_preds) draft_entry.prediction_status = DraftEntry.PredictionStatus.DONE draft_entry.save_geo_data() draft_entry.save() diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 19707c666c..452849f9dd 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -162,9 +162,9 @@ def create(self, data): # --- AssistedTagging class AssistedTaggingModelPredictionCallbackSerializer(serializers.Serializer): - class ModelInfoCallbackSerializer(serializers.Serializer): - id = serializers.CharField() - version = serializers.CharField() + # class ModelInfoCallbackSerializer(serializers.Serializer): + # id = serializers.CharField() + # version = serializers.CharField() class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): prediction = serializers.DecimalField( @@ -181,7 +181,7 @@ class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): ) is_selected = serializers.BooleanField() - model_info = ModelInfoCallbackSerializer() + # model_info = ModelInfoCallbackSerializer() removed from the DEEPL TODO Use different api for model information values = serializers.ListSerializer( child=serializers.CharField(), required=False, @@ -192,7 +192,6 @@ class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): ), required=False, ) - prediction_status = serializers.IntegerField() # 0 -> Failure, 1 -> Success class AutoAssistedTaggingModelPredicationCallBackSerializer(serializers.Serializer): @@ -222,8 +221,10 @@ class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): class AssistedTaggingDraftEntryPredictionCallbackSerializer(BaseCallbackSerializer): - model_preds = AssistedTaggingModelPredictionCallbackSerializer(many=True) - + # model_tags = AssistedTaggingModelPredictionCallbackSerializer() + model_tags = serializers.DictField(child=serializers.DictField()) + prediction_status = serializers.BooleanField() + model_info = serializers.DictField() nlp_handler = AssistedTaggingDraftEntryHandler def create(self, validated_data): diff --git a/deep/deepl.py b/deep/deepl.py index 20fe9b1be8..bc305229d2 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -8,9 +8,9 @@ class DeeplServiceEndpoint(): # DEEPL Service Endpoints (Existing/Legacy) # NOTE: This will be moved to server endpoints in near future - ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/api/v1/nlp-tags/' - ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/model_info' - ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/entry_predict' + ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/nlp-tags/' + ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/model_info' + ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/api/v1/entry-classification/' # DEEPL Server Endpoints (New) DOCS_EXTRACTOR_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/text-extraction/' From 7a545b0cd8f1adeacda3d9cd53611c4917b22a0e Mon Sep 17 00:00:00 2001 From: sudan45 Date: Mon, 11 Dec 2023 11:36:37 +0545 Subject: [PATCH 14/24] discard draft entry --- .../migrations/0012_draftentry_is_discarded.py | 18 ++++++++++++++++++ apps/assisted_tagging/models.py | 3 ++- apps/assisted_tagging/mutation.py | 11 +++++++++++ apps/assisted_tagging/serializers.py | 10 ++++++---- schema.graphql | 8 ++++++++ 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 apps/assisted_tagging/migrations/0012_draftentry_is_discarded.py diff --git a/apps/assisted_tagging/migrations/0012_draftentry_is_discarded.py b/apps/assisted_tagging/migrations/0012_draftentry_is_discarded.py new file mode 100644 index 0000000000..63746219e8 --- /dev/null +++ b/apps/assisted_tagging/migrations/0012_draftentry_is_discarded.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-12-11 05:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assisted_tagging', '0011_draftentry_draft_entry_type'), + ] + + operations = [ + migrations.AddField( + model_name='draftentry', + name='is_discarded', + field=models.BooleanField(default=False), + ), + ] diff --git a/apps/assisted_tagging/models.py b/apps/assisted_tagging/models.py index 81370cdbf0..5026079e14 100644 --- a/apps/assisted_tagging/models.py +++ b/apps/assisted_tagging/models.py @@ -92,7 +92,7 @@ class PredictionStatus(models.IntegerChoices): class DraftEntryType(models.IntegerChoices): AUTO = 0, 'Auto Extraction' # NLP defiend extraction text - MANUAL = 1, 'Manual Extraction' # manaul defiend extraction text + MANUAL = 1, 'Manual Extraction' # manaul defined extraction text project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='+') lead = models.ForeignKey(Lead, on_delete=models.CASCADE, related_name='+') @@ -103,6 +103,7 @@ class DraftEntryType(models.IntegerChoices): # Additional attribues related_geoareas = models.ManyToManyField(GeoArea, blank=True) draft_entry_type = models.SmallIntegerField(choices=DraftEntryType.choices, default=DraftEntryType.MANUAL) + is_discarded = models.BooleanField(default=False) def __str__(self): return f'{self.id}' diff --git a/apps/assisted_tagging/mutation.py b/apps/assisted_tagging/mutation.py index e6568d34ae..c7ab1e7aae 100644 --- a/apps/assisted_tagging/mutation.py +++ b/apps/assisted_tagging/mutation.py @@ -114,6 +114,16 @@ class Arguments: permissions = [PP.Permission.CREATE_ENTRY] +class DiscardDraftEntry(PsGrapheneMutation): + class Arguments: + data = AutoDraftEntryInputType(required=True) + id = graphene.ID(required=True) + model = DraftEntry + serializer_class = AutoDraftEntryGqlSerializer + result = graphene.Field(DraftEntryType) + permissions = [PP.Permission.CREATE_ENTRY] + + class AssistedTaggingMutationType(graphene.ObjectType): draft_entry_create = CreateDraftEntry.Field() missing_prediction_review_create = CreateMissingPredictionReview.Field() @@ -121,3 +131,4 @@ class AssistedTaggingMutationType(graphene.ObjectType): missing_prediction_review_delete = DeleteMissingPredictionReview.Field() wrong_prediction_review_delete = DeleteWrongPredictionReview.Field() auto_draft_entry_create = CreateAutoDraftEntry.Field() + discard_draft_entry = DiscardDraftEntry.Field() diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index f88de8cf0f..d2fa602fbc 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -134,6 +134,7 @@ class Meta: model = DraftEntry fields = ( 'lead', + 'is_discarded' ) def validate_lead(self, lead): @@ -154,8 +155,6 @@ def validate(self, data): def create(self, data): lead = Lead.objects.get(id=self.data['lead']) - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING - lead.save() if lead.auto_entry_extraction_status == (Lead.AutoExtractionStatus.SUCCESS): raise serializers.DjangoValidationError("Already Tiggered") if not lead.leadpreview.text_extract: @@ -165,11 +164,14 @@ def create(self, data): raise serializers.DjangoValidationError('Draft entry already exit') # Use already existing draft entry if found # Create new one and send trigger to deepl + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING + lead.save() transaction.on_commit( lambda: trigger_request_for_mock_entry_task.delay(self.data['lead']) ) return True - def update(self, *_): - raise Exception('Update not allowed') + def update(self, instance, validate_data): + validate_data['is_discarded'] = True + return super().update(instance, validate_data) diff --git a/schema.graphql b/schema.graphql index 1ce8485dae..275d205003 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1384,6 +1384,7 @@ type AssistedTaggingMutationType { missingPredictionReviewDelete(id: ID!): DeleteMissingPredictionReview wrongPredictionReviewDelete(id: ID!): DeleteWrongPredictionReview autoDraftEntryCreate(data: AutoDraftEntryInputType!): CreateAutoDraftEntry + discardDraftEntry(data: AutoDraftEntryInputType!, id: ID!): DiscardDraftEntry } enum AssistedTaggingPredictionDataTypeEnum { @@ -1446,6 +1447,7 @@ type AttributeType { input AutoDraftEntryInputType { lead: ID! + isDiscarded: Boolean } enum AutoEntryExtractionTypeEnum { @@ -1918,6 +1920,12 @@ type DeleteWrongPredictionReview { result: WrongPredictionReviewType } +type DiscardDraftEntry { + errors: [GenericScalar!] + ok: Boolean + result: DraftEntryType +} + input DiscardedEntryCreateInputType { id: ID analysisPillar: ID! From f7a0981c6c3bb14ab233e0c381183835d36d9f7a Mon Sep 17 00:00:00 2001 From: sudan45 Date: Mon, 11 Dec 2023 23:21:35 +0545 Subject: [PATCH 15/24] clean up code --- apps/assisted_tagging/filters.py | 10 +--- apps/assisted_tagging/mutation.py | 22 +++++--- apps/assisted_tagging/serializers.py | 82 ++++++++++++++-------------- apps/assisted_tagging/tasks.py | 11 ++-- apps/lead/models.py | 4 +- deep/deepl.py | 6 +- 6 files changed, 68 insertions(+), 67 deletions(-) diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index 277fac72ae..64f6adbea7 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -9,19 +9,13 @@ class DraftEntryFilterSet(django_filters.FilterSet): - lead = IDListFilter() - draft_entry_type = MultipleInputFilter(DraftEntryTypeEnum) + leads = IDListFilter(field_name='lead') + draft_entry_types = MultipleInputFilter(DraftEntryTypeEnum, field_name='draft_entry_type') class Meta: model = DraftEntry fields = () - def filter_lead(self, qs, _, value): - return qs if value is None else qs.filter(lead=value) - - def filter_draft_entry_type(self, qs, _, value): - return qs if value is None else qs.filter(draft_entry_type=value) - DraftEntryFilterDataType, DraftEntryFilterDataInputType = generate_type_for_filter_set( DraftEntryFilterSet, diff --git a/apps/assisted_tagging/mutation.py b/apps/assisted_tagging/mutation.py index c7ab1e7aae..075c9f873b 100644 --- a/apps/assisted_tagging/mutation.py +++ b/apps/assisted_tagging/mutation.py @@ -21,7 +21,8 @@ DraftEntryGqlSerializer, WrongPredictionReviewGqlSerializer, MissingPredictionReviewGqlSerializer, - AutoDraftEntryGqlSerializer + TriggeredDraftEntryGqlSerializer, + UpdateDraftEntrySerializer ) @@ -40,9 +41,14 @@ serializer_class=MissingPredictionReviewGqlSerializer, ) -AutoDraftEntryInputType = generate_input_type_for_serializer( - "AutoDraftEntryInputType", - serializer_class=AutoDraftEntryGqlSerializer +TriggerDraftEntryInputType = generate_input_type_for_serializer( + "TriggerDraftEntryInputType", + serializer_class=TriggeredDraftEntryGqlSerializer +) + +UpdateDraftEntryInputType = generate_input_type_for_serializer( + "UpdateDraftEntryInputType", + serializer_class=UpdateDraftEntrySerializer ) @@ -107,19 +113,19 @@ def filter_queryset(cls, qs, info): class CreateAutoDraftEntry(PsGrapheneMutation): class Arguments: - data = AutoDraftEntryInputType(required=True) + data = TriggerDraftEntryInputType(required=True) model = DraftEntry - serializer_class = AutoDraftEntryGqlSerializer + serializer_class = TriggeredDraftEntryGqlSerializer result = graphene.Field(DraftEntryType) permissions = [PP.Permission.CREATE_ENTRY] class DiscardDraftEntry(PsGrapheneMutation): class Arguments: - data = AutoDraftEntryInputType(required=True) + data = UpdateDraftEntryInputType(required=True) id = graphene.ID(required=True) model = DraftEntry - serializer_class = AutoDraftEntryGqlSerializer + serializer_class = UpdateDraftEntrySerializer result = graphene.Field(DraftEntryType) permissions = [PP.Permission.CREATE_ENTRY] diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index d2fa602fbc..c08bdb8e5b 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -11,33 +11,34 @@ WrongPredictionReview, ) from lead.models import Lead -from .tasks import trigger_request_for_draft_entry_task, trigger_request_for_mock_entry_task +from .tasks import ( + trigger_request_for_draft_entry_task, + trigger_request_for_auto_draft_entry_task +) # ---------- Graphql --------------------------- -class DraftEntryGqlSerializer(ProjectPropertySerializerMixin, UserResourceCreatedMixin, serializers.ModelSerializer): - class Meta: - model = DraftEntry - fields = ( - 'lead', - 'excerpt', - ) - +class DraftEntryBaseSerializer(serializers.Serializer): def validate_lead(self, lead): if lead.project != self.project: raise serializers.ValidationError('Only lead from current project are allowed.') af = lead.project.analysis_framework if af is None or not af.assisted_tagging_enabled: raise serializers.ValidationError('Assisted tagging is disabled for the Framework used by this project.') - return lead - - def validate(self, data): - if self.instance and self.instance.created_by != self.context['request'].user: - raise serializers.ValidationError('Only reviewer can edit this review') - data['project'] = self.project if self.project.is_private: raise serializers.ValidationError('Assisted tagging is not available for private projects.') - return data + return lead + + +class DraftEntryGqlSerializer( + ProjectPropertySerializerMixin, DraftEntryBaseSerializer, UserResourceCreatedMixin, serializers.ModelSerializer +): + class Meta: + model = DraftEntry + fields = ( + 'lead', + 'excerpt', + ) def create(self, data): # Use already existing draft entry if found @@ -56,7 +57,7 @@ def create(self, data): return instance def update(self, *_): - raise Exception('Update not allowed') + raise Exception("Update is not allowed") class WrongPredictionReviewGqlSerializer(UserResourceSerializer, serializers.ModelSerializer): @@ -129,30 +130,16 @@ def validate(self, data): return data -class AutoDraftEntryGqlSerializer(ProjectPropertySerializerMixin, UserResourceCreatedMixin, serializers.ModelSerializer): +class TriggeredDraftEntryGqlSerializer( + DraftEntryBaseSerializer, + ProjectPropertySerializerMixin, + UserResourceCreatedMixin, serializers.ModelSerializer): class Meta: model = DraftEntry fields = ( 'lead', - 'is_discarded' ) - def validate_lead(self, lead): - if lead.project != self.project: - raise serializers.ValidationError('Only lead from current project are allowed.') - af = lead.project.analysis_framework - if af is None or not af.assisted_tagging_enabled: - raise serializers.ValidationError('Assisted tagging is disabled for the Framework used by this project.') - return lead - - def validate(self, data): - if self.instance and self.instance.created_by != self.context['request'].user: - raise serializers.ValidationError('Only reviewer can edit this review') - data['project'] = self.project - if self.project.is_private: - raise serializers.ValidationError('Assisted tagging is not available for private projects.') - return data - def create(self, data): lead = Lead.objects.get(id=self.data['lead']) if lead.auto_entry_extraction_status == (Lead.AutoExtractionStatus.SUCCESS): @@ -160,18 +147,33 @@ def create(self, data): if not lead.leadpreview.text_extract: raise serializers.DjangoValidationError('Simplifed Text is empty') draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], draft_entry_type=0) - if draft_entry: + if draft_entry.exists(): raise serializers.DjangoValidationError('Draft entry already exit') # Use already existing draft entry if found # Create new one and send trigger to deepl lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING - lead.save() + lead.save(update_fields=['auto_entry_extraction_status']) transaction.on_commit( - lambda: trigger_request_for_mock_entry_task.delay(self.data['lead']) + lambda: trigger_request_for_auto_draft_entry_task.delay(self.data['lead']) ) - return True def update(self, instance, validate_data): - validate_data['is_discarded'] = True + raise Exception("updated is not allowed") + + +class UpdateDraftEntrySerializer( + DraftEntryBaseSerializer, ProjectPropertySerializerMixin, UserResourceSerializer, serializers.ModelSerializer +): + class Meta: + model = DraftEntry + fields = ( + 'lead', + 'is_discarded', + ) + + def create(self, validate_data): + raise Exception("Create is not Allowed") + + def update(self, instance, validate_data): return super().update(instance, validate_data) diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py index a0b4fb4f56..03327fb87f 100644 --- a/apps/assisted_tagging/tasks.py +++ b/apps/assisted_tagging/tasks.py @@ -1,3 +1,4 @@ +from django.conf import settings import logging import requests import json @@ -26,9 +27,9 @@ def _get_existing_tags_by_tagid(): tag.tag_id: tag # tag_id is from deepl for tag in AssistedTaggingModelPredictionTag.objects.all() } - headers = {"Authorization": "TOKEN c2e6c102ad3b4e1097242d0730a091c54f112c66"} - response = requests.get(DeeplServiceEndpoint.ASSISTED_TAGGING_TAGS_ENDPOINT, headers=headers) - response = json.loads(response.text) + + headers = {'Authorization': f'Token {settings.DEEPL_SERVER_TOKEN}'} + response = requests.get(DeeplServiceEndpoint.ASSISTED_TAGGING_TAGS_ENDPOINT, headers=headers).json() existing_tags_by_tagid = _get_existing_tags_by_tagid() new_tags = [] @@ -97,8 +98,8 @@ def trigger_request_for_draft_entry_task(draft_entry_id): @shared_task -@redis_lock('trigger_request_for_mock_entry_task_{0}', 60 * 60 * 0.5) -def trigger_request_for_mock_entry_task(lead): +@redis_lock('trigger_request_for_auto_draft_entry_task_{0}', 60 * 60 * 0.5) +def trigger_request_for_auto_draft_entry_task(lead): lead = Lead.objects.get(id=lead) return AutoAssistedTaggingDraftEntryHandler.auto_trigger_request_to_extractor(lead) diff --git a/apps/lead/models.py b/apps/lead/models.py index 981074774f..eb04b1fe5a 100644 --- a/apps/lead/models.py +++ b/apps/lead/models.py @@ -365,8 +365,8 @@ class ClassificationStatus(models.TextChoices): choices=ClassificationStatus.choices, default=ClassificationStatus.NONE, ) - entry_extraction_id = models.TextField(blank=True, null=True) - text_extraction_id = models.TextField(blank=True, null=True) + entry_extraction_id = models.UUIDField(blank=True, null=True) # Saved when EntryExtraction is completed + text_extraction_id = models.UUIDField(blank=True, null=True) # Saved when TextExtraction is completed def __str__(self): return 'Text extracted for {}'.format(self.lead) diff --git a/deep/deepl.py b/deep/deepl.py index bc305229d2..5233d91f3a 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -8,16 +8,14 @@ class DeeplServiceEndpoint(): # DEEPL Service Endpoints (Existing/Legacy) # NOTE: This will be moved to server endpoints in near future - ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/nlp-tags/' ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/model_info' - ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/api/v1/entry-classification/' # DEEPL Server Endpoints (New) + ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/nlp-tags/' DOCS_EXTRACTOR_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/text-extraction/' ANALYSIS_TOPIC_MODEL = f'{DEEPL_SERVER_DOMAIN}/api/v1/topicmodel/' ANALYSIS_AUTOMATIC_SUMMARY = f'{DEEPL_SERVER_DOMAIN}/api/v1/summarization/' ANALYSIS_AUTOMATIC_NGRAM = f'{DEEPL_SERVER_DOMAIN}/api/v1/ngrams/' ANALYSIS_GEO = f'{DEEPL_SERVER_DOMAIN}/api/v1/geolocation/' - - # AutoExtraction DeepL endpoint + ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/api/v1/entry-classification/' ENTRY_EXTRACTION_CLASSIFICATION = f'{DEEPL_SERVER_DOMAIN}/api/v1/entry-extraction-classification/' From 74260b19fbabf7559dd884e62a0dcaa2d4aecf9c Mon Sep 17 00:00:00 2001 From: sudan45 Date: Tue, 12 Dec 2023 13:46:44 +0545 Subject: [PATCH 16/24] test case updated --- apps/assisted_tagging/tasks.py | 1 - apps/deepl_integration/handlers.py | 2 -- apps/lead/tests/test_apis.py | 17 +++++++++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py index 03327fb87f..eceb2ac70a 100644 --- a/apps/assisted_tagging/tasks.py +++ b/apps/assisted_tagging/tasks.py @@ -1,7 +1,6 @@ from django.conf import settings import logging import requests -import json from celery import shared_task from lead.models import Lead diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index aa649ed382..f7113a3e5a 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -381,8 +381,6 @@ def auto_trigger_request_to_extractor(cls, lead): except Exception: logger.error('Entry Extraction send failed, Exception occurred!!', exc_info=True) - # lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.FAILED - # lead.save() _response = locals().get('response') logger.error(payload) logger.error( diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 827039b7f2..6cec3f5694 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1,5 +1,6 @@ import logging from datetime import date +import uuid from django.db.models import Q from django.core.files.uploadedfile import SimpleUploadedFile @@ -43,7 +44,7 @@ ) from user_group.models import UserGroup, GroupMembership from ary.models import Assessment -from lead.factories import LeadFactory +from lead.factories import LeadFactory, LeadPreviewFactory from unittest import mock @@ -1858,22 +1859,22 @@ class AutoEntryExtractionTestCase(TestCase): def setUp(self): super().setUp() self.lead = LeadFactory.create() - # @mock.patch('assisted_tagging.mutation. + self.lead_preview = LeadPreviewFactory.create(lead=self.lead, text_extraction_id=str(uuid.uuid1())) - @mock.patch('deepl_integration.handlers.RequestHelper.json') - def test_entry_extraction_callback(self, get_json_mock): + def test_entry_extraction_callback(self): url = '/api/v1/callback/auto-assisted-tagging-draft-entry-prediction/' self.authenticate() - get_json_mock.return_value = "testing" - print(get_json_mock) data = { "client_id": AutoAssistedTaggingDraftEntryHandler.get_client_id(self.lead), - 'entry_extraction_classification_path': 'https://server-deepl.dev.datafriendlyspace.org/media/', - 'text_extraction_id': '43545', + 'entry_extraction_classification_path': 'https://server-deepl.dev.datafriendlyspace.org/media/mock_responses/entry_extraction/entry-extraction-client-6ppp.json', # noqa: E501 + 'text_extraction_id': self.lead_preview.text_extraction_id, 'status': 1 } response = self.client.post(url, data) self.assert_200(response) + self.lead.refresh_from_db() + self.assertEqual(self.lead.auto_entry_extraction_status, Lead.AutoExtractionStatus.SUCCESS) + self.assertEqual(LeadPreview.objects.get(lead=self.lead).text_extraction_id, data['text_extraction_id']) def test_auto_extraction_mutation(self): pass From 37d97049666780770e682e85f1487249af96cd34 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 13 Dec 2023 10:44:17 +0545 Subject: [PATCH 17/24] discard added in the filterset --- apps/assisted_tagging/filters.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index 64f6adbea7..ec86f3a1f5 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -11,6 +11,12 @@ class DraftEntryFilterSet(django_filters.FilterSet): leads = IDListFilter(field_name='lead') draft_entry_types = MultipleInputFilter(DraftEntryTypeEnum, field_name='draft_entry_type') + is_discarded = django_filters.BooleanFilter(method='filter_discarded') + + def filter_discarded(self, queryset, name, value): + if value: + return queryset.filter(is_discarded=value) + return queryset.filter(is_discarded = False) class Meta: model = DraftEntry From 78d341808fad86b9bd59adc942e59187e2303ad9 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 13 Dec 2023 16:55:43 +0545 Subject: [PATCH 18/24] discarded count and refactor the code --- apps/assisted_tagging/filters.py | 6 ++-- apps/assisted_tagging/schema.py | 3 +- apps/deepl_integration/handlers.py | 37 ++++++++++++------------ apps/deepl_integration/views.py | 1 + apps/lead/dataloaders.py | 16 +++++++++++ apps/lead/schema.py | 20 +++++++++++++ schema.graphql | 45 ++++++++++++++++++++++++------ 7 files changed, 97 insertions(+), 31 deletions(-) diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index ec86f3a1f5..8570798895 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -2,21 +2,21 @@ from deep.filter_set import generate_type_for_filter_set from .models import DraftEntry -from utils.graphene.filters import IDListFilter, MultipleInputFilter +from utils.graphene.filters import IDFilter, MultipleInputFilter from .enums import ( DraftEntryTypeEnum ) class DraftEntryFilterSet(django_filters.FilterSet): - leads = IDListFilter(field_name='lead') + leads = IDFilter(field_name='lead') draft_entry_types = MultipleInputFilter(DraftEntryTypeEnum, field_name='draft_entry_type') is_discarded = django_filters.BooleanFilter(method='filter_discarded') def filter_discarded(self, queryset, name, value): if value: return queryset.filter(is_discarded=value) - return queryset.filter(is_discarded = False) + return queryset.filter(is_discarded=False) class Meta: model = DraftEntry diff --git a/apps/assisted_tagging/schema.py b/apps/assisted_tagging/schema.py index 1338b5d37b..bf646c88ec 100644 --- a/apps/assisted_tagging/schema.py +++ b/apps/assisted_tagging/schema.py @@ -215,7 +215,7 @@ def resolve_missing_prediction_reviews(root, info, **kwargs): @staticmethod def resolve_related_geoareas(root, info, **kwargs): - return root.related_geoareas.all() # NOTE: Prefetched by DraftEntry + return root.related_geoareas.all() # NOTE: Prefetched by DraftEntry class DraftEntryByLeadType(DjangoObjectType): @@ -230,6 +230,7 @@ class DraftEntryByLeadType(DjangoObjectType): related_geoareas = graphene.List( graphene.NonNull(ProjectGeoAreaType) ) + is_discarded = graphene.Int(required=True) class Meta: model = DraftEntry diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index f7113a3e5a..6993da957d 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -509,11 +509,14 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode AssistedTaggingPrediction.objects.bulk_create(new_predictions) @classmethod + @transaction.atomic def save_data(cls, lead, data): draft_entry = DraftEntry.objects.filter(lead=lead, draft_entry_type=0) - if draft_entry: - raise serializers.ValidationError('Draft entry already exit') + if draft_entry.exists(): + raise serializers.ValidationError('Draft entries already exit') for model_preds in data['blocks']: + if not model_preds['relevant']: + continue classification = model_preds['classification'] current_tags_map = cls._get_or_create_tags_map([ tag @@ -527,22 +530,20 @@ def save_data(cls, lead, data): data['classification_model_info'] ]) - with transaction.atomic(): - if model_preds['relevant']: - draft = DraftEntry.objects.create( - project=lead.project, - lead=lead, - excerpt=model_preds['text'], - draft_entry_type=0 - ) - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS - lead.save() - draft.save() - model_version = models_version_map[ - (data['classification_model_info']['name'], data['classification_model_info']['version']) - ] - cls._process_model_preds(model_version, current_tags_map, draft, model_preds) - + draft = DraftEntry.objects.create( + project=lead.project, + lead=lead, + excerpt=model_preds['text'], + prediction_status=DraftEntry.PredictionStatus.DONE, + draft_entry_type=0 + ) + draft.save() + model_version = models_version_map[ + (data['classification_model_info']['name'], data['classification_model_info']['version']) + ] + cls._process_model_preds(model_version, current_tags_map, draft, model_preds) + lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS + lead.save(update_fields=('auto_entry_extraction_status',)) return lead diff --git a/apps/deepl_integration/views.py b/apps/deepl_integration/views.py index 264d0ce6ec..115e918e98 100644 --- a/apps/deepl_integration/views.py +++ b/apps/deepl_integration/views.py @@ -24,6 +24,7 @@ class BaseCallbackView(views.APIView): permission_classes = [permissions.AllowAny] def post(self, request, **_): + print(request.data) serializer = self.serializer(data=request.data) serializer.is_valid(raise_exception=True) serializer.save() diff --git a/apps/lead/dataloaders.py b/apps/lead/dataloaders.py index 408d70ca49..26edc5500e 100644 --- a/apps/lead/dataloaders.py +++ b/apps/lead/dataloaders.py @@ -13,6 +13,7 @@ from organization.dataloaders import OrganizationLoader from .models import Lead, LeadPreview, LeadGroup +from assisted_tagging.models import DraftEntry class LeadPreviewLoader(DataLoaderWithContext): @@ -103,6 +104,17 @@ def batch_load_fn(self, keys): return Promise.resolve([_map.get(key) for key in keys]) +class LeadDraftEntryDiscardCountLoader(DataLoaderWithContext): + def batch_load_fn(self, keys): + draft_entry_qs = DraftEntry.objects.filter(lead__in=keys, is_discarded=True).annotate(count = models.Count('id')) + _map = { + id: count for _id , count in draft_entry_qs + + } + + return Promise.resolve([_map.get(key) for key in keys]) + + class DataLoaders(WithContextMixin): @cached_property def lead_preview(self): @@ -127,3 +139,7 @@ def author_organizations(self): @cached_property def assessment_id(self): return LeadAssessmentIdLoader(context=self.context) + + @cached_property + def draftentry_discarded_count(self): + return LeadDraftEntryDiscardCountLoader(context=self.context) diff --git a/apps/lead/schema.py b/apps/lead/schema.py index c20392f1f1..43b79534c0 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -5,6 +5,7 @@ from django.db.models import QuerySet from graphene_django import DjangoObjectType, DjangoListField from graphene_django_extras import DjangoObjectField, PageGraphqlPagination +from assisted_tagging.models import DraftEntry from utils.graphene.pagination import NoOrderingPageGraphqlPagination from utils.graphene.enums import EnumDescription @@ -405,6 +406,19 @@ def resolve_share_view_url(root: Lead, info, **kwargs): return Permalink.lead_share_view(root.uuid) +class DraftEntryCountByLead(graphene.ObjectType): + undiscarded_draft_entry = graphene.Int(required=False) + discarded_draft_enrty = graphene.Int(required=False) + + @staticmethod + def resolve_discarded_draft_enrty(root, info, **kwargs): + return DraftEntry.objects.filter(lead=root.id, is_discarded=True).count() + + @staticmethod + def resolve_undiscarded_draft_entry(root, info, **kwargs): + return DraftEntry.objects.filter(lead=root.id, is_discarded=False).count() + + class LeadDetailType(LeadType): class Meta: model = Lead @@ -416,6 +430,7 @@ class Meta: ) entries = graphene.List(graphene.NonNull('entry.schema.EntryType')) + draft_entry_stat = graphene.Field(DraftEntryCountByLead) @staticmethod def resolve_entries(root, info, **kwargs): @@ -423,6 +438,10 @@ def resolve_entries(root, info, **kwargs): analysis_framework=info.context.active_project.analysis_framework, ).all() + @staticmethod + def resolve_draft_entry_stat(root, info, **kwargs): + return root + class LeadListType(CustomDjangoListObjectType): class Meta: @@ -462,6 +481,7 @@ class Query: emm_risk_factors = graphene.List(graphene.NonNull(EmmKeyRiskFactorType)) user_saved_lead_filter = graphene.Field(UserSavedLeadFilterType) + draft_entry_discarded_count = graphene.Field(DraftEntryCountByLead) @staticmethod def resolve_leads(root, info, **kwargs) -> QuerySet: diff --git a/schema.graphql b/schema.graphql index 275d205003..3f468d2b54 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1005,6 +1005,8 @@ type AppEnumCollection { ConnectorLeadExtractionStatus: [AppEnumCollectionConnectorLeadExtractionStatus!] DraftEntryPredictionStatus: [AppEnumCollectionDraftEntryPredictionStatus!] AssistedTaggingPredictionDataType: [AppEnumCollectionAssistedTaggingPredictionDataType!] + DraftEntryDraftEntryType: [AppEnumCollectionDraftEntryDraftEntryType!] + LeadAutoEntryExtractionStatus: [AppEnumCollectionLeadAutoEntryExtractionStatus!] UnusedAssessmentMethodologyProtectionInfo: [AppEnumCollectionUnusedAssessmentMethodologyProtectionInfo!] } @@ -1104,6 +1106,12 @@ type AppEnumCollectionDiscardedEntryTag { description: String } +type AppEnumCollectionDraftEntryDraftEntryType { + enum: DraftEntryTypeEnum! + label: String! + description: String +} + type AppEnumCollectionDraftEntryPredictionStatus { enum: DraftEntryPredictionStatusEnum! label: String! @@ -1194,6 +1202,12 @@ type AppEnumCollectionGroupMembershipRole { description: String } +type AppEnumCollectionLeadAutoEntryExtractionStatus { + enum: AutoEntryExtractionTypeEnum! + label: String! + description: String +} + type AppEnumCollectionLeadConfidentiality { enum: LeadConfidentialityEnum! label: String! @@ -1383,8 +1397,8 @@ type AssistedTaggingMutationType { wrongPredictionReviewCreate(data: WrongPredictionReviewInputType!): CreateWrongPredictionReview missingPredictionReviewDelete(id: ID!): DeleteMissingPredictionReview wrongPredictionReviewDelete(id: ID!): DeleteWrongPredictionReview - autoDraftEntryCreate(data: AutoDraftEntryInputType!): CreateAutoDraftEntry - discardDraftEntry(data: AutoDraftEntryInputType!, id: ID!): DiscardDraftEntry + autoDraftEntryCreate(data: TriggerDraftEntryInputType!): CreateAutoDraftEntry + discardDraftEntry(data: UpdateDraftEntryInputType!, id: ID!): DiscardDraftEntry } enum AssistedTaggingPredictionDataTypeEnum { @@ -1445,11 +1459,6 @@ type AttributeType { geoSelectedOptions: [ProjectGeoAreaType!] } -input AutoDraftEntryInputType { - lead: ID! - isDiscarded: Boolean -} - enum AutoEntryExtractionTypeEnum { NONE STARTED @@ -1977,11 +1986,18 @@ type DraftEntryByLeadType { predictions: [AssistedTaggingPredictionType!] missingPredictionReviews: [MissingPredictionReviewType!] relatedGeoareas: [ProjectGeoAreaType!] + isDiscarded: Int! +} + +type DraftEntryCountByLead { + undiscardedDraftEntry: Int + discardedDraftEnrty: Int } input DraftEntryFilterDataInputType { - lead: [ID!] - draftEntryType: [DraftEntryTypeEnum!] + leads: ID + draftEntryTypes: [DraftEntryTypeEnum!] + isDiscarded: Boolean } input DraftEntryInputType { @@ -2751,6 +2767,7 @@ type LeadDetailType { duplicateLeadsCount: Int shareViewUrl: String! entries: [EntryType!] + draftEntryStat: DraftEntryCountByLead } input LeadEMMTriggerInputType { @@ -3312,6 +3329,7 @@ type ProjectDetailType { emmKeywords: [EmmKeyWordType!] emmRiskFactors: [EmmKeyRiskFactorType!] userSavedLeadFilter: UserSavedLeadFilterType + draftEntryDiscardedCount: DraftEntryCountByLead activityLog: GenericScalar recentActiveUsers: [UserEntityDateType!] topSourcers: [UserEntityCountType!] @@ -3995,6 +4013,10 @@ type TriggerAnalysisTopicModel { result: AnalysisTopicModelType } +input TriggerDraftEntryInputType { + lead: ID! +} + type TriggerUnifiedConnector { errors: [GenericScalar!] ok: Boolean @@ -4094,6 +4116,11 @@ type UpdateConnectorSourceLead { result: ConnectorSourceLeadType } +input UpdateDraftEntryInputType { + lead: ID! + isDiscarded: Boolean +} + type UpdateEntry { errors: [GenericScalar!] ok: Boolean From d4ccbea25e1bbc9f8927a8a625b7653e7439b068 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Fri, 15 Dec 2023 15:09:02 +0545 Subject: [PATCH 19/24] mutation changed refactor the code --- apps/assisted_tagging/enums.py | 2 +- apps/assisted_tagging/filters.py | 7 +-- apps/assisted_tagging/models.py | 4 +- apps/assisted_tagging/mutation.py | 20 +++--- apps/assisted_tagging/schema.py | 63 ++----------------- apps/assisted_tagging/serializers.py | 21 ++++--- apps/assisted_tagging/tasks.py | 4 +- apps/deepl_integration/handlers.py | 9 +-- apps/deepl_integration/serializers.py | 19 +----- apps/lead/dataloaders.py | 33 +++++++--- .../migrations/0054_auto_20231218_0552.py | 23 +++++++ apps/lead/schema.py | 14 +---- deep/deepl.py | 2 +- schema.graphql | 45 +++++-------- 14 files changed, 110 insertions(+), 156 deletions(-) create mode 100644 apps/lead/migrations/0054_auto_20231218_0552.py diff --git a/apps/assisted_tagging/enums.py b/apps/assisted_tagging/enums.py index 2bd8a1f0c2..9ecb13537b 100644 --- a/apps/assisted_tagging/enums.py +++ b/apps/assisted_tagging/enums.py @@ -16,7 +16,7 @@ AssistedTaggingPredictionDataTypeEnum = convert_enum_to_graphene_enum( AssistedTaggingPrediction.DataType, name='AssistedTaggingPredictionDataTypeEnum') DraftEntryTypeEnum = convert_enum_to_graphene_enum( - DraftEntry.DraftEntryType, name="DraftEntryTypeEnum" + DraftEntry.Type, name="DraftEntryTypeEnum" ) AutoEntryExtractionTypeEnum = convert_enum_to_graphene_enum( Lead.AutoExtractionStatus, name="AutoEntryExtractionTypeEnum" diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index 8570798895..32885b7453 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -11,12 +11,7 @@ class DraftEntryFilterSet(django_filters.FilterSet): leads = IDFilter(field_name='lead') draft_entry_types = MultipleInputFilter(DraftEntryTypeEnum, field_name='draft_entry_type') - is_discarded = django_filters.BooleanFilter(method='filter_discarded') - - def filter_discarded(self, queryset, name, value): - if value: - return queryset.filter(is_discarded=value) - return queryset.filter(is_discarded=False) + is_discarded = django_filters.BooleanFilter() class Meta: model = DraftEntry diff --git a/apps/assisted_tagging/models.py b/apps/assisted_tagging/models.py index 5026079e14..8764cdce42 100644 --- a/apps/assisted_tagging/models.py +++ b/apps/assisted_tagging/models.py @@ -90,7 +90,7 @@ class PredictionStatus(models.IntegerChoices): DONE = 2, 'Done' SEND_FAILED = 3, 'Send Failed' - class DraftEntryType(models.IntegerChoices): + class Type(models.IntegerChoices): AUTO = 0, 'Auto Extraction' # NLP defiend extraction text MANUAL = 1, 'Manual Extraction' # manaul defined extraction text @@ -102,7 +102,7 @@ class DraftEntryType(models.IntegerChoices): prediction_received_at = models.DateTimeField(null=True, blank=True) # Additional attribues related_geoareas = models.ManyToManyField(GeoArea, blank=True) - draft_entry_type = models.SmallIntegerField(choices=DraftEntryType.choices, default=DraftEntryType.MANUAL) + draft_entry_type = models.SmallIntegerField(choices=Type.choices, default=Type.MANUAL) is_discarded = models.BooleanField(default=False) def __str__(self): diff --git a/apps/assisted_tagging/mutation.py b/apps/assisted_tagging/mutation.py index 075c9f873b..2b1a6df3ab 100644 --- a/apps/assisted_tagging/mutation.py +++ b/apps/assisted_tagging/mutation.py @@ -21,7 +21,7 @@ DraftEntryGqlSerializer, WrongPredictionReviewGqlSerializer, MissingPredictionReviewGqlSerializer, - TriggeredDraftEntryGqlSerializer, + TriggerDraftEntryGqlSerializer, UpdateDraftEntrySerializer ) @@ -41,9 +41,9 @@ serializer_class=MissingPredictionReviewGqlSerializer, ) -TriggerDraftEntryInputType = generate_input_type_for_serializer( - "TriggerDraftEntryInputType", - serializer_class=TriggeredDraftEntryGqlSerializer +TriggerAutoDraftEntryInputType = generate_input_type_for_serializer( + "TriggerAutoDraftEntryInputType", + serializer_class=TriggerDraftEntryGqlSerializer ) UpdateDraftEntryInputType = generate_input_type_for_serializer( @@ -111,16 +111,16 @@ def filter_queryset(cls, qs, info): # auto draft_entry_create -class CreateAutoDraftEntry(PsGrapheneMutation): +class TriggerAutoDraftEntry(PsGrapheneMutation): class Arguments: - data = TriggerDraftEntryInputType(required=True) + data = TriggerAutoDraftEntryInputType(required=True) model = DraftEntry - serializer_class = TriggeredDraftEntryGqlSerializer + serializer_class = TriggerDraftEntryGqlSerializer result = graphene.Field(DraftEntryType) permissions = [PP.Permission.CREATE_ENTRY] -class DiscardDraftEntry(PsGrapheneMutation): +class UpdateDraftEntry(PsGrapheneMutation): class Arguments: data = UpdateDraftEntryInputType(required=True) id = graphene.ID(required=True) @@ -136,5 +136,5 @@ class AssistedTaggingMutationType(graphene.ObjectType): wrong_prediction_review_create = CreateWrongPredictionReview.Field() missing_prediction_review_delete = DeleteMissingPredictionReview.Field() wrong_prediction_review_delete = DeleteWrongPredictionReview.Field() - auto_draft_entry_create = CreateAutoDraftEntry.Field() - discard_draft_entry = DiscardDraftEntry.Field() + trigger_auto_draft_entry = TriggerAutoDraftEntry.Field() + update_draft_entry = UpdateDraftEntry.Field() diff --git a/apps/assisted_tagging/schema.py b/apps/assisted_tagging/schema.py index bf646c88ec..c445c33958 100644 --- a/apps/assisted_tagging/schema.py +++ b/apps/assisted_tagging/schema.py @@ -25,7 +25,7 @@ from .enums import ( DraftEntryPredictionStatusEnum, AssistedTaggingPredictionDataTypeEnum, - AutoEntryExtractionTypeEnum + AutoEntryExtractionTypeEnum, ) @@ -188,8 +188,8 @@ class Meta: ) @staticmethod - def get_custom_queryset(queryset, info, **kwargs): - return get_draft_entry_qs(info).prefetch_related( + def get_custom_queryset(root, info, _filter, **kwargs): + return get_draft_entry_with_filter_qs(info, _filter).prefetch_related( Prefetch( 'predictions', queryset=AssistedTaggingPrediction.objects.order_by('id'), @@ -218,59 +218,6 @@ def resolve_related_geoareas(root, info, **kwargs): return root.related_geoareas.all() # NOTE: Prefetched by DraftEntry -class DraftEntryByLeadType(DjangoObjectType): - prediction_status = graphene.Field(DraftEntryPredictionStatusEnum, required=True) - prediction_status_display = EnumDescription(source='get_prediction_status_display', required=True) - predictions = graphene.List( - graphene.NonNull(AssistedTaggingPredictionType) - ) - missing_prediction_reviews = graphene.List( - graphene.NonNull(MissingPredictionReviewType), - ) - related_geoareas = graphene.List( - graphene.NonNull(ProjectGeoAreaType) - ) - is_discarded = graphene.Int(required=True) - - class Meta: - model = DraftEntry - only_fields = ( - 'id', - 'excerpt', - 'prediction_received_at', - ) - - @staticmethod - def custom_queryset(queryset, info, _filter, **kwargs): - return get_draft_entry_with_filter_qs(info, _filter).prefetch_related( - Prefetch( - 'predictions', - queryset=AssistedTaggingPrediction.objects.order_by('id'), - ), - Prefetch( - 'related_geoareas', - queryset=get_geo_area_queryset_for_project_geo_area_type().order_by('id'), - ), - 'predictions__model_version', - 'predictions__model_version__model', - 'predictions__wrong_prediction_reviews', - 'missing_prediction_reviews', - 'related_geoareas', - ) - - @staticmethod - def resolve_predictions(root, info, **kwargs): - return root.predictions.filter(is_selected=True) # NOTE: Prefetched by DraftEntry - - @staticmethod - def resolve_missing_prediction_reviews(root, info, **kwargs): - return root.missing_prediction_reviews.all() # NOTE: Prefetched by DraftEntry - - @staticmethod - def resolve_related_geoareas(root, info, **kwargs): - return root.related_geoareas.all() - - class AutoextractionStatusType(graphene.ObjectType): auto_entry_extraction_status = graphene.Field(AutoEntryExtractionTypeEnum, required=True) @@ -282,11 +229,11 @@ def custom_queryset(root, info, lead_id): class AssistedTaggingQueryType(graphene.ObjectType): draft_entry = DjangoObjectField(DraftEntryType) - draft_entry_by_leads = DjangoListField(DraftEntryByLeadType, filter=DraftEntryFilterDataInputType()) + draft_entry_by_leads = DjangoListField(DraftEntryType, filter=DraftEntryFilterDataInputType()) extraction_status_by_lead = graphene.Field(AutoextractionStatusType, lead_id=graphene.ID(required=True)) def resolve_draft_entry_by_leads(root, info, filter): - return DraftEntryByLeadType.custom_queryset(root, info, filter) + return DraftEntryType.get_custom_queryset(root, info, filter) def resolve_extraction_status_by_lead(root, info, lead_id): return AutoextractionStatusType.custom_queryset(root, info, lead_id) diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index c08bdb8e5b..a609db2b1a 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -27,6 +27,8 @@ def validate_lead(self, lead): raise serializers.ValidationError('Assisted tagging is disabled for the Framework used by this project.') if self.project.is_private: raise serializers.ValidationError('Assisted tagging is not available for private projects.') + if lead.confidentiality == Lead.Confidentiality.CONFIDENTIAL: + raise serializers.ValidationError('Assisted tagging is not available for confidential lead') return lead @@ -42,14 +44,16 @@ class Meta: def create(self, data): # Use already existing draft entry if found + project = data['lead'].project already_existing_draft_entry = DraftEntry.get_existing_draft_entry( - data['project'], + project, data['lead'], data['excerpt'], ) if already_existing_draft_entry: return already_existing_draft_entry # Create new one and send trigger to deepl. + data['project'] = project instance = super().create(data) transaction.on_commit( lambda: trigger_request_for_draft_entry_task.delay(instance.pk) @@ -130,9 +134,9 @@ def validate(self, data): return data -class TriggeredDraftEntryGqlSerializer( - DraftEntryBaseSerializer, - ProjectPropertySerializerMixin, +class TriggerDraftEntryGqlSerializer( + DraftEntryBaseSerializer, + ProjectPropertySerializerMixin, UserResourceCreatedMixin, serializers.ModelSerializer): class Meta: model = DraftEntry @@ -141,14 +145,17 @@ class Meta: ) def create(self, data): - lead = Lead.objects.get(id=self.data['lead']) + lead = data['lead'] + if lead.leadpreview.text_extraction_id is None: + raise serializers.DjangoValidationError("Assisted tagging is not available in old lead") if lead.auto_entry_extraction_status == (Lead.AutoExtractionStatus.SUCCESS): raise serializers.DjangoValidationError("Already Tiggered") if not lead.leadpreview.text_extract: raise serializers.DjangoValidationError('Simplifed Text is empty') - draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], draft_entry_type=0) + draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], + draft_entry_type=DraftEntry.Type.AUTO) if draft_entry.exists(): - raise serializers.DjangoValidationError('Draft entry already exit') + raise serializers.DjangoValidationError('Draft entry already exists') # Use already existing draft entry if found # Create new one and send trigger to deepl lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py index eceb2ac70a..b02b50f887 100644 --- a/apps/assisted_tagging/tasks.py +++ b/apps/assisted_tagging/tasks.py @@ -98,8 +98,8 @@ def trigger_request_for_draft_entry_task(draft_entry_id): @shared_task @redis_lock('trigger_request_for_auto_draft_entry_task_{0}', 60 * 60 * 0.5) -def trigger_request_for_auto_draft_entry_task(lead): - lead = Lead.objects.get(id=lead) +def trigger_request_for_auto_draft_entry_task(lead_id): + lead = Lead.objects.get(id=lead_id) return AutoAssistedTaggingDraftEntryHandler.auto_trigger_request_to_extractor(lead) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 6993da957d..f3887f452d 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -204,7 +204,7 @@ def send_trigger_request_to_extractor(cls, draft_entry): }, ) -# --- Callback logics + # --- Callback logics @staticmethod def _get_or_create_models_version(models_data): def get_versions_map(): @@ -280,7 +280,7 @@ def get_tags_map(): @classmethod def _process_model_preds(cls, model_version, current_tags_map, draft_entry, model_prediction): prediction_status = model_prediction['prediction_status'] - if not prediction_status: # If 0 no tags are provided + if not prediction_status: # If False no tags are provided return tags = model_prediction.get('model_tags', {}) # NLP TagId @@ -325,14 +325,12 @@ def save_data(cls, draft_entry, data): # Save if new tags are provided current_tags_map = cls._get_or_create_tags_map([ tag - # for prediction in model_preds['model_tags'] for category_tag, tags in model_preds['model_tags'].items() for tag in [ category_tag, *tags.keys(), ] ]) - print(current_tags_map) models_version_map = cls._get_or_create_models_version( [ model_preds['model_info'] @@ -343,7 +341,6 @@ def save_data(cls, draft_entry, data): draft_entry.calculated_at = timezone.now() # for prediction in model_preds: model_version = models_version_map[(model_preds['model_info']['id'], model_preds['model_info']['version'])] - print(model_preds) cls._process_model_preds(model_version, current_tags_map, draft_entry, model_preds) draft_entry.prediction_status = DraftEntry.PredictionStatus.DONE draft_entry.save_geo_data() @@ -362,7 +359,7 @@ def auto_trigger_request_to_extractor(cls, lead): "documents": [ { "client_id": cls.get_client_id(lead), # static clientid for mock - "text_extraction_id": lead_preview.text_extraction_id + "text_extraction_id": str(lead_preview.text_extraction_id) } ], "callback_url": cls.get_callback_url() diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 452849f9dd..14d10d410d 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -72,10 +72,10 @@ class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): child=serializers.CharField(allow_blank=True), required=False, default=[], ) - text_path = serializers.CharField(required=False) - total_words_count = serializers.IntegerField(required=False, default=0) + text_path = serializers.CharField(required=False, allow_null=True) + total_words_count = serializers.IntegerField(required=False, default=0, allow_null=True) total_pages = serializers.IntegerField(required=False, default=0) - text_extraction_id = serializers.CharField(required=False) + text_extraction_id = serializers.CharField(required=True) nlp_handler = LeadExtractionHandler def validate(self, data): @@ -221,7 +221,6 @@ class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): class AssistedTaggingDraftEntryPredictionCallbackSerializer(BaseCallbackSerializer): - # model_tags = AssistedTaggingModelPredictionCallbackSerializer() model_tags = serializers.DictField(child=serializers.DictField()) prediction_status = serializers.BooleanField() model_info = serializers.DictField() @@ -241,7 +240,6 @@ class AutoAssistedBlockPredicationCallbackSerializer(serializers.Serializer): text = serializers.CharField() relevant = serializers.BooleanField() prediction_status = serializers.BooleanField() - # classification = AutoAssistedTaggingModelPredicationCallBackSerializer() classification = serializers.DictField(child=serializers.DictField()) @@ -259,17 +257,6 @@ def create(self, validated_data): validated_data ) -# class AutoAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): -# blocks = AutoAssistedBlockPredicationCallbackSerializer(many=True) -# classification_model_info = serializers.DictField() -# nlp_handler = AutoAssistedTaggingDraftEntryHandler - -# def create(self, validated_data): -# return self.nlp_handler.save_data( -# validated_data['object'], -# validated_data -# ) - class EntriesCollectionBaseCallbackSerializer(DeeplServerBaseCallbackSerializer): model: Type[DeeplTrackBaseModel] diff --git a/apps/lead/dataloaders.py b/apps/lead/dataloaders.py index 26edc5500e..bef4bdcf9d 100644 --- a/apps/lead/dataloaders.py +++ b/apps/lead/dataloaders.py @@ -104,15 +104,32 @@ def batch_load_fn(self, keys): return Promise.resolve([_map.get(key) for key in keys]) -class LeadDraftEntryDiscardCountLoader(DataLoaderWithContext): +class LeadDraftEntryCountLoader(DataLoaderWithContext): def batch_load_fn(self, keys): - draft_entry_qs = DraftEntry.objects.filter(lead__in=keys, is_discarded=True).annotate(count = models.Count('id')) + stat_qs = DraftEntry.objects\ + .filter(lead__in=keys)\ + .order_by('lead').values('lead')\ + .annotate( + discarded_draft_entry=models.functions.Coalesce( + models.Count( + 'id', + filter=models.Q(is_discarded=True) + ), + 0, + ), + undiscarded_draft_entry=models.functions.Coalesce( + models.Count( + 'id', + filter=models.Q(is_discarded=False) + ), + 0, + ), + ).values('lead_id', 'undiscarded_draft_entry', 'discarded_draft_entry') _map = { - id: count for _id , count in draft_entry_qs - + stat.pop('lead_id'): stat + for stat in stat_qs } - - return Promise.resolve([_map.get(key) for key in keys]) + return Promise.resolve([_map.get(key, _map) for key in keys]) class DataLoaders(WithContextMixin): @@ -141,5 +158,5 @@ def assessment_id(self): return LeadAssessmentIdLoader(context=self.context) @cached_property - def draftentry_discarded_count(self): - return LeadDraftEntryDiscardCountLoader(context=self.context) + def draftentry_count(self): + return LeadDraftEntryCountLoader(context=self.context) diff --git a/apps/lead/migrations/0054_auto_20231218_0552.py b/apps/lead/migrations/0054_auto_20231218_0552.py new file mode 100644 index 0000000000..72409a0663 --- /dev/null +++ b/apps/lead/migrations/0054_auto_20231218_0552.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.17 on 2023-12-18 05:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lead', '0053_alter_lead_auto_entry_extraction_status'), + ] + + operations = [ + migrations.AlterField( + model_name='leadpreview', + name='entry_extraction_id', + field=models.UUIDField(blank=True, null=True), + ), + migrations.AlterField( + model_name='leadpreview', + name='text_extraction_id', + field=models.UUIDField(blank=True, null=True), + ), + ] diff --git a/apps/lead/schema.py b/apps/lead/schema.py index 43b79534c0..354d07bae0 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -5,7 +5,6 @@ from django.db.models import QuerySet from graphene_django import DjangoObjectType, DjangoListField from graphene_django_extras import DjangoObjectField, PageGraphqlPagination -from assisted_tagging.models import DraftEntry from utils.graphene.pagination import NoOrderingPageGraphqlPagination from utils.graphene.enums import EnumDescription @@ -210,6 +209,7 @@ class Meta: 'thumbnail_width', 'word_count', 'page_count', + 'text_extraction_id' # 'classified_doc_id', # 'classification_status', ) @@ -408,15 +408,7 @@ def resolve_share_view_url(root: Lead, info, **kwargs): class DraftEntryCountByLead(graphene.ObjectType): undiscarded_draft_entry = graphene.Int(required=False) - discarded_draft_enrty = graphene.Int(required=False) - - @staticmethod - def resolve_discarded_draft_enrty(root, info, **kwargs): - return DraftEntry.objects.filter(lead=root.id, is_discarded=True).count() - - @staticmethod - def resolve_undiscarded_draft_entry(root, info, **kwargs): - return DraftEntry.objects.filter(lead=root.id, is_discarded=False).count() + discarded_draft_entry = graphene.Int(required=False) class LeadDetailType(LeadType): @@ -440,7 +432,7 @@ def resolve_entries(root, info, **kwargs): @staticmethod def resolve_draft_entry_stat(root, info, **kwargs): - return root + return info.context.dl.lead.draftentry_count.load(root.pk) class LeadListType(CustomDjangoListObjectType): diff --git a/deep/deepl.py b/deep/deepl.py index 5233d91f3a..32f6d3c135 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -8,7 +8,7 @@ class DeeplServiceEndpoint(): # DEEPL Service Endpoints (Existing/Legacy) # NOTE: This will be moved to server endpoints in near future - ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/model_info' + ASSISTED_TAGGING_MODELS_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/model_info' # DEEPL Server Endpoints (New) ASSISTED_TAGGING_TAGS_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/nlp-tags/' diff --git a/schema.graphql b/schema.graphql index 3f468d2b54..a59dd877e3 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1397,8 +1397,8 @@ type AssistedTaggingMutationType { wrongPredictionReviewCreate(data: WrongPredictionReviewInputType!): CreateWrongPredictionReview missingPredictionReviewDelete(id: ID!): DeleteMissingPredictionReview wrongPredictionReviewDelete(id: ID!): DeleteWrongPredictionReview - autoDraftEntryCreate(data: TriggerDraftEntryInputType!): CreateAutoDraftEntry - discardDraftEntry(data: UpdateDraftEntryInputType!, id: ID!): DiscardDraftEntry + triggerAutoDraftEntry(data: TriggerAutoDraftEntryInputType!): TriggerAutoDraftEntry + updateDraftEntry(data: UpdateDraftEntryInputType!, id: ID!): UpdateDraftEntry } enum AssistedTaggingPredictionDataTypeEnum { @@ -1424,7 +1424,7 @@ type AssistedTaggingPredictionType { type AssistedTaggingQueryType { draftEntry(id: ID!): DraftEntryType - draftEntryByLeads(filter: DraftEntryFilterDataInputType): [DraftEntryByLeadType!] + draftEntryByLeads(filter: DraftEntryFilterDataInputType): [DraftEntryType!] extractionStatusByLead(leadId: ID!): AutoextractionStatusType } @@ -1762,12 +1762,6 @@ type CreateAnalysisReportUpload { result: AnalysisReportUploadType } -type CreateAutoDraftEntry { - errors: [GenericScalar!] - ok: Boolean - result: DraftEntryType -} - type CreateDraftEntry { errors: [GenericScalar!] ok: Boolean @@ -1929,12 +1923,6 @@ type DeleteWrongPredictionReview { result: WrongPredictionReviewType } -type DiscardDraftEntry { - errors: [GenericScalar!] - ok: Boolean - result: DraftEntryType -} - input DiscardedEntryCreateInputType { id: ID analysisPillar: ID! @@ -1977,18 +1965,6 @@ type DjangoDebugSQL { encoding: String } -type DraftEntryByLeadType { - id: ID! - excerpt: String! - predictionReceivedAt: DateTime - predictionStatus: DraftEntryPredictionStatusEnum! - predictionStatusDisplay: EnumDescription! - predictions: [AssistedTaggingPredictionType!] - missingPredictionReviews: [MissingPredictionReviewType!] - relatedGeoareas: [ProjectGeoAreaType!] - isDiscarded: Int! -} - type DraftEntryCountByLead { undiscardedDraftEntry: Int discardedDraftEnrty: Int @@ -2893,6 +2869,7 @@ type LeadPreviewType { thumbnailWidth: Int wordCount: Int pageCount: Int + textExtractionId: UUID } enum LeadPriorityEnum { @@ -4013,7 +3990,13 @@ type TriggerAnalysisTopicModel { result: AnalysisTopicModelType } -input TriggerDraftEntryInputType { +type TriggerAutoDraftEntry { + errors: [GenericScalar!] + ok: Boolean + result: DraftEntryType +} + +input TriggerAutoDraftEntryInputType { lead: ID! } @@ -4116,6 +4099,12 @@ type UpdateConnectorSourceLead { result: ConnectorSourceLeadType } +type UpdateDraftEntry { + errors: [GenericScalar!] + ok: Boolean + result: DraftEntryType +} + input UpdateDraftEntryInputType { lead: ID! isDiscarded: Boolean From 45a5a8e8066972455005cc64cc7eeba4b0631ca2 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Wed, 20 Dec 2023 17:07:23 +0545 Subject: [PATCH 20/24] pagination in draft entries --- apps/assisted_tagging/enums.py | 2 +- apps/assisted_tagging/filters.py | 4 +- ...rename_draft_entry_type_draftentry_type.py | 18 +++++ apps/assisted_tagging/models.py | 2 +- apps/assisted_tagging/mutation.py | 3 +- apps/assisted_tagging/schema.py | 72 ++++++++++++------- apps/assisted_tagging/serializers.py | 8 +-- apps/deepl_integration/handlers.py | 8 +-- apps/deepl_integration/serializers.py | 5 +- deep/deepl.py | 2 +- schema.graphql | 8 +-- 11 files changed, 82 insertions(+), 50 deletions(-) create mode 100644 apps/assisted_tagging/migrations/0013_rename_draft_entry_type_draftentry_type.py diff --git a/apps/assisted_tagging/enums.py b/apps/assisted_tagging/enums.py index 9ecb13537b..d73f9712a9 100644 --- a/apps/assisted_tagging/enums.py +++ b/apps/assisted_tagging/enums.py @@ -27,7 +27,7 @@ for field, enum in ( (DraftEntry.prediction_status, DraftEntryPredictionStatusEnum), (AssistedTaggingPrediction.data_type, AssistedTaggingPredictionDataTypeEnum), - (DraftEntry.draft_entry_type, DraftEntryTypeEnum), + (DraftEntry.type, DraftEntryTypeEnum), (Lead.auto_entry_extraction_status, AutoEntryExtractionTypeEnum), ) } diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index 32885b7453..1d46eb698e 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -9,8 +9,8 @@ class DraftEntryFilterSet(django_filters.FilterSet): - leads = IDFilter(field_name='lead') - draft_entry_types = MultipleInputFilter(DraftEntryTypeEnum, field_name='draft_entry_type') + lead = IDFilter(field_name='lead') + draft_entry_types = MultipleInputFilter(DraftEntryTypeEnum, field_name='type') is_discarded = django_filters.BooleanFilter() class Meta: diff --git a/apps/assisted_tagging/migrations/0013_rename_draft_entry_type_draftentry_type.py b/apps/assisted_tagging/migrations/0013_rename_draft_entry_type_draftentry_type.py new file mode 100644 index 0000000000..9d479d4355 --- /dev/null +++ b/apps/assisted_tagging/migrations/0013_rename_draft_entry_type_draftentry_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-12-20 11:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assisted_tagging', '0012_draftentry_is_discarded'), + ] + + operations = [ + migrations.RenameField( + model_name='draftentry', + old_name='draft_entry_type', + new_name='type', + ), + ] diff --git a/apps/assisted_tagging/models.py b/apps/assisted_tagging/models.py index 8764cdce42..ac86c08fc2 100644 --- a/apps/assisted_tagging/models.py +++ b/apps/assisted_tagging/models.py @@ -102,7 +102,7 @@ class Type(models.IntegerChoices): prediction_received_at = models.DateTimeField(null=True, blank=True) # Additional attribues related_geoareas = models.ManyToManyField(GeoArea, blank=True) - draft_entry_type = models.SmallIntegerField(choices=Type.choices, default=Type.MANUAL) + type = models.SmallIntegerField(choices=Type.choices, default=Type.MANUAL) is_discarded = models.BooleanField(default=False) def __str__(self): diff --git a/apps/assisted_tagging/mutation.py b/apps/assisted_tagging/mutation.py index 2b1a6df3ab..cb33447cf0 100644 --- a/apps/assisted_tagging/mutation.py +++ b/apps/assisted_tagging/mutation.py @@ -7,6 +7,7 @@ ) from deep.permissions import ProjectPermissions as PP +from lead.schema import LeadType from .models import ( DraftEntry, MissingPredictionReview, @@ -116,7 +117,7 @@ class Arguments: data = TriggerAutoDraftEntryInputType(required=True) model = DraftEntry serializer_class = TriggerDraftEntryGqlSerializer - result = graphene.Field(DraftEntryType) + result = graphene.Field(LeadType) permissions = [PP.Permission.CREATE_ENTRY] diff --git a/apps/assisted_tagging/schema.py b/apps/assisted_tagging/schema.py index c445c33958..1dcf89cabd 100644 --- a/apps/assisted_tagging/schema.py +++ b/apps/assisted_tagging/schema.py @@ -1,8 +1,8 @@ import graphene -from graphene_django import DjangoObjectType, DjangoListField +from graphene_django import DjangoObjectType from graphene_django_extras import DjangoObjectField from django.db.models import Prefetch -from assisted_tagging.filters import DraftEntryFilterDataInputType, DraftEntryFilterSet +from assisted_tagging.filters import DraftEntryFilterSet from utils.graphene.enums import EnumDescription from user_resource.schema import UserResourceMixin @@ -12,6 +12,9 @@ ProjectGeoAreaType, get_geo_area_queryset_for_project_geo_area_type, ) +from utils.graphene.fields import DjangoPaginatedListObjectField +from utils.graphene.pagination import NoOrderingPageGraphqlPagination +from utils.graphene.types import CustomDjangoListObjectType from .models import ( DraftEntry, AssistedTaggingModel, @@ -99,10 +102,24 @@ def resolve_prediction_tags(root, info, **kwargs): # -- Project Level -def get_draft_entry_qs(info): +def get_draft_entry_qs(info): # TODO use dataloder qs = DraftEntry.objects.filter(project=info.context.active_project) if PP.check_permission(info, PP.Permission.VIEW_ENTRY): - return qs + return qs.prefetch_related( + Prefetch( + 'predictions', + queryset=AssistedTaggingPrediction.objects.filter(is_selected=True).order_by('id'), + ), + Prefetch( + 'related_geoareas', + queryset=get_geo_area_queryset_for_project_geo_area_type().order_by('id'), + ), + 'predictions__model_version', + 'predictions__model_version__model', + 'predictions__wrong_prediction_reviews', + 'missing_prediction_reviews', + 'related_geoareas', + ) return qs.none() @@ -188,22 +205,8 @@ class Meta: ) @staticmethod - def get_custom_queryset(root, info, _filter, **kwargs): - return get_draft_entry_with_filter_qs(info, _filter).prefetch_related( - Prefetch( - 'predictions', - queryset=AssistedTaggingPrediction.objects.order_by('id'), - ), - Prefetch( - 'related_geoareas', - queryset=get_geo_area_queryset_for_project_geo_area_type().order_by('id'), - ), - 'predictions__model_version', - 'predictions__model_version__model', - 'predictions__wrong_prediction_reviews', - 'missing_prediction_reviews', - 'related_geoareas', - ) + def get_custom_queryset(root, info, **kwargs): + return get_draft_entry_qs(info) @staticmethod def resolve_predictions(root, info, **kwargs): @@ -218,7 +221,13 @@ def resolve_related_geoareas(root, info, **kwargs): return root.related_geoareas.all() # NOTE: Prefetched by DraftEntry -class AutoextractionStatusType(graphene.ObjectType): +class DraftEntryListType(CustomDjangoListObjectType): + class Meta: + model = DraftEntry + filterset_class = DraftEntryFilterSet + + +class AutoExtractionStatusType(graphene.ObjectType): auto_entry_extraction_status = graphene.Field(AutoEntryExtractionTypeEnum, required=True) @staticmethod @@ -229,11 +238,22 @@ def custom_queryset(root, info, lead_id): class AssistedTaggingQueryType(graphene.ObjectType): draft_entry = DjangoObjectField(DraftEntryType) - draft_entry_by_leads = DjangoListField(DraftEntryType, filter=DraftEntryFilterDataInputType()) - extraction_status_by_lead = graphene.Field(AutoextractionStatusType, lead_id=graphene.ID(required=True)) + draft_entries = DjangoPaginatedListObjectField( + DraftEntryListType, + pagination=NoOrderingPageGraphqlPagination( + page_size_query_param='pageSize', + ), + ) + extraction_status_by_lead = graphene.Field(AutoExtractionStatusType, lead_id=graphene.ID(required=True)) - def resolve_draft_entry_by_leads(root, info, filter): - return DraftEntryType.get_custom_queryset(root, info, filter) + @staticmethod + def resolve_draft_entry(root, info, **kwargs): + return DraftEntryType.get_custom_queryset(root, info, **kwargs) + + @staticmethod + def resolve_draft_entries(root, info, **_): + return get_draft_entry_qs(info) + @staticmethod def resolve_extraction_status_by_lead(root, info, lead_id): - return AutoextractionStatusType.custom_queryset(root, info, lead_id) + return AutoExtractionStatusType.custom_queryset(root, info, lead_id) diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index a609db2b1a..fb1a0702b3 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -27,8 +27,8 @@ def validate_lead(self, lead): raise serializers.ValidationError('Assisted tagging is disabled for the Framework used by this project.') if self.project.is_private: raise serializers.ValidationError('Assisted tagging is not available for private projects.') - if lead.confidentiality == Lead.Confidentiality.CONFIDENTIAL: - raise serializers.ValidationError('Assisted tagging is not available for confidential lead') + if lead.confidentiality == (Lead.Confidentiality.CONFIDENTIAL or Lead.Confidentiality.RESTRICTED): + raise serializers.ValidationError('Assisted tagging is not available for confidential lead or restricated') return lead @@ -153,7 +153,7 @@ def create(self, data): if not lead.leadpreview.text_extract: raise serializers.DjangoValidationError('Simplifed Text is empty') draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], - draft_entry_type=DraftEntry.Type.AUTO) + type=DraftEntry.Type.AUTO) if draft_entry.exists(): raise serializers.DjangoValidationError('Draft entry already exists') # Use already existing draft entry if found @@ -161,7 +161,7 @@ def create(self, data): lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING lead.save(update_fields=['auto_entry_extraction_status']) transaction.on_commit( - lambda: trigger_request_for_auto_draft_entry_task.delay(self.data['lead']) + lambda: trigger_request_for_auto_draft_entry_task.delay(lead.id) ) return True diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index f3887f452d..e9c68c3dc6 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -195,7 +195,6 @@ def send_trigger_request_to_extractor(cls, draft_entry): draft_entry.prediction_status = DraftEntry.PredictionStatus.SEND_FAILED draft_entry.save(update_fields=('prediction_status',)) _response = locals().get('response') - logger.error(payload) logger.error( 'Assisted tagging send failed!!', extra={ @@ -339,7 +338,6 @@ def save_data(cls, draft_entry, data): with transaction.atomic(): draft_entry.clear_data() # Clear old data if exists draft_entry.calculated_at = timezone.now() - # for prediction in model_preds: model_version = models_version_map[(model_preds['model_info']['id'], model_preds['model_info']['version'])] cls._process_model_preds(model_version, current_tags_map, draft_entry, model_preds) draft_entry.prediction_status = DraftEntry.PredictionStatus.DONE @@ -370,7 +368,6 @@ def auto_trigger_request_to_extractor(cls, lead): headers=cls.REQUEST_HEADERS, json=payload ) - logger.error(response.status_code) if response.status_code == 202: lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING lead.save() @@ -379,7 +376,6 @@ def auto_trigger_request_to_extractor(cls, lead): except Exception: logger.error('Entry Extraction send failed, Exception occurred!!', exc_info=True) _response = locals().get('response') - logger.error(payload) logger.error( 'Entry Extraction send failed!!', extra={ @@ -508,7 +504,7 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode @classmethod @transaction.atomic def save_data(cls, lead, data): - draft_entry = DraftEntry.objects.filter(lead=lead, draft_entry_type=0) + draft_entry = DraftEntry.objects.filter(lead=lead, type=0) if draft_entry.exists(): raise serializers.ValidationError('Draft entries already exit') for model_preds in data['blocks']: @@ -532,7 +528,7 @@ def save_data(cls, lead, data): lead=lead, excerpt=model_preds['text'], prediction_status=DraftEntry.PredictionStatus.DONE, - draft_entry_type=0 + type=0 ) draft.save() model_version = models_version_map[ diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 14d10d410d..abebc17625 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -74,7 +74,7 @@ class LeadExtractCallbackSerializer(DeeplServerBaseCallbackSerializer): ) text_path = serializers.CharField(required=False, allow_null=True) total_words_count = serializers.IntegerField(required=False, default=0, allow_null=True) - total_pages = serializers.IntegerField(required=False, default=0) + total_pages = serializers.IntegerField(required=False, default=0, allow_null=True) text_extraction_id = serializers.CharField(required=True) nlp_handler = LeadExtractionHandler @@ -162,9 +162,6 @@ def create(self, data): # --- AssistedTagging class AssistedTaggingModelPredictionCallbackSerializer(serializers.Serializer): - # class ModelInfoCallbackSerializer(serializers.Serializer): - # id = serializers.CharField() - # version = serializers.CharField() class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): prediction = serializers.DecimalField( diff --git a/deep/deepl.py b/deep/deepl.py index 32f6d3c135..78e47a5ab6 100644 --- a/deep/deepl.py +++ b/deep/deepl.py @@ -17,5 +17,5 @@ class DeeplServiceEndpoint(): ANALYSIS_AUTOMATIC_SUMMARY = f'{DEEPL_SERVER_DOMAIN}/api/v1/summarization/' ANALYSIS_AUTOMATIC_NGRAM = f'{DEEPL_SERVER_DOMAIN}/api/v1/ngrams/' ANALYSIS_GEO = f'{DEEPL_SERVER_DOMAIN}/api/v1/geolocation/' - ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVICE_DOMAIN}/api/v1/entry-classification/' + ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT = f'{DEEPL_SERVER_DOMAIN}/api/v1/entry-classification/' ENTRY_EXTRACTION_CLASSIFICATION = f'{DEEPL_SERVER_DOMAIN}/api/v1/entry-extraction-classification/' diff --git a/schema.graphql b/schema.graphql index a59dd877e3..24e9a2523b 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1425,7 +1425,7 @@ type AssistedTaggingPredictionType { type AssistedTaggingQueryType { draftEntry(id: ID!): DraftEntryType draftEntryByLeads(filter: DraftEntryFilterDataInputType): [DraftEntryType!] - extractionStatusByLead(leadId: ID!): AutoextractionStatusType + extractionStatusByLead(leadId: ID!): AutoExtractionStatusType } type AssistedTaggingRootQueryType { @@ -1467,7 +1467,7 @@ enum AutoEntryExtractionTypeEnum { FAILED } -type AutoextractionStatusType { +type AutoExtractionStatusType { autoEntryExtractionStatus: AutoEntryExtractionTypeEnum! } @@ -1967,7 +1967,7 @@ type DjangoDebugSQL { type DraftEntryCountByLead { undiscardedDraftEntry: Int - discardedDraftEnrty: Int + discardedDraftEntry: Int } input DraftEntryFilterDataInputType { @@ -3993,7 +3993,7 @@ type TriggerAnalysisTopicModel { type TriggerAutoDraftEntry { errors: [GenericScalar!] ok: Boolean - result: DraftEntryType + result: LeadType } input TriggerAutoDraftEntryInputType { From d29be6ce632e330e5a0a973417ed1a91156ec303 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 21 Dec 2023 09:00:15 +0545 Subject: [PATCH 21/24] autocomplete in foreignkey and m2m --- apps/assisted_tagging/admin.py | 3 ++- apps/deepl_integration/handlers.py | 4 ++-- schema.graphql | 25 +++++++++++++------------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/apps/assisted_tagging/admin.py b/apps/assisted_tagging/admin.py index 155f785a8f..1a6aacef59 100644 --- a/apps/assisted_tagging/admin.py +++ b/apps/assisted_tagging/admin.py @@ -7,11 +7,12 @@ @admin.register(DraftEntry) class DraftEntryAdmin(VersionAdmin): + search_fields = ['lead'] list_display = [ 'lead', 'prediction_status', - 'excerpt', ] + autocomplete_fields = ('project', 'lead', 'related_geoareas',) @admin.register(AssistedTaggingPrediction) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index e9c68c3dc6..a99617560a 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -370,7 +370,7 @@ def auto_trigger_request_to_extractor(cls, lead): ) if response.status_code == 202: lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING - lead.save() + lead.save(update_fields=('auto_entry_extraction_status',)) return True except Exception: @@ -1008,4 +1008,4 @@ def save_data( geo_task.status = AnalyticalStatementGeoTask.Status.SUCCESS else: geo_task.status = AnalyticalStatementGeoTask.Status.FAILED - geo_task.save() + geo_task.save(update_fields=('status',)) diff --git a/schema.graphql b/schema.graphql index 24e9a2523b..3bfcbf18da 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1005,7 +1005,7 @@ type AppEnumCollection { ConnectorLeadExtractionStatus: [AppEnumCollectionConnectorLeadExtractionStatus!] DraftEntryPredictionStatus: [AppEnumCollectionDraftEntryPredictionStatus!] AssistedTaggingPredictionDataType: [AppEnumCollectionAssistedTaggingPredictionDataType!] - DraftEntryDraftEntryType: [AppEnumCollectionDraftEntryDraftEntryType!] + DraftEntryType: [AppEnumCollectionDraftEntryType!] LeadAutoEntryExtractionStatus: [AppEnumCollectionLeadAutoEntryExtractionStatus!] UnusedAssessmentMethodologyProtectionInfo: [AppEnumCollectionUnusedAssessmentMethodologyProtectionInfo!] } @@ -1106,14 +1106,14 @@ type AppEnumCollectionDiscardedEntryTag { description: String } -type AppEnumCollectionDraftEntryDraftEntryType { - enum: DraftEntryTypeEnum! +type AppEnumCollectionDraftEntryPredictionStatus { + enum: DraftEntryPredictionStatusEnum! label: String! description: String } -type AppEnumCollectionDraftEntryPredictionStatus { - enum: DraftEntryPredictionStatusEnum! +type AppEnumCollectionDraftEntryType { + enum: DraftEntryTypeEnum! label: String! description: String } @@ -1424,7 +1424,7 @@ type AssistedTaggingPredictionType { type AssistedTaggingQueryType { draftEntry(id: ID!): DraftEntryType - draftEntryByLeads(filter: DraftEntryFilterDataInputType): [DraftEntryType!] + draftEntries(lead: ID, draftEntryTypes: [DraftEntryTypeEnum!], isDiscarded: Boolean, page: Int = 1, pageSize: Int): DraftEntryListType extractionStatusByLead(leadId: ID!): AutoExtractionStatusType } @@ -1970,17 +1970,18 @@ type DraftEntryCountByLead { discardedDraftEntry: Int } -input DraftEntryFilterDataInputType { - leads: ID - draftEntryTypes: [DraftEntryTypeEnum!] - isDiscarded: Boolean -} - input DraftEntryInputType { lead: ID! excerpt: String! } +type DraftEntryListType { + results: [DraftEntryType!] + totalCount: Int + page: Int + pageSize: Int +} + enum DraftEntryPredictionStatusEnum { PENDING STARTED From c564d8b091daff94a2c04fbf73301ab8e51bc7b1 Mon Sep 17 00:00:00 2001 From: sudan45 Date: Thu, 21 Dec 2023 11:12:20 +0545 Subject: [PATCH 22/24] Filter added in admin.py - Fix test cases --- apps/assisted_tagging/admin.py | 20 +- apps/assisted_tagging/enums.py | 5 - apps/assisted_tagging/filters.py | 9 - apps/assisted_tagging/models.py | 2 +- apps/assisted_tagging/mutation.py | 12 +- apps/assisted_tagging/schema.py | 17 - apps/assisted_tagging/serializers.py | 22 +- apps/assisted_tagging/tasks.py | 10 +- .../tests/snapshots/snap_test_query.py | 2358 ++++------------- apps/assisted_tagging/tests/test_query.py | 751 ++---- apps/deepl_integration/handlers.py | 14 +- apps/deepl_integration/serializers.py | 5 +- apps/deepl_integration/views.py | 1 - apps/lead/enums.py | 4 + apps/lead/schema.py | 3 +- apps/lead/tests/test_apis.py | 5 +- apps/unified_connector/tests/test_mutation.py | 4 +- schema.graphql | 29 +- 18 files changed, 869 insertions(+), 2402 deletions(-) diff --git a/apps/assisted_tagging/admin.py b/apps/assisted_tagging/admin.py index 1a6aacef59..8bcc2de9f4 100644 --- a/apps/assisted_tagging/admin.py +++ b/apps/assisted_tagging/admin.py @@ -1,4 +1,5 @@ # Register your models here. +from admin_auto_filters.filters import AutocompleteFilterFactory from django.contrib import admin from assisted_tagging.models import AssistedTaggingModelPredictionTag, AssistedTaggingPrediction, DraftEntry @@ -12,23 +13,40 @@ class DraftEntryAdmin(VersionAdmin): 'lead', 'prediction_status', ] + list_filter = ( + AutocompleteFilterFactory('Lead', 'lead'), + AutocompleteFilterFactory('Project', 'project'), + 'type' + ) + autocomplete_fields = ('project', 'lead', 'related_geoareas',) @admin.register(AssistedTaggingPrediction) class AssistedTaggingPredictionAdmin(VersionAdmin): + search_fields = ['draft_entry'] list_display = [ "data_type", "draft_entry", "value", - "is_selected" + "is_selected", + "tag", ] + list_filter = ( + AutocompleteFilterFactory('DraftEntry', 'draft_entry'), + + ) + # NOTE: Skipping model_version. Only few of them exists + autocomplete_fields = ('draft_entry', 'category', 'tag') @admin.register(AssistedTaggingModelPredictionTag) class AssistedTaggingModelPredictionTagAdmin(VersionAdmin): + search_fields = ['parent_tag'] list_display = [ 'name', 'is_category', 'tag_id', + 'parent_tag', ] + autocomplete_fields = ('parent_tag',) diff --git a/apps/assisted_tagging/enums.py b/apps/assisted_tagging/enums.py index d73f9712a9..3301924fb3 100644 --- a/apps/assisted_tagging/enums.py +++ b/apps/assisted_tagging/enums.py @@ -9,7 +9,6 @@ DraftEntry, AssistedTaggingPrediction, ) -from lead.models import Lead DraftEntryPredictionStatusEnum = convert_enum_to_graphene_enum( DraftEntry.PredictionStatus, name='DraftEntryPredictionStatusEnum') @@ -18,9 +17,6 @@ DraftEntryTypeEnum = convert_enum_to_graphene_enum( DraftEntry.Type, name="DraftEntryTypeEnum" ) -AutoEntryExtractionTypeEnum = convert_enum_to_graphene_enum( - Lead.AutoExtractionStatus, name="AutoEntryExtractionTypeEnum" -) enum_map = { get_enum_name_from_django_field(field): enum @@ -28,7 +24,6 @@ (DraftEntry.prediction_status, DraftEntryPredictionStatusEnum), (AssistedTaggingPrediction.data_type, AssistedTaggingPredictionDataTypeEnum), (DraftEntry.type, DraftEntryTypeEnum), - (Lead.auto_entry_extraction_status, AutoEntryExtractionTypeEnum), ) } diff --git a/apps/assisted_tagging/filters.py b/apps/assisted_tagging/filters.py index 1d46eb698e..0a32ef768b 100644 --- a/apps/assisted_tagging/filters.py +++ b/apps/assisted_tagging/filters.py @@ -1,6 +1,5 @@ import django_filters -from deep.filter_set import generate_type_for_filter_set from .models import DraftEntry from utils.graphene.filters import IDFilter, MultipleInputFilter from .enums import ( @@ -16,11 +15,3 @@ class DraftEntryFilterSet(django_filters.FilterSet): class Meta: model = DraftEntry fields = () - - -DraftEntryFilterDataType, DraftEntryFilterDataInputType = generate_type_for_filter_set( - DraftEntryFilterSet, - "project.schema.ProjectListType", - "DraftEntryFilterDataType", - "DraftEntryFilterDataInputType", -) diff --git a/apps/assisted_tagging/models.py b/apps/assisted_tagging/models.py index ac86c08fc2..1c7af87028 100644 --- a/apps/assisted_tagging/models.py +++ b/apps/assisted_tagging/models.py @@ -92,7 +92,7 @@ class PredictionStatus(models.IntegerChoices): class Type(models.IntegerChoices): AUTO = 0, 'Auto Extraction' # NLP defiend extraction text - MANUAL = 1, 'Manual Extraction' # manaul defined extraction text + MANUAL = 1, 'Manual Extraction' # manual defined extraction text project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='+') lead = models.ForeignKey(Lead, on_delete=models.CASCADE, related_name='+') diff --git a/apps/assisted_tagging/mutation.py b/apps/assisted_tagging/mutation.py index cb33447cf0..abfd134c37 100644 --- a/apps/assisted_tagging/mutation.py +++ b/apps/assisted_tagging/mutation.py @@ -4,10 +4,10 @@ generate_input_type_for_serializer, PsGrapheneMutation, PsDeleteMutation, + mutation_is_not_valid ) from deep.permissions import ProjectPermissions as PP -from lead.schema import LeadType from .models import ( DraftEntry, MissingPredictionReview, @@ -117,9 +117,17 @@ class Arguments: data = TriggerAutoDraftEntryInputType(required=True) model = DraftEntry serializer_class = TriggerDraftEntryGqlSerializer - result = graphene.Field(LeadType) permissions = [PP.Permission.CREATE_ENTRY] + @classmethod + def perform_mutate(cls, root, info, **kwargs): + data = kwargs['data'] + serializer = cls.serializer_class(data=data, context={'request': info.context.request}) + if errors := mutation_is_not_valid(serializer): + return cls(errors=errors, ok=False) + serializer.save() + return cls(errors=None, ok=True) + class UpdateDraftEntry(PsGrapheneMutation): class Arguments: diff --git a/apps/assisted_tagging/schema.py b/apps/assisted_tagging/schema.py index 1dcf89cabd..6cc3718637 100644 --- a/apps/assisted_tagging/schema.py +++ b/apps/assisted_tagging/schema.py @@ -24,11 +24,9 @@ MissingPredictionReview, WrongPredictionReview, ) -from lead.models import Lead from .enums import ( DraftEntryPredictionStatusEnum, AssistedTaggingPredictionDataTypeEnum, - AutoEntryExtractionTypeEnum, ) @@ -227,12 +225,6 @@ class Meta: filterset_class = DraftEntryFilterSet -class AutoExtractionStatusType(graphene.ObjectType): - auto_entry_extraction_status = graphene.Field(AutoEntryExtractionTypeEnum, required=True) - - @staticmethod - def custom_queryset(root, info, lead_id): - return Lead.objects.get(id=lead_id) # This is attached to project type. @@ -244,16 +236,7 @@ class AssistedTaggingQueryType(graphene.ObjectType): page_size_query_param='pageSize', ), ) - extraction_status_by_lead = graphene.Field(AutoExtractionStatusType, lead_id=graphene.ID(required=True)) - - @staticmethod - def resolve_draft_entry(root, info, **kwargs): - return DraftEntryType.get_custom_queryset(root, info, **kwargs) @staticmethod def resolve_draft_entries(root, info, **_): return get_draft_entry_qs(info) - - @staticmethod - def resolve_extraction_status_by_lead(root, info, lead_id): - return AutoExtractionStatusType.custom_queryset(root, info, lead_id) diff --git a/apps/assisted_tagging/serializers.py b/apps/assisted_tagging/serializers.py index fb1a0702b3..c6798ad070 100644 --- a/apps/assisted_tagging/serializers.py +++ b/apps/assisted_tagging/serializers.py @@ -27,8 +27,8 @@ def validate_lead(self, lead): raise serializers.ValidationError('Assisted tagging is disabled for the Framework used by this project.') if self.project.is_private: raise serializers.ValidationError('Assisted tagging is not available for private projects.') - if lead.confidentiality == (Lead.Confidentiality.CONFIDENTIAL or Lead.Confidentiality.RESTRICTED): - raise serializers.ValidationError('Assisted tagging is not available for confidential lead or restricated') + if lead.confidentiality in (Lead.Confidentiality.CONFIDENTIAL, Lead.Confidentiality.RESTRICTED): + raise serializers.ValidationError('Assisted tagging is not available for confidential or restricated leads') return lead @@ -137,7 +137,9 @@ def validate(self, data): class TriggerDraftEntryGqlSerializer( DraftEntryBaseSerializer, ProjectPropertySerializerMixin, - UserResourceCreatedMixin, serializers.ModelSerializer): + UserResourceCreatedMixin, + serializers.ModelSerializer, +): class Meta: model = DraftEntry fields = ( @@ -148,13 +150,12 @@ def create(self, data): lead = data['lead'] if lead.leadpreview.text_extraction_id is None: raise serializers.DjangoValidationError("Assisted tagging is not available in old lead") - if lead.auto_entry_extraction_status == (Lead.AutoExtractionStatus.SUCCESS): - raise serializers.DjangoValidationError("Already Tiggered") + if lead.auto_entry_extraction_status == Lead.AutoExtractionStatus.SUCCESS: + raise serializers.DjangoValidationError("Already Triggered") if not lead.leadpreview.text_extract: raise serializers.DjangoValidationError('Simplifed Text is empty') - draft_entry = DraftEntry.objects.filter(lead=self.data['lead'], - type=DraftEntry.Type.AUTO) - if draft_entry.exists(): + draft_entry_qs = DraftEntry.objects.filter(lead=lead, type=DraftEntry.Type.AUTO) + if draft_entry_qs.exists(): raise serializers.DjangoValidationError('Draft entry already exists') # Use already existing draft entry if found # Create new one and send trigger to deepl @@ -179,8 +180,5 @@ class Meta: 'is_discarded', ) - def create(self, validate_data): + def create(self, _): raise Exception("Create is not Allowed") - - def update(self, instance, validate_data): - return super().update(instance, validate_data) diff --git a/apps/assisted_tagging/tasks.py b/apps/assisted_tagging/tasks.py index b02b50f887..3a7ca7fa46 100644 --- a/apps/assisted_tagging/tasks.py +++ b/apps/assisted_tagging/tasks.py @@ -1,4 +1,3 @@ -from django.conf import settings import logging import requests @@ -7,7 +6,11 @@ from utils.common import redis_lock from deep.deepl import DeeplServiceEndpoint -from deepl_integration.handlers import AssistedTaggingDraftEntryHandler, AutoAssistedTaggingDraftEntryHandler +from deepl_integration.handlers import ( + AssistedTaggingDraftEntryHandler, + AutoAssistedTaggingDraftEntryHandler, + BaseHandler as DeepHandler +) from .models import ( DraftEntry, @@ -27,8 +30,7 @@ def _get_existing_tags_by_tagid(): for tag in AssistedTaggingModelPredictionTag.objects.all() } - headers = {'Authorization': f'Token {settings.DEEPL_SERVER_TOKEN}'} - response = requests.get(DeeplServiceEndpoint.ASSISTED_TAGGING_TAGS_ENDPOINT, headers=headers).json() + response = requests.get(DeeplServiceEndpoint.ASSISTED_TAGGING_TAGS_ENDPOINT, headers=DeepHandler.REQUEST_HEADERS).json() existing_tags_by_tagid = _get_existing_tags_by_tagid() new_tags = [] diff --git a/apps/assisted_tagging/tests/snapshots/snap_test_query.py b/apps/assisted_tagging/tests/snapshots/snap_test_query.py index cd4f72eacd..ec82ec4f53 100644 --- a/apps/assisted_tagging/tests/snapshots/snap_test_query.py +++ b/apps/assisted_tagging/tests/snapshots/snap_test_query.py @@ -8,954 +8,750 @@ snapshots = Snapshot() snapshots['AssistedTaggingCallbackApiTest::test_save_draft_entry final-current-model-stats'] = { - 'model_count': 3, - 'model_version_count': 3, + 'model_count': 1, + 'model_version_count': 1, 'model_versions': [ { 'model__model_id': 'all_tags_model', 'version': '1.0.0' - }, - { - 'model__model_id': 'geolocation', - 'version': '1.0.0' - }, - { - 'model__model_id': 'reliability', - 'version': '1.0.0' } ], 'models': [ { 'model_id': 'all_tags_model', 'name': 'all_tags_model' - }, - { - 'model_id': 'geolocation', - 'name': 'geolocation' - }, - { - 'model_id': 'reliability', - 'name': 'reliability' } ], - 'tag_count': 101, + 'tag_count': 136, 'tags': [ { 'is_deprecated': False, - 'name': '1', + 'name': 'reliability', + 'tag_id': '10' + }, + { + 'is_deprecated': False, + 'name': 'sectors', 'tag_id': '1' }, { 'is_deprecated': False, - 'name': '101', - 'tag_id': '101' + 'name': 'subpillars_2d', + 'tag_id': '3' }, { 'is_deprecated': False, - 'name': '103', - 'tag_id': '103' + 'name': 'subpillars_1d', + 'tag_id': '2' }, { 'is_deprecated': False, - 'name': '104', - 'tag_id': '104' + 'name': 'age', + 'tag_id': '6' + }, + { + 'is_deprecated': False, + 'name': 'gender', + 'tag_id': '5' + }, + { + 'is_deprecated': False, + 'name': 'affected_groups', + 'tag_id': '8' + }, + { + 'is_deprecated': False, + 'name': 'specific_needs_groups', + 'tag_id': '4' }, { 'is_deprecated': False, - 'name': '105', + 'name': 'severity', + 'tag_id': '7' + }, + { + 'is_deprecated': False, + 'name': 'demographic_group', + 'tag_id': '9' + }, + { + 'is_deprecated': False, + 'name': 'Health', 'tag_id': '105' }, { 'is_deprecated': False, - 'name': '106', + 'name': 'Livelihoods', 'tag_id': '106' }, { 'is_deprecated': False, - 'name': '107', + 'name': 'Logistics', 'tag_id': '107' }, { 'is_deprecated': False, - 'name': '108', + 'name': 'Nutrition', 'tag_id': '108' }, { 'is_deprecated': False, - 'name': '109', + 'name': 'Protection', 'tag_id': '109' }, { 'is_deprecated': False, - 'name': '110', + 'name': 'Shelter', 'tag_id': '110' }, { 'is_deprecated': False, - 'name': '111', + 'name': 'WASH', 'tag_id': '111' }, { 'is_deprecated': False, - 'name': '3', - 'tag_id': '3' + 'name': 'Environment', + 'tag_id': '201' }, { 'is_deprecated': False, - 'name': '301', - 'tag_id': '301' + 'name': 'Socio Cultural', + 'tag_id': '202' }, { 'is_deprecated': False, - 'name': '302', - 'tag_id': '302' + 'name': 'Economy', + 'tag_id': '203' }, { 'is_deprecated': False, - 'name': '303', - 'tag_id': '303' + 'name': 'Legal & Policy', + 'tag_id': '205' }, { 'is_deprecated': False, - 'name': '304', - 'tag_id': '304' + 'name': 'Security & Stability', + 'tag_id': '206' }, { 'is_deprecated': False, - 'name': '305', - 'tag_id': '305' + 'name': 'Politics', + 'tag_id': '207' }, { 'is_deprecated': False, - 'name': '306', - 'tag_id': '306' + 'name': 'Type And Characteristics', + 'tag_id': '208' }, { 'is_deprecated': False, - 'name': '307', - 'tag_id': '307' + 'name': 'Hazard & Threats', + 'tag_id': '210' }, { 'is_deprecated': False, - 'name': '308', - 'tag_id': '308' + 'name': 'Type/Numbers/Movements', + 'tag_id': '212' }, { 'is_deprecated': False, - 'name': '309', - 'tag_id': '309' + 'name': 'Push Factors', + 'tag_id': '213' }, { 'is_deprecated': False, - 'name': '310', - 'tag_id': '310' + 'name': 'Intentions', + 'tag_id': '215' }, { 'is_deprecated': False, - 'name': '311', - 'tag_id': '311' + 'name': 'Relief To Population', + 'tag_id': '220' }, { 'is_deprecated': False, - 'name': '312', - 'tag_id': '312' + 'name': 'Population To Relief', + 'tag_id': '221' }, { 'is_deprecated': False, - 'name': '313', - 'tag_id': '313' + 'name': 'Physical Constraints', + 'tag_id': '222' }, { 'is_deprecated': False, - 'name': '314', - 'tag_id': '314' + 'name': 'Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps', + 'tag_id': '223' }, { 'is_deprecated': False, - 'name': '315', - 'tag_id': '315' + 'name': 'Communication Means And Preferences', + 'tag_id': '224' }, { 'is_deprecated': False, - 'name': '316', - 'tag_id': '316' + 'name': 'Information Challenges And Barriers', + 'tag_id': '225' }, { 'is_deprecated': False, - 'name': '317', - 'tag_id': '317' + 'name': 'Knowledge And Info Gaps (Pop)', + 'tag_id': '226' }, { 'is_deprecated': False, - 'name': '318', - 'tag_id': '318' + 'name': 'Knowledge And Info Gaps (Hum)', + 'tag_id': '227' }, { 'is_deprecated': False, - 'name': '2', - 'tag_id': '2' + 'name': 'Cases', + 'tag_id': '228' }, { 'is_deprecated': False, - 'name': '219', - 'tag_id': '219' + 'name': 'Contact Tracing', + 'tag_id': '229' }, { 'is_deprecated': False, - 'name': '217', - 'tag_id': '217' + 'name': 'Deaths', + 'tag_id': '230' }, { 'is_deprecated': False, - 'name': '218', - 'tag_id': '218' + 'name': 'Hospitalization & Care', + 'tag_id': '231' }, { 'is_deprecated': False, - 'name': '204', - 'tag_id': '204' + 'name': 'Restriction Measures', + 'tag_id': '232' }, { 'is_deprecated': False, - 'name': '203', - 'tag_id': '203' + 'name': 'Testing', + 'tag_id': '233' }, { 'is_deprecated': False, - 'name': '201', - 'tag_id': '201' + 'name': 'Vaccination', + 'tag_id': '234' }, { 'is_deprecated': False, - 'name': '205', - 'tag_id': '205' + 'name': 'Technological', + 'tag_id': '235' }, { 'is_deprecated': False, - 'name': '207', - 'tag_id': '207' + 'name': 'Prevention campaign', + 'tag_id': '236' }, { 'is_deprecated': False, - 'name': '206', - 'tag_id': '206' + 'name': 'Research and outlook', + 'tag_id': '237' }, { 'is_deprecated': False, - 'name': '202', - 'tag_id': '202' + 'name': 'National Response', + 'tag_id': '305' }, { 'is_deprecated': False, - 'name': '228', - 'tag_id': '228' + 'name': 'Number Of People Reached/Response Gaps', + 'tag_id': '306' }, { 'is_deprecated': False, - 'name': '229', - 'tag_id': '229' + 'name': 'Coping Mechanisms', + 'tag_id': '307' }, { 'is_deprecated': False, - 'name': '230', - 'tag_id': '230' + 'name': 'Living Standards', + 'tag_id': '308' }, { 'is_deprecated': False, - 'name': '231', - 'tag_id': '231' + 'name': 'Number Of People In Need', + 'tag_id': '309' }, { 'is_deprecated': False, - 'name': '232', - 'tag_id': '232' + 'name': 'Physical And Mental Well Being', + 'tag_id': '310' }, { 'is_deprecated': False, - 'name': '233', - 'tag_id': '233' + 'name': 'Driver/Aggravating Factors', + 'tag_id': '311' }, { 'is_deprecated': False, - 'name': '234', - 'tag_id': '234' + 'name': 'Impact On People', + 'tag_id': '312' }, { 'is_deprecated': False, - 'name': '215', - 'tag_id': '215' + 'name': 'Impact On Systems, Services And Networks', + 'tag_id': '313' }, { 'is_deprecated': False, - 'name': '216', - 'tag_id': '216' + 'name': 'Number Of People Affected', + 'tag_id': '314' }, { 'is_deprecated': False, - 'name': '214', - 'tag_id': '214' + 'name': 'Humanitarian coordination', + 'tag_id': '319' }, { 'is_deprecated': False, - 'name': '213', - 'tag_id': '213' + 'name': 'People reached/response gaps', + 'tag_id': '320' }, { 'is_deprecated': False, - 'name': '212', - 'tag_id': '212' + 'name': 'Red cross/red crescent', + 'tag_id': '321' }, { 'is_deprecated': False, - 'name': '223', - 'tag_id': '223' + 'name': 'Elderly Head of Household', + 'tag_id': '403' }, { 'is_deprecated': False, - 'name': '222', - 'tag_id': '222' + 'name': 'Female Head of Household', + 'tag_id': '404' }, { 'is_deprecated': False, - 'name': '221', - 'tag_id': '221' + 'name': 'GBV survivors', + 'tag_id': '405' }, { 'is_deprecated': False, - 'name': '220', - 'tag_id': '220' + 'name': 'Indigenous people', + 'tag_id': '406' }, { 'is_deprecated': False, - 'name': '224', - 'tag_id': '224' + 'name': 'Persons with Disability', + 'tag_id': '409' }, { 'is_deprecated': False, - 'name': '225', - 'tag_id': '225' + 'name': 'Pregnant or Lactating Women', + 'tag_id': '410' }, { 'is_deprecated': False, - 'name': '227', - 'tag_id': '227' + 'name': 'Single Women (including Widows)', + 'tag_id': '411' }, { 'is_deprecated': False, - 'name': '226', - 'tag_id': '226' + 'name': 'Lgbtqia+', + 'tag_id': '413' }, { 'is_deprecated': False, - 'name': '210', - 'tag_id': '210' + 'name': 'Unaccompanied or/and separated children', + 'tag_id': '414' }, { 'is_deprecated': False, - 'name': '208', - 'tag_id': '208' + 'name': 'Infants/Toddlers (<5 years old) ', + 'tag_id': '901' }, { 'is_deprecated': False, - 'name': '209', - 'tag_id': '209' + 'name': 'Female Children/Youth (5 to 17 years old)', + 'tag_id': '902' }, { 'is_deprecated': False, - 'name': '6', - 'tag_id': '6' + 'name': 'Male Children/Youth (5 to 17 years old)', + 'tag_id': '903' }, { 'is_deprecated': False, - 'name': '601', - 'tag_id': '601' + 'name': 'Female Older Persons (60+ years old)', + 'tag_id': '906' }, { 'is_deprecated': False, - 'name': '602', - 'tag_id': '602' + 'name': 'Major', + 'tag_id': '702' }, { 'is_deprecated': False, - 'name': '603', - 'tag_id': '603' + 'name': 'Minor Problem', + 'tag_id': '703' }, { 'is_deprecated': False, - 'name': '604', - 'tag_id': '604' + 'name': 'No problem', + 'tag_id': '704' }, { 'is_deprecated': False, - 'name': '5', - 'tag_id': '5' + 'name': 'Of Concern', + 'tag_id': '705' }, { 'is_deprecated': False, - 'name': '501', - 'tag_id': '501' + 'name': 'Critical issue', + 'tag_id': '706' }, { 'is_deprecated': False, - 'name': '502', - 'tag_id': '502' + 'name': 'Issue of concern', + 'tag_id': '707' }, { 'is_deprecated': False, - 'name': '8', - 'tag_id': '8' + 'name': 'Minor issue', + 'tag_id': '708' }, { 'is_deprecated': False, - 'name': '801', - 'tag_id': '801' + 'name': 'No issue', + 'tag_id': '709' }, { 'is_deprecated': False, - 'name': '802', - 'tag_id': '802' + 'name': 'Severe issue', + 'tag_id': '710' }, { 'is_deprecated': False, - 'name': '803', + 'name': 'IDP', 'tag_id': '803' }, { 'is_deprecated': False, - 'name': '804', + 'name': 'Migrants', 'tag_id': '804' }, { 'is_deprecated': False, - 'name': '805', + 'name': 'Refugees', 'tag_id': '805' }, { 'is_deprecated': False, - 'name': '806', + 'name': 'Returnees', 'tag_id': '806' }, { 'is_deprecated': False, - 'name': '4', - 'tag_id': '4' + 'name': 'Completely reliable', + 'tag_id': '1001' }, { 'is_deprecated': False, - 'name': '401', - 'tag_id': '401' + 'name': 'Usually reliable', + 'tag_id': '1002' }, { 'is_deprecated': False, - 'name': '402', - 'tag_id': '402' + 'name': 'Fairly Reliable', + 'tag_id': '1003' }, { 'is_deprecated': False, - 'name': '403', - 'tag_id': '403' + 'name': 'Unreliable', + 'tag_id': '1004' }, { 'is_deprecated': False, - 'name': '404', - 'tag_id': '404' + 'name': 'All', + 'tag_id': '503' }, { 'is_deprecated': False, - 'name': '405', - 'tag_id': '405' + 'name': '12-17 years old', + 'tag_id': '605' }, { 'is_deprecated': False, - 'name': '406', - 'tag_id': '406' + 'name': '18-24 years old', + 'tag_id': '606' }, { 'is_deprecated': False, - 'name': '407', - 'tag_id': '407' + 'name': '18-59 years old', + 'tag_id': '607' }, { 'is_deprecated': False, - 'name': '408', - 'tag_id': '408' + 'name': '25-59 years old', + 'tag_id': '608' }, { 'is_deprecated': False, - 'name': '409', - 'tag_id': '409' + 'name': '5-11 years old', + 'tag_id': '609' }, { 'is_deprecated': False, - 'name': '410', - 'tag_id': '410' + 'name': '5-17 years old', + 'tag_id': '610' }, { 'is_deprecated': False, - 'name': '411', - 'tag_id': '411' + 'name': '<18 years', + 'tag_id': '611' }, { 'is_deprecated': False, - 'name': '412', - 'tag_id': '412' + 'name': '<18 years old', + 'tag_id': '612' }, { 'is_deprecated': False, - 'name': '9', - 'tag_id': '9' + 'name': '<5 years old', + 'tag_id': '613' }, { 'is_deprecated': False, - 'name': '904', - 'tag_id': '904' + 'name': '>60 years old', + 'tag_id': '614' }, { 'is_deprecated': False, - 'name': '905', - 'tag_id': '905' + 'name': 'Agriculture', + 'tag_id': '101' }, { 'is_deprecated': False, - 'name': '902', - 'tag_id': '902' + 'name': 'Cross', + 'tag_id': '102' }, { 'is_deprecated': False, - 'name': '903', - 'tag_id': '903' + 'name': 'Education', + 'tag_id': '103' }, { 'is_deprecated': False, - 'name': '906', - 'tag_id': '906' + 'name': 'Food Security', + 'tag_id': '104' }, { 'is_deprecated': False, - 'name': '907', - 'tag_id': '907' + 'name': 'Number Of People At Risk', + 'tag_id': '301' }, { 'is_deprecated': False, - 'name': '10', - 'tag_id': '10' + 'name': 'Risk And Vulnerabilities', + 'tag_id': '302' }, { 'is_deprecated': False, - 'name': '1002', - 'tag_id': '1002' - } - ] -} - -snapshots['AssistedTaggingCallbackApiTest::test_save_draft_entry final-current-prediction-stats'] = { - 'prediction_count': 188, - 'predictions': [ + 'name': 'International Response', + 'tag_id': '303' + }, { - 'category__tag_id': None, - 'data_type': 0, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'geolocation', - 'prediction': None, - 'tag__tag_id': None, - 'threshold': None, - 'value': 'Nepal' + 'is_deprecated': False, + 'name': 'Local Response', + 'tag_id': '304' }, { - 'category__tag_id': None, - 'data_type': 0, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'geolocation', - 'prediction': None, - 'tag__tag_id': None, - 'threshold': None, - 'value': 'Paris' + 'is_deprecated': False, + 'name': 'Expressed By Humanitarian Staff', + 'tag_id': '315' }, { - 'category__tag_id': None, - 'data_type': 0, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, - 'model_version__model__model_id': 'geolocation', - 'prediction': None, - 'tag__tag_id': None, - 'threshold': None, - 'value': 'Nepal' + 'is_deprecated': False, + 'name': 'Expressed By Population', + 'tag_id': '316' }, { - 'category__tag_id': None, - 'data_type': 0, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, - 'model_version__model__model_id': 'geolocation', - 'prediction': None, - 'tag__tag_id': None, - 'threshold': None, - 'value': 'Paris' + 'is_deprecated': False, + 'name': 'Expressed By Humanitarian Staff', + 'tag_id': '317' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00131315333064554660')"), - 'tag__tag_id': '101', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Expressed By Population', + 'tag_id': '318' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00301082416073135700')"), - 'tag__tag_id': '103', - 'threshold': GenericRepr("Decimal('0.46000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Dead', + 'tag_id': '219' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00256628797311956700')"), - 'tag__tag_id': '104', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Injured', + 'tag_id': '217' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('2.67795523007710800000')"), - 'tag__tag_id': '105', - 'threshold': GenericRepr("Decimal('0.36000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Missing', + 'tag_id': '218' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01722483797685096000')"), - 'tag__tag_id': '106', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Demography', + 'tag_id': '204' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00367074832320213300')"), - 'tag__tag_id': '107', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Local Integration', + 'tag_id': '216' }, { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00410134124816680450')"), - 'tag__tag_id': '108', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02810047168670029600')"), - 'tag__tag_id': '109', - 'threshold': GenericRepr("Decimal('0.58000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00356446807494475730')"), - 'tag__tag_id': '110', - 'threshold': GenericRepr("Decimal('0.42000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00885658950175879000')"), - 'tag__tag_id': '111', - 'threshold': GenericRepr("Decimal('0.53000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00161047546977275300')"), - 'tag__tag_id': '201', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Pull Factors', + 'tag_id': '214' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00249261039939215920')"), - 'tag__tag_id': '202', - 'threshold': GenericRepr("Decimal('0.17000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Underlying/Aggravating Factors', + 'tag_id': '209' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00276056816801428800')"), - 'tag__tag_id': '203', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Adult (18 to 59 years old)', + 'tag_id': '601' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01951472795739466000')"), - 'tag__tag_id': '204', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Children/Youth (5 to 17 years old)', + 'tag_id': '602' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00284144639848701430')"), - 'tag__tag_id': '205', - 'threshold': GenericRepr("Decimal('0.31000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Infants/Toddlers (<5 years old)', + 'tag_id': '603' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00284233643800358870')"), - 'tag__tag_id': '206', - 'threshold': GenericRepr("Decimal('0.44000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Older Persons (60+ years old)', + 'tag_id': '604' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00300193069657931750')"), - 'tag__tag_id': '207', - 'threshold': GenericRepr("Decimal('0.30000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Female', + 'tag_id': '501' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00357941840775311000')"), - 'tag__tag_id': '208', - 'threshold': GenericRepr("Decimal('0.20000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Male', + 'tag_id': '502' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00632169711239197600')"), - 'tag__tag_id': '209', - 'threshold': GenericRepr("Decimal('0.17000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Asylum Seekers', + 'tag_id': '801' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00697963860938730400')"), - 'tag__tag_id': '210', - 'threshold': GenericRepr("Decimal('0.23000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Host', + 'tag_id': '802' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01237871689035704300')"), - 'tag__tag_id': '212', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Child Head of Household', + 'tag_id': '401' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00024467206802345010')"), - 'tag__tag_id': '213', - 'threshold': GenericRepr("Decimal('0.37000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Chronically Ill', + 'tag_id': '402' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00036748812096354010')"), - 'tag__tag_id': '214', - 'threshold': GenericRepr("Decimal('0.29000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'LGBTQI+', + 'tag_id': '407' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00047817865331461160')"), - 'tag__tag_id': '215', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Minorities', + 'tag_id': '408' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00696396781131625200')"), - 'tag__tag_id': '216', - 'threshold': GenericRepr("Decimal('0.25000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Unaccompanied or Separated Children', + 'tag_id': '412' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00091310044249089870')"), - 'tag__tag_id': '217', - 'threshold': GenericRepr("Decimal('0.13000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Critical', + 'tag_id': '701' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00106291823053302660')"), - 'tag__tag_id': '218', - 'threshold': GenericRepr("Decimal('0.13000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Female Adult (18 to 59 years old)', + 'tag_id': '904' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00187798162057463590')"), - 'tag__tag_id': '219', - 'threshold': GenericRepr("Decimal('0.28000000000000000000')"), - 'value': '' + 'is_deprecated': False, + 'name': 'Male Adult (18 to 59 years old)', + 'tag_id': '905' }, { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01125925638038536600')"), - 'tag__tag_id': '220', - 'threshold': GenericRepr("Decimal('0.29000000000000000000')"), - 'value': '' - }, + 'is_deprecated': False, + 'name': 'Male Older Persons (60+ years old)', + 'tag_id': '907' + } + ] +} + +snapshots['AssistedTaggingCallbackApiTest::test_save_draft_entry final-current-prediction-stats'] = { + 'prediction_count': 72, + 'predictions': [ { - 'category__tag_id': '2', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00166666776701611900')"), - 'tag__tag_id': '221', - 'threshold': GenericRepr("Decimal('0.19000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00200000000000000004')"), + 'tag__tag_id': '101', + 'threshold': GenericRepr("Decimal('0.14000000000000001332')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, + 'is_selected': True, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00146527563629206270')"), - 'tag__tag_id': '222', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.64800000000000002043')"), + 'tag__tag_id': '102', + 'threshold': GenericRepr("Decimal('0.17000000000000001221')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00115551359139065800')"), - 'tag__tag_id': '223', - 'threshold': GenericRepr("Decimal('0.47000000000000003000')"), + 'prediction': GenericRepr("Decimal('0.02699999999999999969')"), + 'tag__tag_id': '103', + 'threshold': GenericRepr("Decimal('0.10000000000000000555')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00758105556347540500')"), - 'tag__tag_id': '224', - 'threshold': GenericRepr("Decimal('0.21000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.06199999999999999956')"), + 'tag__tag_id': '104', + 'threshold': GenericRepr("Decimal('0.14000000000000001332')"), 'value': '' }, { @@ -964,20 +760,20 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00033728324827582890')"), - 'tag__tag_id': '225', - 'threshold': GenericRepr("Decimal('0.15000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00700000000000000015')"), + 'tag__tag_id': '204', + 'threshold': GenericRepr("Decimal('0.14000000000000001332')"), 'value': '' }, { 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, + 'is_selected': True, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00077029108069837090')"), - 'tag__tag_id': '226', - 'threshold': GenericRepr("Decimal('0.18000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.45800000000000001821')"), + 'tag__tag_id': '209', + 'threshold': GenericRepr("Decimal('0.05000000000000000278')"), 'value': '' }, { @@ -986,9 +782,9 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00090097592975426880')"), - 'tag__tag_id': '227', - 'threshold': GenericRepr("Decimal('0.18000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '214', + 'threshold': GenericRepr("Decimal('0.08999999999999999667')"), 'value': '' }, { @@ -997,9 +793,9 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00497279979754239300')"), - 'tag__tag_id': '228', - 'threshold': GenericRepr("Decimal('0.80000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00300000000000000006')"), + 'tag__tag_id': '216', + 'threshold': GenericRepr("Decimal('0.13000000000000000444')"), 'value': '' }, { @@ -1008,9 +804,9 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00032880847216941987')"), - 'tag__tag_id': '229', - 'threshold': GenericRepr("Decimal('0.39000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '217', + 'threshold': GenericRepr("Decimal('0.04000000000000000083')"), 'value': '' }, { @@ -1019,9 +815,9 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00116735643615233300')"), - 'tag__tag_id': '230', - 'threshold': GenericRepr("Decimal('0.81000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00400000000000000008')"), + 'tag__tag_id': '218', + 'threshold': GenericRepr("Decimal('0.08999999999999999667')"), 'value': '' }, { @@ -1030,53 +826,53 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00244935224877624970')"), - 'tag__tag_id': '231', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), + 'prediction': GenericRepr("Decimal('0.00300000000000000006')"), + 'tag__tag_id': '219', + 'threshold': GenericRepr("Decimal('0.13000000000000000444')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00542857805671899200')"), - 'tag__tag_id': '232', - 'threshold': GenericRepr("Decimal('0.46000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '301', + 'threshold': GenericRepr("Decimal('0.01000000000000000021')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00188743645513925370')"), - 'tag__tag_id': '233', - 'threshold': GenericRepr("Decimal('0.79000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '302', + 'threshold': GenericRepr("Decimal('0.11000000000000000056')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00117788418989490570')"), - 'tag__tag_id': '234', - 'threshold': GenericRepr("Decimal('0.54000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.08300000000000000433')"), + 'tag__tag_id': '303', + 'threshold': GenericRepr("Decimal('0.38000000000000000444')"), 'value': '' }, { 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, + 'is_selected': True, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00023104241032948875')"), - 'tag__tag_id': '301', - 'threshold': GenericRepr("Decimal('0.12000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.08599999999999999312')"), + 'tag__tag_id': '304', + 'threshold': GenericRepr("Decimal('0.01000000000000000021')"), 'value': '' }, { @@ -1085,20 +881,20 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00684022131126101400')"), - 'tag__tag_id': '302', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), + 'prediction': GenericRepr("Decimal('0.00300000000000000006')"), + 'tag__tag_id': '315', + 'threshold': GenericRepr("Decimal('0.45000000000000001110')"), 'value': '' }, { 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, + 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('1.51390548675291000000')"), - 'tag__tag_id': '303', - 'threshold': GenericRepr("Decimal('0.62000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '316', + 'threshold': GenericRepr("Decimal('0.05999999999999999778')"), 'value': '' }, { @@ -1107,9 +903,9 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00246191542828455570')"), - 'tag__tag_id': '304', - 'threshold': GenericRepr("Decimal('0.10000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00400000000000000008')"), + 'tag__tag_id': '317', + 'threshold': GenericRepr("Decimal('0.28000000000000002665')"), 'value': '' }, { @@ -1118,1142 +914,317 @@ 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.19748103480006374000')"), - 'tag__tag_id': '305', - 'threshold': GenericRepr("Decimal('0.43000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '318', + 'threshold': GenericRepr("Decimal('0.13000000000000000444')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '4', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.13266879380965720000')"), - 'tag__tag_id': '306', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '401', + 'threshold': GenericRepr("Decimal('0.28999999999999998002')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '4', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00847395147300428900')"), - 'tag__tag_id': '307', - 'threshold': GenericRepr("Decimal('0.36000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '402', + 'threshold': GenericRepr("Decimal('0.45000000000000001110')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '4', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01439434579677051900')"), - 'tag__tag_id': '308', - 'threshold': GenericRepr("Decimal('0.45000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '407', + 'threshold': GenericRepr("Decimal('0.07000000000000000666')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '4', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00275349894147967100')"), - 'tag__tag_id': '309', - 'threshold': GenericRepr("Decimal('0.31000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '408', + 'threshold': GenericRepr("Decimal('0.11000000000000000056')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '4', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02261752535293742000')"), - 'tag__tag_id': '310', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00280699276022220930')"), - 'tag__tag_id': '311', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00353863737969235940')"), - 'tag__tag_id': '312', - 'threshold': GenericRepr("Decimal('0.33000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00474455507679118000')"), - 'tag__tag_id': '313', - 'threshold': GenericRepr("Decimal('0.45000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00243518249286959600')"), - 'tag__tag_id': '314', - 'threshold': GenericRepr("Decimal('0.24000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00498411668972535500')"), - 'tag__tag_id': '315', - 'threshold': GenericRepr("Decimal('0.55000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00342778279446065430')"), - 'tag__tag_id': '316', - 'threshold': GenericRepr("Decimal('0.15000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00183609818729261560')"), - 'tag__tag_id': '317', - 'threshold': GenericRepr("Decimal('0.30000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '3', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00765169737860560400')"), - 'tag__tag_id': '318', - 'threshold': GenericRepr("Decimal('0.25000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00020814205345232040')"), - 'tag__tag_id': '401', - 'threshold': GenericRepr("Decimal('0.25000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00299776057263129780')"), - 'tag__tag_id': '402', - 'threshold': GenericRepr("Decimal('0.58000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00299218206367056270')"), - 'tag__tag_id': '403', - 'threshold': GenericRepr("Decimal('0.14000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00241597760274695900')"), - 'tag__tag_id': '404', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02053049989999868700')"), - 'tag__tag_id': '405', - 'threshold': GenericRepr("Decimal('0.78000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00281014967745599850')"), - 'tag__tag_id': '406', - 'threshold': GenericRepr("Decimal('0.13000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00022843408415366598')"), - 'tag__tag_id': '407', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00943289911848003600')"), - 'tag__tag_id': '408', - 'threshold': GenericRepr("Decimal('0.59000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00091892403101415500')"), - 'tag__tag_id': '409', - 'threshold': GenericRepr("Decimal('0.56000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('2.03979988487399360000')"), - 'tag__tag_id': '410', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00750677951145917200')"), - 'tag__tag_id': '411', - 'threshold': GenericRepr("Decimal('0.20000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '4', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00019092757914525768')"), + 'prediction': GenericRepr("Decimal('0E-20')"), 'tag__tag_id': '412', - 'threshold': GenericRepr("Decimal('0.60000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '5', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('1.40336945023335200000')"), - 'tag__tag_id': '501', - 'threshold': GenericRepr("Decimal('0.71000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '5', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00778131599707359600')"), - 'tag__tag_id': '502', - 'threshold': GenericRepr("Decimal('0.44000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '6', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.04068703080217044000')"), - 'tag__tag_id': '601', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '6', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02458783670921217300')"), - 'tag__tag_id': '602', - 'threshold': GenericRepr("Decimal('0.44000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '6', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.04259871318936348000')"), - 'tag__tag_id': '603', - 'threshold': GenericRepr("Decimal('0.40000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '6', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00641449491997234200')"), - 'tag__tag_id': '604', - 'threshold': GenericRepr("Decimal('0.61000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00027582379877694870')"), - 'tag__tag_id': '801', - 'threshold': GenericRepr("Decimal('0.73000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00997524285181002000')"), - 'tag__tag_id': '802', - 'threshold': GenericRepr("Decimal('0.55000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00476177378710526100')"), - 'tag__tag_id': '803', - 'threshold': GenericRepr("Decimal('0.67000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00084620605533321700')"), - 'tag__tag_id': '804', - 'threshold': GenericRepr("Decimal('0.75000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00070480359681823760')"), - 'tag__tag_id': '805', - 'threshold': GenericRepr("Decimal('0.64000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00703320267416958500')"), - 'tag__tag_id': '806', - 'threshold': GenericRepr("Decimal('0.53000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '902', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '903', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '904', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '905', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '906', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '907', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00131315333064554660')"), - 'tag__tag_id': '101', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00301082416073135700')"), - 'tag__tag_id': '103', - 'threshold': GenericRepr("Decimal('0.46000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00256628797311956700')"), - 'tag__tag_id': '104', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('2.67795523007710800000')"), - 'tag__tag_id': '105', - 'threshold': GenericRepr("Decimal('0.36000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01722483797685096000')"), - 'tag__tag_id': '106', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00367074832320213300')"), - 'tag__tag_id': '107', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00410134124816680450')"), - 'tag__tag_id': '108', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02810047168670029600')"), - 'tag__tag_id': '109', - 'threshold': GenericRepr("Decimal('0.58000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00356446807494475730')"), - 'tag__tag_id': '110', - 'threshold': GenericRepr("Decimal('0.42000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '1', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00885658950175879000')"), - 'tag__tag_id': '111', - 'threshold': GenericRepr("Decimal('0.53000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00161047546977275300')"), - 'tag__tag_id': '201', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00249261039939215920')"), - 'tag__tag_id': '202', - 'threshold': GenericRepr("Decimal('0.17000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00276056816801428800')"), - 'tag__tag_id': '203', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01951472795739466000')"), - 'tag__tag_id': '204', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00284144639848701430')"), - 'tag__tag_id': '205', - 'threshold': GenericRepr("Decimal('0.31000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00284233643800358870')"), - 'tag__tag_id': '206', - 'threshold': GenericRepr("Decimal('0.44000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00300193069657931750')"), - 'tag__tag_id': '207', - 'threshold': GenericRepr("Decimal('0.30000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00357941840775311000')"), - 'tag__tag_id': '208', - 'threshold': GenericRepr("Decimal('0.20000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00632169711239197600')"), - 'tag__tag_id': '209', - 'threshold': GenericRepr("Decimal('0.17000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00697963860938730400')"), - 'tag__tag_id': '210', - 'threshold': GenericRepr("Decimal('0.23000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01237871689035704300')"), - 'tag__tag_id': '212', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00024467206802345010')"), - 'tag__tag_id': '213', - 'threshold': GenericRepr("Decimal('0.37000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00036748812096354010')"), - 'tag__tag_id': '214', - 'threshold': GenericRepr("Decimal('0.29000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00047817865331461160')"), - 'tag__tag_id': '215', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00696396781131625200')"), - 'tag__tag_id': '216', - 'threshold': GenericRepr("Decimal('0.25000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00091310044249089870')"), - 'tag__tag_id': '217', - 'threshold': GenericRepr("Decimal('0.13000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00106291823053302660')"), - 'tag__tag_id': '218', - 'threshold': GenericRepr("Decimal('0.13000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00187798162057463590')"), - 'tag__tag_id': '219', - 'threshold': GenericRepr("Decimal('0.28000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01125925638038536600')"), - 'tag__tag_id': '220', - 'threshold': GenericRepr("Decimal('0.29000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00166666776701611900')"), - 'tag__tag_id': '221', - 'threshold': GenericRepr("Decimal('0.19000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00146527563629206270')"), - 'tag__tag_id': '222', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00115551359139065800')"), - 'tag__tag_id': '223', - 'threshold': GenericRepr("Decimal('0.47000000000000003000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00758105556347540500')"), - 'tag__tag_id': '224', - 'threshold': GenericRepr("Decimal('0.21000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00033728324827582890')"), - 'tag__tag_id': '225', - 'threshold': GenericRepr("Decimal('0.15000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00077029108069837090')"), - 'tag__tag_id': '226', - 'threshold': GenericRepr("Decimal('0.18000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00090097592975426880')"), - 'tag__tag_id': '227', - 'threshold': GenericRepr("Decimal('0.18000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '2', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00497279979754239300')"), - 'tag__tag_id': '228', - 'threshold': GenericRepr("Decimal('0.80000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.35999999999999998668')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '5', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00032880847216941987')"), - 'tag__tag_id': '229', - 'threshold': GenericRepr("Decimal('0.39000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '501', + 'threshold': GenericRepr("Decimal('0.45000000000000001110')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '5', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00116735643615233300')"), - 'tag__tag_id': '230', - 'threshold': GenericRepr("Decimal('0.81000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '502', + 'threshold': GenericRepr("Decimal('0.47999999999999998224')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '6', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00244935224877624970')"), - 'tag__tag_id': '231', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '601', + 'threshold': GenericRepr("Decimal('0.05999999999999999778')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '6', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00542857805671899200')"), - 'tag__tag_id': '232', - 'threshold': GenericRepr("Decimal('0.46000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '602', + 'threshold': GenericRepr("Decimal('0.47999999999999998224')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '6', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00188743645513925370')"), - 'tag__tag_id': '233', - 'threshold': GenericRepr("Decimal('0.79000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.02199999999999999872')"), + 'tag__tag_id': '603', + 'threshold': GenericRepr("Decimal('0.34000000000000002442')"), 'value': '' }, { - 'category__tag_id': '2', + 'category__tag_id': '6', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00117788418989490570')"), - 'tag__tag_id': '234', - 'threshold': GenericRepr("Decimal('0.54000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '604', + 'threshold': GenericRepr("Decimal('0.16000000000000000333')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '7', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00023104241032948875')"), - 'tag__tag_id': '301', - 'threshold': GenericRepr("Decimal('0.12000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00800000000000000017')"), + 'tag__tag_id': '701', + 'threshold': GenericRepr("Decimal('0.27000000000000001776')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '8', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00684022131126101400')"), - 'tag__tag_id': '302', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '801', + 'threshold': GenericRepr("Decimal('0.66000000000000003109')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '8', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, + 'draft_entry__excerpt': 'sample excerpt 101', + 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('1.51390548675291000000')"), - 'tag__tag_id': '303', - 'threshold': GenericRepr("Decimal('0.62000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '802', + 'threshold': GenericRepr("Decimal('0.29999999999999998890')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '9', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00246191542828455570')"), - 'tag__tag_id': '304', - 'threshold': GenericRepr("Decimal('0.10000000000000000000')"), + 'prediction': GenericRepr("Decimal('-1.00000000000000000000')"), + 'tag__tag_id': '904', + 'threshold': GenericRepr("Decimal('-1.00000000000000000000')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '9', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.19748103480006374000')"), - 'tag__tag_id': '305', - 'threshold': GenericRepr("Decimal('0.43000000000000000000')"), + 'prediction': GenericRepr("Decimal('-1.00000000000000000000')"), + 'tag__tag_id': '905', + 'threshold': GenericRepr("Decimal('-1.00000000000000000000')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '9', 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', + 'draft_entry__excerpt': 'sample excerpt 101', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.13266879380965720000')"), - 'tag__tag_id': '306', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), + 'prediction': GenericRepr("Decimal('-1.00000000000000000000')"), + 'tag__tag_id': '907', + 'threshold': GenericRepr("Decimal('-1.00000000000000000000')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00847395147300428900')"), - 'tag__tag_id': '307', - 'threshold': GenericRepr("Decimal('0.36000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00200000000000000004')"), + 'tag__tag_id': '101', + 'threshold': GenericRepr("Decimal('0.14000000000000001332')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, + 'is_selected': True, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.01439434579677051900')"), - 'tag__tag_id': '308', - 'threshold': GenericRepr("Decimal('0.45000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.64800000000000002043')"), + 'tag__tag_id': '102', + 'threshold': GenericRepr("Decimal('0.17000000000000001221')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00275349894147967100')"), - 'tag__tag_id': '309', - 'threshold': GenericRepr("Decimal('0.31000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.02699999999999999969')"), + 'tag__tag_id': '103', + 'threshold': GenericRepr("Decimal('0.10000000000000000555')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '1', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02261752535293742000')"), - 'tag__tag_id': '310', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), + 'prediction': GenericRepr("Decimal('0.06199999999999999956')"), + 'tag__tag_id': '104', + 'threshold': GenericRepr("Decimal('0.14000000000000001332')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00280699276022220930')"), - 'tag__tag_id': '311', - 'threshold': GenericRepr("Decimal('0.38000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00700000000000000015')"), + 'tag__tag_id': '204', + 'threshold': GenericRepr("Decimal('0.14000000000000001332')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, + 'is_selected': True, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00353863737969235940')"), - 'tag__tag_id': '312', - 'threshold': GenericRepr("Decimal('0.33000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.45800000000000001821')"), + 'tag__tag_id': '209', + 'threshold': GenericRepr("Decimal('0.05000000000000000278')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00474455507679118000')"), - 'tag__tag_id': '313', - 'threshold': GenericRepr("Decimal('0.45000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '214', + 'threshold': GenericRepr("Decimal('0.08999999999999999667')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00243518249286959600')"), - 'tag__tag_id': '314', - 'threshold': GenericRepr("Decimal('0.24000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00300000000000000006')"), + 'tag__tag_id': '216', + 'threshold': GenericRepr("Decimal('0.13000000000000000444')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00498411668972535500')"), - 'tag__tag_id': '315', - 'threshold': GenericRepr("Decimal('0.55000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '217', + 'threshold': GenericRepr("Decimal('0.04000000000000000083')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00342778279446065430')"), - 'tag__tag_id': '316', - 'threshold': GenericRepr("Decimal('0.15000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00400000000000000008')"), + 'tag__tag_id': '218', + 'threshold': GenericRepr("Decimal('0.08999999999999999667')"), 'value': '' }, { - 'category__tag_id': '3', + 'category__tag_id': '2', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00183609818729261560')"), - 'tag__tag_id': '317', - 'threshold': GenericRepr("Decimal('0.30000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00300000000000000006')"), + 'tag__tag_id': '219', + 'threshold': GenericRepr("Decimal('0.13000000000000000444')"), 'value': '' }, { @@ -2262,86 +1233,86 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00765169737860560400')"), - 'tag__tag_id': '318', - 'threshold': GenericRepr("Decimal('0.25000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '301', + 'threshold': GenericRepr("Decimal('0.01000000000000000021')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00020814205345232040')"), - 'tag__tag_id': '401', - 'threshold': GenericRepr("Decimal('0.25000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '302', + 'threshold': GenericRepr("Decimal('0.11000000000000000056')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00299776057263129780')"), - 'tag__tag_id': '402', - 'threshold': GenericRepr("Decimal('0.58000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.08300000000000000433')"), + 'tag__tag_id': '303', + 'threshold': GenericRepr("Decimal('0.38000000000000000444')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, + 'is_selected': True, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00299218206367056270')"), - 'tag__tag_id': '403', - 'threshold': GenericRepr("Decimal('0.14000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.08599999999999999312')"), + 'tag__tag_id': '304', + 'threshold': GenericRepr("Decimal('0.01000000000000000021')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00241597760274695900')"), - 'tag__tag_id': '404', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00300000000000000006')"), + 'tag__tag_id': '315', + 'threshold': GenericRepr("Decimal('0.45000000000000001110')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02053049989999868700')"), - 'tag__tag_id': '405', - 'threshold': GenericRepr("Decimal('0.78000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '316', + 'threshold': GenericRepr("Decimal('0.05999999999999999778')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00281014967745599850')"), - 'tag__tag_id': '406', - 'threshold': GenericRepr("Decimal('0.13000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00400000000000000008')"), + 'tag__tag_id': '317', + 'threshold': GenericRepr("Decimal('0.28000000000000002665')"), 'value': '' }, { - 'category__tag_id': '4', + 'category__tag_id': '3', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00022843408415366598')"), - 'tag__tag_id': '407', - 'threshold': GenericRepr("Decimal('0.41000000000000003000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '318', + 'threshold': GenericRepr("Decimal('0.13000000000000000444')"), 'value': '' }, { @@ -2350,9 +1321,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00943289911848003600')"), - 'tag__tag_id': '408', - 'threshold': GenericRepr("Decimal('0.59000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '401', + 'threshold': GenericRepr("Decimal('0.28999999999999998002')"), 'value': '' }, { @@ -2361,20 +1332,20 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00091892403101415500')"), - 'tag__tag_id': '409', - 'threshold': GenericRepr("Decimal('0.56000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '402', + 'threshold': GenericRepr("Decimal('0.45000000000000001110')"), 'value': '' }, { 'category__tag_id': '4', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, + 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('2.03979988487399360000')"), - 'tag__tag_id': '410', - 'threshold': GenericRepr("Decimal('0.49000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '407', + 'threshold': GenericRepr("Decimal('0.07000000000000000666')"), 'value': '' }, { @@ -2383,9 +1354,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00750677951145917200')"), - 'tag__tag_id': '411', - 'threshold': GenericRepr("Decimal('0.20000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), + 'tag__tag_id': '408', + 'threshold': GenericRepr("Decimal('0.11000000000000000056')"), 'value': '' }, { @@ -2394,20 +1365,20 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00019092757914525768')"), + 'prediction': GenericRepr("Decimal('0E-20')"), 'tag__tag_id': '412', - 'threshold': GenericRepr("Decimal('0.60000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.35999999999999998668')"), 'value': '' }, { 'category__tag_id': '5', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, + 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('1.40336945023335200000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), 'tag__tag_id': '501', - 'threshold': GenericRepr("Decimal('0.71000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.45000000000000001110')"), 'value': '' }, { @@ -2416,20 +1387,20 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00778131599707359600')"), + 'prediction': GenericRepr("Decimal('0E-20')"), 'tag__tag_id': '502', - 'threshold': GenericRepr("Decimal('0.44000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.47999999999999998224')"), 'value': '' }, { 'category__tag_id': '6', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, + 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.04068703080217044000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), 'tag__tag_id': '601', - 'threshold': GenericRepr("Decimal('0.48000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.05999999999999999778')"), 'value': '' }, { @@ -2438,9 +1409,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.02458783670921217300')"), + 'prediction': GenericRepr("Decimal('0.00100000000000000002')"), 'tag__tag_id': '602', - 'threshold': GenericRepr("Decimal('0.44000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.47999999999999998224')"), 'value': '' }, { @@ -2449,9 +1420,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.04259871318936348000')"), + 'prediction': GenericRepr("Decimal('0.02199999999999999872')"), 'tag__tag_id': '603', - 'threshold': GenericRepr("Decimal('0.40000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.34000000000000002442')"), 'value': '' }, { @@ -2460,53 +1431,20 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00641449491997234200')"), + 'prediction': GenericRepr("Decimal('0E-20')"), 'tag__tag_id': '604', - 'threshold': GenericRepr("Decimal('0.61000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00027582379877694870')"), - 'tag__tag_id': '801', - 'threshold': GenericRepr("Decimal('0.73000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00997524285181002000')"), - 'tag__tag_id': '802', - 'threshold': GenericRepr("Decimal('0.55000000000000000000')"), + 'threshold': GenericRepr("Decimal('0.16000000000000000333')"), 'value': '' }, { - 'category__tag_id': '8', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00476177378710526100')"), - 'tag__tag_id': '803', - 'threshold': GenericRepr("Decimal('0.67000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '8', + 'category__tag_id': '7', 'data_type': 1, 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00084620605533321700')"), - 'tag__tag_id': '804', - 'threshold': GenericRepr("Decimal('0.75000000000000000000')"), + 'prediction': GenericRepr("Decimal('0.00800000000000000017')"), + 'tag__tag_id': '701', + 'threshold': GenericRepr("Decimal('0.27000000000000001776')"), 'value': '' }, { @@ -2515,9 +1453,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00070480359681823760')"), - 'tag__tag_id': '805', - 'threshold': GenericRepr("Decimal('0.64000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '801', + 'threshold': GenericRepr("Decimal('0.66000000000000003109')"), 'value': '' }, { @@ -2526,20 +1464,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.00703320267416958500')"), - 'tag__tag_id': '806', - 'threshold': GenericRepr("Decimal('0.53000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '902', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), + 'prediction': GenericRepr("Decimal('0E-20')"), + 'tag__tag_id': '802', + 'threshold': GenericRepr("Decimal('0.29999999999999998890')"), 'value': '' }, { @@ -2548,20 +1475,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '903', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), + 'prediction': GenericRepr("Decimal('-1.00000000000000000000')"), 'tag__tag_id': '904', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), + 'threshold': GenericRepr("Decimal('-1.00000000000000000000')"), 'value': '' }, { @@ -2570,20 +1486,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), + 'prediction': GenericRepr("Decimal('-1.00000000000000000000')"), 'tag__tag_id': '905', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '9', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': False, - 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), - 'tag__tag_id': '906', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), + 'threshold': GenericRepr("Decimal('-1.00000000000000000000')"), 'value': '' }, { @@ -2592,31 +1497,9 @@ 'draft_entry__excerpt': 'sample excerpt 102', 'is_selected': False, 'model_version__model__model_id': 'all_tags_model', - 'prediction': GenericRepr("Decimal('0.50000000000000000000')"), + 'prediction': GenericRepr("Decimal('-1.00000000000000000000')"), 'tag__tag_id': '907', - 'threshold': GenericRepr("Decimal('0.50000000000000000000')"), - 'value': '' - }, - { - 'category__tag_id': '10', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 101', - 'is_selected': True, - 'model_version__model__model_id': 'reliability', - 'prediction': None, - 'tag__tag_id': '1002', - 'threshold': None, - 'value': '' - }, - { - 'category__tag_id': '10', - 'data_type': 1, - 'draft_entry__excerpt': 'sample excerpt 102', - 'is_selected': True, - 'model_version__model__model_id': 'reliability', - 'prediction': None, - 'tag__tag_id': '1002', - 'threshold': None, + 'threshold': GenericRepr("Decimal('-1.00000000000000000000')"), 'value': '' } ] @@ -2713,69 +1596,6 @@ 'parent_tag__tag_id': '1', 'tag_id': '104' }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'Health', - 'parent_tag__tag_id': '1', - 'tag_id': '105' - }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'Livelihoods', - 'parent_tag__tag_id': '1', - 'tag_id': '106' - }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'Logistics', - 'parent_tag__tag_id': '1', - 'tag_id': '107' - }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'Nutrition', - 'parent_tag__tag_id': '1', - 'tag_id': '108' - }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'Protection', - 'parent_tag__tag_id': '1', - 'tag_id': '109' - }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'Shelter', - 'parent_tag__tag_id': '1', - 'tag_id': '110' - }, - { - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'is_deprecated': False, - 'name': 'WASH', - 'parent_tag__tag_id': '1', - 'tag_id': '111' - }, { 'group': None, 'hide_in_analysis_framework_mapping': True, diff --git a/apps/assisted_tagging/tests/test_query.py b/apps/assisted_tagging/tests/test_query.py index 63488b594a..1a376a1586 100644 --- a/apps/assisted_tagging/tests/test_query.py +++ b/apps/assisted_tagging/tests/test_query.py @@ -333,516 +333,216 @@ def _query_check(**kwargs): class AssistedTaggingCallbackApiTest(TestCase, SnapShotTextCase): DEEPL_CALLBACK_MOCK_DATA = { - 'client_id': 'random-client-id', - 'model_preds': [ - { - 'tags': { - '1': { - '101': { - 'prediction': 0.0013131533306455466, - 'threshold': 0.41000000000000003, - 'is_selected': False, - }, - '103': { - 'prediction': 0.003010824160731357, - 'threshold': 0.46, - 'is_selected': False, - }, - '104': { - 'prediction': 0.002566287973119567, - 'threshold': 0.48, - 'is_selected': False, - }, - '105': { - 'prediction': 2.677955230077108, - 'threshold': 0.36, - 'is_selected': True, - }, - '106': { - 'prediction': 0.01722483797685096, - 'threshold': 0.38, - 'is_selected': False, - }, - '107': { - 'prediction': 0.003670748323202133, - 'threshold': 0.5, - 'is_selected': False, - }, - '108': { - 'prediction': 0.0041013412481668045, - 'threshold': 0.49, - 'is_selected': False, - }, - '109': { - 'prediction': 0.028100471686700296, - 'threshold': 0.58, - 'is_selected': False, - }, - '110': { - 'prediction': 0.0035644680749447573, - 'threshold': 0.42, - 'is_selected': False, - }, - '111': { - 'prediction': 0.00885658950175879, - 'threshold': 0.53, - 'is_selected': False, - } - }, - '3': { - '301': { - 'prediction': 0.00023104241032948875, - 'threshold': 0.12, - 'is_selected': False, - }, - '302': { - 'prediction': 0.006840221311261014, - 'threshold': 0.41000000000000003, - 'is_selected': False, - }, - '303': { - 'prediction': 1.51390548675291, - 'threshold': 0.62, - 'is_selected': True, - }, - '304': { - 'prediction': 0.0024619154282845557, - 'threshold': 0.1, - 'is_selected': False, - }, - '305': { - 'prediction': 0.19748103480006374, - 'threshold': 0.43, - 'is_selected': False, - }, - '306': { - 'prediction': 0.1326687938096572, - 'threshold': 0.49, - 'is_selected': False, - }, - '307': { - 'prediction': 0.008473951473004289, - 'threshold': 0.36, - 'is_selected': False, - }, - '308': { - 'prediction': 0.014394345796770519, - 'threshold': 0.45, - 'is_selected': False, - }, - '309': { - 'prediction': 0.002753498941479671, - 'threshold': 0.31, - 'is_selected': False, - }, - '310': { - 'prediction': 0.02261752535293742, - 'threshold': 0.41000000000000003, - 'is_selected': False, - }, - '311': { - 'prediction': 0.0028069927602222093, - 'threshold': 0.38, - 'is_selected': False, - }, - '312': { - 'prediction': 0.0035386373796923594, - 'threshold': 0.33, - 'is_selected': False, - }, - '313': { - 'prediction': 0.00474455507679118, - 'threshold': 0.45, - 'is_selected': False, - }, - '314': { - 'prediction': 0.002435182492869596, - 'threshold': 0.24, - 'is_selected': False, - }, - '315': { - 'prediction': 0.004984116689725355, - 'threshold': 0.55, - 'is_selected': False, - }, - '316': { - 'prediction': 0.0034277827944606543, - 'threshold': 0.15, - 'is_selected': False, - }, - '317': { - 'prediction': 0.0018360981872926156, - 'threshold': 0.3, - 'is_selected': False, - }, - '318': { - 'prediction': 0.007651697378605604, - 'threshold': 0.25, - 'is_selected': False, - } - }, - '2': { - '219': { - 'prediction': 0.0018779816205746359, - 'threshold': 0.28, - 'is_selected': False, - }, - '217': { - 'prediction': 0.0009131004424908987, - 'threshold': 0.13, - 'is_selected': False, - }, - '218': { - 'prediction': 0.0010629182305330266, - 'threshold': 0.13, - 'is_selected': False, - }, - '204': { - 'prediction': 0.01951472795739466, - 'threshold': 0.49, - 'is_selected': False, - }, - '203': { - 'prediction': 0.002760568168014288, - 'threshold': 0.41000000000000003, - 'is_selected': False, - }, - '201': { - 'prediction': 0.001610475469772753, - 'threshold': 0.38, - 'is_selected': False, - }, - '205': { - 'prediction': 0.0028414463984870143, - 'threshold': 0.31, - 'is_selected': False, - }, - '207': { - 'prediction': 0.0030019306965793175, - 'threshold': 0.3, - 'is_selected': False, - }, - '206': { - 'prediction': 0.0028423364380035887, - 'threshold': 0.44, - 'is_selected': False, - }, - '202': { - 'prediction': 0.0024926103993921592, - 'threshold': 0.17, - 'is_selected': False, - }, - '228': { - 'prediction': 0.004972799797542393, - 'threshold': 0.8, - 'is_selected': False, - }, - '229': { - 'prediction': 0.00032880847216941987, - 'threshold': 0.39, - 'is_selected': False, - }, - '230': { - 'prediction': 0.001167356436152333, - 'threshold': 0.81, - 'is_selected': False, - }, - '231': { - 'prediction': 0.0024493522487762497, - 'threshold': 0.41000000000000003, - 'is_selected': False, - }, - '232': { - 'prediction': 0.005428578056718992, - 'threshold': 0.46, - 'is_selected': False, - }, - '233': { - 'prediction': 0.0018874364551392537, - 'threshold': 0.79, - 'is_selected': False, - }, - '234': { - 'prediction': 0.0011778841898949057, - 'threshold': 0.54, - 'is_selected': False, - }, - '215': { - 'prediction': 0.0004781786533146116, - 'threshold': 0.38, - 'is_selected': False, - }, - '216': { - 'prediction': 0.006963967811316252, - 'threshold': 0.25, - 'is_selected': False, - }, - '214': { - 'prediction': 0.0003674881209635401, - 'threshold': 0.29, - 'is_selected': False, - }, - '213': { - 'prediction': 0.0002446720680234501, - 'threshold': 0.37, - 'is_selected': False, - }, - '212': { - 'prediction': 0.012378716890357043, - 'threshold': 0.38, - 'is_selected': False, - }, - '223': { - 'prediction': 0.001155513591390658, - 'threshold': 0.47000000000000003, - 'is_selected': False, - }, - '222': { - 'prediction': 0.0014652756362920627, - 'threshold': 0.48, - 'is_selected': False, - }, - '221': { - 'prediction': 0.001666667767016119, - 'threshold': 0.19, - 'is_selected': False, - }, - '220': { - 'prediction': 0.011259256380385366, - 'threshold': 0.29, - 'is_selected': False, - }, - '224': { - 'prediction': 0.007581055563475405, - 'threshold': 0.21, - 'is_selected': False, - }, - '225': { - 'prediction': 0.0003372832482758289, - 'threshold': 0.15, - 'is_selected': False, - }, - '227': { - 'prediction': 0.0009009759297542688, - 'threshold': 0.18, - 'is_selected': False, - }, - '226': { - 'prediction': 0.0007702910806983709, - 'threshold': 0.18, - 'is_selected': False, - }, - '210': { - 'prediction': 0.006979638609387304, - 'threshold': 0.23, - 'is_selected': False, - }, - '208': { - 'prediction': 0.00357941840775311, - 'threshold': 0.2, - 'is_selected': False, - }, - '209': { - 'prediction': 0.006321697112391976, - 'threshold': 0.17, - 'is_selected': False, - } - }, - '6': { - '601': { - 'prediction': 0.04068703080217044, - 'threshold': 0.48, - 'is_selected': True, - }, - '602': { - 'prediction': 0.024587836709212173, - 'threshold': 0.44, - 'is_selected': False, - }, - '603': { - 'prediction': 0.04259871318936348, - 'threshold': 0.4, - 'is_selected': False, - }, - '604': { - 'prediction': 0.006414494919972342, - 'threshold': 0.61, - 'is_selected': False, - } - }, - '5': { - '501': { - 'prediction': 1.403369450233352, - 'threshold': 0.71, - 'is_selected': True, - }, - '502': { - 'prediction': 0.007781315997073596, - 'threshold': 0.44, - 'is_selected': False, - } - }, - '8': { - '801': { - 'prediction': 0.0002758237987769487, - 'threshold': 0.73, - 'is_selected': False, - }, - '802': { - 'prediction': 0.00997524285181002, - 'threshold': 0.55, - 'is_selected': False, - }, - '803': { - 'prediction': 0.004761773787105261, - 'threshold': 0.67, - 'is_selected': False, - }, - '804': { - 'prediction': 0.000846206055333217, - 'threshold': 0.75, - 'is_selected': False, - }, - '805': { - 'prediction': 0.0007048035968182376, - 'threshold': 0.64, - 'is_selected': False, - }, - '806': { - 'prediction': 0.007033202674169585, - 'threshold': 0.53, - 'is_selected': False, - } - }, - '4': { - '401': { - 'prediction': 0.0002081420534523204, - 'threshold': 0.25, - 'is_selected': False, - }, - '402': { - 'prediction': 0.0029977605726312978, - 'threshold': 0.58, - 'is_selected': False, - }, - '403': { - 'prediction': 0.0029921820636705627, - 'threshold': 0.14, - 'is_selected': False, - }, - '404': { - 'prediction': 0.002415977602746959, - 'threshold': 0.48, - 'is_selected': False, - }, - '405': { - 'prediction': 0.020530499899998687, - 'threshold': 0.78, - 'is_selected': False, - }, - '406': { - 'prediction': 0.0028101496774559985, - 'threshold': 0.13, - 'is_selected': False, - }, - '407': { - 'prediction': 0.00022843408415366598, - 'threshold': 0.41000000000000003, - 'is_selected': False, - }, - '408': { - 'prediction': 0.009432899118480036, - 'threshold': 0.59, - 'is_selected': False, - }, - '409': { - 'prediction': 0.000918924031014155, - 'threshold': 0.56, - 'is_selected': False, - }, - '410': { - 'prediction': 2.0397998848739936, - 'threshold': 0.49, - 'is_selected': True, - }, - '411': { - 'prediction': 0.007506779511459172, - 'threshold': 0.2, - 'is_selected': False, - }, - '412': { - 'prediction': 0.00019092757914525768, - 'threshold': 0.6, - 'is_selected': False, - } - }, - '9': { - '904': { - 'prediction': 0.5, - 'threshold': 0.5, - 'is_selected': True, - }, - '905': { - 'prediction': 0.5, - 'threshold': 0.5, - 'is_selected': False, - }, - '902': { - 'prediction': 0.5, - 'threshold': 0.5, - 'is_selected': False, - }, - '903': { - 'prediction': 0.5, - 'threshold': 0.5, - 'is_selected': False, - }, - '906': { - 'prediction': 0.5, - 'threshold': 0.5, - 'is_selected': False, - }, - '907': { - 'prediction': 0.5, - 'threshold': 0.5, - 'is_selected': False, - } - } + "client_id": "random-client-id", + "model_tags": { + "1": { + "101": { + "prediction": 0.002, + "threshold": 0.14, + "is_selected": False }, - 'prediction_status': 1, - 'model_info': { - 'id': 'all_tags_model', - 'version': '1.0.0', + "102": { + "prediction": 0.648, + "threshold": 0.17, + "is_selected": True + }, + "103": { + "prediction": 0.027, + "threshold": 0.1, + "is_selected": False + }, + "104": { + "prediction": 0.062, + "threshold": 0.14, + "is_selected": False } }, - { - 'model_info': { - 'id': 'geolocation', - 'version': '1.0.0', + "3": { + "301": { + "prediction": 0.001, + "threshold": 0.01, + "is_selected": False }, - 'values': [ - 'Nepal', - 'Paris', - ], - 'prediction_status': 1 + "302": { + "prediction": 0.001, + "threshold": 0.11, + "is_selected": False + }, + "303": { + "prediction": 0.083, + "threshold": 0.38, + "is_selected": False + }, + "304": { + "prediction": 0.086, + "threshold": 0.01, + "is_selected": True + }, + "315": { + "prediction": 0.003, + "threshold": 0.45, + "is_selected": False + }, + "316": { + "prediction": 0.001, + "threshold": 0.06, + "is_selected": False + }, + "317": { + "prediction": 0.004, + "threshold": 0.28, + "is_selected": False + }, + "318": { + "prediction": 0.0, + "threshold": 0.13, + "is_selected": False + } + }, + "2": { + "219": { + "prediction": 0.003, + "threshold": 0.13, + "is_selected": False + }, + "217": { + "prediction": 0.001, + "threshold": 0.04, + "is_selected": False + }, + "218": { + "prediction": 0.004, + "threshold": 0.09, + "is_selected": False + }, + "204": { + "prediction": 0.007, + "threshold": 0.14, + "is_selected": False + }, + "216": { + "prediction": 0.003, + "threshold": 0.13, + "is_selected": False + }, + "214": { + "prediction": 0.001, + "threshold": 0.09, + "is_selected": False + }, + "209": { + "prediction": 0.458, + "threshold": 0.05, + "is_selected": True + } + }, + "6": { + "601": { + "prediction": 0.0, + "threshold": 0.06, + "is_selected": False + }, + "602": { + "prediction": 0.001, + "threshold": 0.48, + "is_selected": False + }, + "603": { + "prediction": 0.022, + "threshold": 0.34, + "is_selected": False + }, + "604": { + "prediction": 0.0, + "threshold": 0.16, + "is_selected": False + } + }, + "5": { + "501": { + "prediction": 0.0, + "threshold": 0.45, + "is_selected": False + }, + "502": { + "prediction": 0.0, + "threshold": 0.48, + "is_selected": False + } + }, + "8": { + "801": { + "prediction": 0.0, + "threshold": 0.66, + "is_selected": False + }, + "802": { + "prediction": 0.0, + "threshold": 0.3, + "is_selected": False + } }, - { - 'model_info': { - 'id': 'reliability', - 'version': '1.0.0', + "4": { + "401": { + "prediction": 0.001, + "threshold": 0.29, + "is_selected": False }, - 'tags': { - '10': { - '1002': { - 'is_selected': True, - } - } + "402": { + "prediction": 0.001, + "threshold": 0.45, + "is_selected": False }, - 'prediction_status': 1 + "407": { + "prediction": 0.0, + "threshold": 0.07, + "is_selected": False + }, + "408": { + "prediction": 0.001, + "threshold": 0.11, + "is_selected": False + }, + "412": { + "prediction": 0.0, + "threshold": 0.36, + "is_selected": False + } + }, + "7": { + "701": { + "prediction": 0.008, + "threshold": 0.27, + "is_selected": False + } + }, + "9": { + "904": { + "prediction": -1, + "threshold": -1, + "is_selected": False + }, + "905": { + "prediction": -1, + "threshold": -1, + "is_selected": False + }, + "907": { + "prediction": -1, + "threshold": -1, + "is_selected": False + } } - ] + }, + "geolocations": [ + "New York" + ], + "model_info": { + "id": "all_tags_model", + "version": "1.0.0" + }, + "prediction_status": True } - DEEPL_TAGS_MOCK_RESPONSE = { '101': { 'label': 'Agriculture', @@ -872,55 +572,6 @@ class AssistedTaggingCallbackApiTest(TestCase, SnapShotTextCase): 'is_category': False, 'parent_id': '1', }, - '105': { - 'label': 'Health', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '106': { - 'label': 'Livelihoods', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '107': { - 'label': 'Logistics', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '108': { - 'label': 'Nutrition', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '109': { - 'label': 'Protection', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '110': { - 'label': 'Shelter', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '111': { - 'label': 'WASH', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, '201': { 'label': 'Environment', 'group': 'Context', diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index a99617560a..2c8d744cb2 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -347,6 +347,8 @@ def save_data(cls, draft_entry, data): class AutoAssistedTaggingDraftEntryHandler(BaseHandler): + # TODO: Fix N+1 issues here. Try to do bulk_update for each models. + # Or do this Async model = Lead callback_url_name = 'auto-assisted_tagging_draft_entry_prediction_callback' @@ -503,9 +505,12 @@ def _process_model_preds(cls, model_version, current_tags_map, draft_entry, mode @classmethod @transaction.atomic - def save_data(cls, lead, data): - draft_entry = DraftEntry.objects.filter(lead=lead, type=0) - if draft_entry.exists(): + def save_data(cls, lead, data_url): + # NOTE: Schema defined here + # - https://docs.google.com/document/d/1NmjOO5sOrhJU6b4QXJBrGAVk57_NW87mLJ9wzeY_NZI/edit#heading=h.t3u7vdbps5pt + data = RequestHelper(url=data_url, ignore_error=True).json() + draft_entry_qs = DraftEntry.objects.filter(lead=lead, type=DraftEntry.Type.AUTO) + if draft_entry_qs.exists(): raise serializers.ValidationError('Draft entries already exit') for model_preds in data['blocks']: if not model_preds['relevant']: @@ -528,9 +533,8 @@ def save_data(cls, lead, data): lead=lead, excerpt=model_preds['text'], prediction_status=DraftEntry.PredictionStatus.DONE, - type=0 + type=DraftEntry.Type.AUTO ) - draft.save() model_version = models_version_map[ (data['classification_model_info']['name'], data['classification_model_info']['version']) ] diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index abebc17625..0f0924dcf5 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -31,8 +31,6 @@ from .models import DeeplTrackBaseModel -from utils.request import RequestHelper - class BaseCallbackSerializer(serializers.Serializer): nlp_handler: Type[BaseHandler] @@ -248,10 +246,9 @@ class AutoAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): def create(self, validated_data): obj = validated_data['object'] - validated_data = RequestHelper(url=validated_data['entry_extraction_classification_path'], ignore_error=True).json() return self.nlp_handler.save_data( obj, - validated_data + validated_data['entry_extraction_classification_path'], ) diff --git a/apps/deepl_integration/views.py b/apps/deepl_integration/views.py index 115e918e98..264d0ce6ec 100644 --- a/apps/deepl_integration/views.py +++ b/apps/deepl_integration/views.py @@ -24,7 +24,6 @@ class BaseCallbackView(views.APIView): permission_classes = [permissions.AllowAny] def post(self, request, **_): - print(request.data) serializer = self.serializer(data=request.data) serializer.is_valid(raise_exception=True) serializer.save() diff --git a/apps/lead/enums.py b/apps/lead/enums.py index 11b86805ce..a767fe58de 100644 --- a/apps/lead/enums.py +++ b/apps/lead/enums.py @@ -12,6 +12,9 @@ LeadPriorityEnum = convert_enum_to_graphene_enum(Lead.Priority, name='LeadPriorityEnum') LeadSourceTypeEnum = convert_enum_to_graphene_enum(Lead.SourceType, name='LeadSourceTypeEnum') LeadExtractionStatusEnum = convert_enum_to_graphene_enum(Lead.ExtractionStatus, name='LeadExtractionStatusEnum') +LeadAutoEntryExtractionTypeEnum = convert_enum_to_graphene_enum( + Lead.AutoExtractionStatus, name='LeadAutoEntryExtractionTypeEnum' +) enum_map = { get_enum_name_from_django_field(field): enum @@ -21,6 +24,7 @@ (Lead.priority, LeadPriorityEnum), (Lead.source_type, LeadSourceTypeEnum), (Lead.extraction_status, LeadExtractionStatusEnum), + (Lead.auto_entry_extraction_status, LeadAutoEntryExtractionTypeEnum), ) } diff --git a/apps/lead/schema.py b/apps/lead/schema.py index 354d07bae0..5d068531e3 100644 --- a/apps/lead/schema.py +++ b/apps/lead/schema.py @@ -40,6 +40,7 @@ LeadPriorityEnum, LeadSourceTypeEnum, LeadExtractionStatusEnum, + LeadAutoEntryExtractionTypeEnum, ) from .filter_set import ( LeadGQFilterSet, @@ -367,6 +368,7 @@ class Meta: # For external accessible link share_view_url = graphene.String(required=True) + auto_entry_extraction_status = graphene.Field(LeadAutoEntryExtractionTypeEnum) @staticmethod def get_custom_queryset(queryset, info, **kwargs): @@ -473,7 +475,6 @@ class Query: emm_risk_factors = graphene.List(graphene.NonNull(EmmKeyRiskFactorType)) user_saved_lead_filter = graphene.Field(UserSavedLeadFilterType) - draft_entry_discarded_count = graphene.Field(DraftEntryCountByLead) @staticmethod def resolve_leads(root, info, **kwargs) -> QuerySet: diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 6cec3f5694..1b0a21421d 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1803,6 +1803,7 @@ def test_extractor_callback_url(self, get_file_mock, get_text_mock, index_lead_f 'total_words_count': 300, 'total_pages': 4, 'status': DeeplServerBaseCallbackSerializer.Status.FAILED.value, + 'text_extraction_id': '00431349-5879-4d59-9827-0b12491c4baa' } # After callback [Failure] @@ -1867,14 +1868,14 @@ def test_entry_extraction_callback(self): data = { "client_id": AutoAssistedTaggingDraftEntryHandler.get_client_id(self.lead), 'entry_extraction_classification_path': 'https://server-deepl.dev.datafriendlyspace.org/media/mock_responses/entry_extraction/entry-extraction-client-6ppp.json', # noqa: E501 - 'text_extraction_id': self.lead_preview.text_extraction_id, + 'text_extraction_id': str(self.lead_preview.text_extraction_id), 'status': 1 } response = self.client.post(url, data) self.assert_200(response) self.lead.refresh_from_db() self.assertEqual(self.lead.auto_entry_extraction_status, Lead.AutoExtractionStatus.SUCCESS) - self.assertEqual(LeadPreview.objects.get(lead=self.lead).text_extraction_id, data['text_extraction_id']) + self.assertEqual(str(LeadPreview.objects.get(lead=self.lead).text_extraction_id), data['text_extraction_id']) def test_auto_extraction_mutation(self): pass diff --git a/apps/unified_connector/tests/test_mutation.py b/apps/unified_connector/tests/test_mutation.py index 9ee21837b3..000460172b 100644 --- a/apps/unified_connector/tests/test_mutation.py +++ b/apps/unified_connector/tests/test_mutation.py @@ -372,14 +372,14 @@ def _query_okay_check(): ( 'source-invalid', [500, 'invalid-content'], - [200, {}], + [202, {}], ConnectorSource.Status.FAILURE, [], ), ( 'all-good', [200, RELIEF_WEB_MOCK_DATA_PAGE_2_RAW], - [200, {}], + [202, {}], ConnectorSource.Status.SUCCESS, [ConnectorLead.ExtractionStatus.STARTED], ), diff --git a/schema.graphql b/schema.graphql index 3bfcbf18da..73913f8d9a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -974,6 +974,7 @@ type AppEnumCollection { LeadPriority: [AppEnumCollectionLeadPriority!] LeadSourceType: [AppEnumCollectionLeadSourceType!] LeadExtractionStatus: [AppEnumCollectionLeadExtractionStatus!] + LeadAutoEntryExtractionStatus: [AppEnumCollectionLeadAutoEntryExtractionStatus!] EntryEntryType: [AppEnumCollectionEntryEntryType!] ExportFormat: [AppEnumCollectionExportFormat!] ExportStatus: [AppEnumCollectionExportStatus!] @@ -1006,7 +1007,6 @@ type AppEnumCollection { DraftEntryPredictionStatus: [AppEnumCollectionDraftEntryPredictionStatus!] AssistedTaggingPredictionDataType: [AppEnumCollectionAssistedTaggingPredictionDataType!] DraftEntryType: [AppEnumCollectionDraftEntryType!] - LeadAutoEntryExtractionStatus: [AppEnumCollectionLeadAutoEntryExtractionStatus!] UnusedAssessmentMethodologyProtectionInfo: [AppEnumCollectionUnusedAssessmentMethodologyProtectionInfo!] } @@ -1203,7 +1203,7 @@ type AppEnumCollectionGroupMembershipRole { } type AppEnumCollectionLeadAutoEntryExtractionStatus { - enum: AutoEntryExtractionTypeEnum! + enum: LeadAutoEntryExtractionTypeEnum! label: String! description: String } @@ -1425,7 +1425,6 @@ type AssistedTaggingPredictionType { type AssistedTaggingQueryType { draftEntry(id: ID!): DraftEntryType draftEntries(lead: ID, draftEntryTypes: [DraftEntryTypeEnum!], isDiscarded: Boolean, page: Int = 1, pageSize: Int): DraftEntryListType - extractionStatusByLead(leadId: ID!): AutoExtractionStatusType } type AssistedTaggingRootQueryType { @@ -1459,18 +1458,6 @@ type AttributeType { geoSelectedOptions: [ProjectGeoAreaType!] } -enum AutoEntryExtractionTypeEnum { - NONE - STARTED - PENDING - SUCCESS - FAILED -} - -type AutoExtractionStatusType { - autoEntryExtractionStatus: AutoEntryExtractionTypeEnum! -} - enum AutomaticSummaryStatusEnum { PENDING STARTED @@ -2691,6 +2678,14 @@ type JwtTokenType { expiresIn: String } +enum LeadAutoEntryExtractionTypeEnum { + NONE + STARTED + PENDING + SUCCESS + FAILED +} + enum LeadConfidentialityEnum { UNPROTECTED RESTRICTED @@ -2743,6 +2738,7 @@ type LeadDetailType { filteredEntriesCount: Int duplicateLeadsCount: Int shareViewUrl: String! + autoEntryExtractionStatus: LeadAutoEntryExtractionTypeEnum entries: [EntryType!] draftEntryStat: DraftEntryCountByLead } @@ -2932,6 +2928,7 @@ type LeadType { filteredEntriesCount: Int duplicateLeadsCount: Int shareViewUrl: String! + autoEntryExtractionStatus: LeadAutoEntryExtractionTypeEnum } input LeadsFilterDataInputType { @@ -3307,7 +3304,6 @@ type ProjectDetailType { emmKeywords: [EmmKeyWordType!] emmRiskFactors: [EmmKeyRiskFactorType!] userSavedLeadFilter: UserSavedLeadFilterType - draftEntryDiscardedCount: DraftEntryCountByLead activityLog: GenericScalar recentActiveUsers: [UserEntityDateType!] topSourcers: [UserEntityCountType!] @@ -3994,7 +3990,6 @@ type TriggerAnalysisTopicModel { type TriggerAutoDraftEntry { errors: [GenericScalar!] ok: Boolean - result: LeadType } input TriggerAutoDraftEntryInputType { From 6ca89b16b9a589c17f8b259902c284b8037ad825 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Thu, 21 Dec 2023 17:28:55 +0545 Subject: [PATCH 23/24] Squash PR migrations --- ...rename_draft_entry_type_draftentry_type.py | 25 +++++++++++++++ ...1_0926_squashed_0054_auto_20231218_0552.py | 32 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type_squashed_0013_rename_draft_entry_type_draftentry_type.py create mode 100644 apps/lead/migrations/0049_auto_20231121_0926_squashed_0054_auto_20231218_0552.py diff --git a/apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type_squashed_0013_rename_draft_entry_type_draftentry_type.py b/apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type_squashed_0013_rename_draft_entry_type_draftentry_type.py new file mode 100644 index 0000000000..e40d88d2d3 --- /dev/null +++ b/apps/assisted_tagging/migrations/0011_draftentry_draft_entry_type_squashed_0013_rename_draft_entry_type_draftentry_type.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.17 on 2023-12-21 11:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('assisted_tagging', '0011_draftentry_draft_entry_type'), ('assisted_tagging', '0012_draftentry_is_discarded'), ('assisted_tagging', '0013_rename_draft_entry_type_draftentry_type')] + + dependencies = [ + ('assisted_tagging', '0010_draftentry_related_geoareas'), + ] + + operations = [ + migrations.AddField( + model_name='draftentry', + name='is_discarded', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='draftentry', + name='type', + field=models.SmallIntegerField(choices=[(0, 'Auto Extraction'), (1, 'Manual Extraction')], default=1), + ), + ] diff --git a/apps/lead/migrations/0049_auto_20231121_0926_squashed_0054_auto_20231218_0552.py b/apps/lead/migrations/0049_auto_20231121_0926_squashed_0054_auto_20231218_0552.py new file mode 100644 index 0000000000..ab97917c90 --- /dev/null +++ b/apps/lead/migrations/0049_auto_20231121_0926_squashed_0054_auto_20231218_0552.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.17 on 2023-12-21 11:43 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('lead', '0049_auto_20231121_0926'), ('lead', '0050_delete_extractedlead'), ('lead', '0051_auto_20231128_0958'), ('lead', '0052_auto_20231130_0615'), ('lead', '0053_alter_lead_auto_entry_extraction_status'), ('lead', '0054_auto_20231218_0552')] + + dependencies = [ + ('lead', '0048_auto_20230228_0810'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='lead', + name='auto_entry_extraction_status', + field=models.SmallIntegerField(choices=[(0, 'None'), (1, 'Started'), (2, 'Pending'), (3, 'Success'), (4, 'Failed')], default=0), + ), + migrations.AddField( + model_name='leadpreview', + name='entry_extraction_id', + field=models.UUIDField(blank=True, null=True), + ), + migrations.AddField( + model_name='leadpreview', + name='text_extraction_id', + field=models.UUIDField(blank=True, null=True), + ), + ] From ffb969b6c437be57a1310285b63d760ed0a08794 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Thu, 21 Dec 2023 17:51:00 +0545 Subject: [PATCH 24/24] Hack test fix - Real configuration is having issue in test case --- .../tests/snapshots/snap_test_query.py | 613 +++--------------- apps/assisted_tagging/tests/test_query.py | 18 +- apps/deepl_integration/handlers.py | 3 +- 3 files changed, 96 insertions(+), 538 deletions(-) diff --git a/apps/assisted_tagging/tests/snapshots/snap_test_query.py b/apps/assisted_tagging/tests/snapshots/snap_test_query.py index ec82ec4f53..c20c3ca0c4 100644 --- a/apps/assisted_tagging/tests/snapshots/snap_test_query.py +++ b/apps/assisted_tagging/tests/snapshots/snap_test_query.py @@ -22,686 +22,231 @@ 'name': 'all_tags_model' } ], - 'tag_count': 136, + 'tag_count': 45, 'tags': [ { 'is_deprecated': False, - 'name': 'reliability', - 'tag_id': '10' - }, - { - 'is_deprecated': False, - 'name': 'sectors', + 'name': '1', 'tag_id': '1' }, { 'is_deprecated': False, - 'name': 'subpillars_2d', - 'tag_id': '3' - }, - { - 'is_deprecated': False, - 'name': 'subpillars_1d', - 'tag_id': '2' - }, - { - 'is_deprecated': False, - 'name': 'age', - 'tag_id': '6' - }, - { - 'is_deprecated': False, - 'name': 'gender', - 'tag_id': '5' - }, - { - 'is_deprecated': False, - 'name': 'affected_groups', - 'tag_id': '8' - }, - { - 'is_deprecated': False, - 'name': 'specific_needs_groups', - 'tag_id': '4' - }, - { - 'is_deprecated': False, - 'name': 'severity', - 'tag_id': '7' - }, - { - 'is_deprecated': False, - 'name': 'demographic_group', - 'tag_id': '9' - }, - { - 'is_deprecated': False, - 'name': 'Health', - 'tag_id': '105' - }, - { - 'is_deprecated': False, - 'name': 'Livelihoods', - 'tag_id': '106' - }, - { - 'is_deprecated': False, - 'name': 'Logistics', - 'tag_id': '107' - }, - { - 'is_deprecated': False, - 'name': 'Nutrition', - 'tag_id': '108' - }, - { - 'is_deprecated': False, - 'name': 'Protection', - 'tag_id': '109' - }, - { - 'is_deprecated': False, - 'name': 'Shelter', - 'tag_id': '110' - }, - { - 'is_deprecated': False, - 'name': 'WASH', - 'tag_id': '111' - }, - { - 'is_deprecated': False, - 'name': 'Environment', - 'tag_id': '201' - }, - { - 'is_deprecated': False, - 'name': 'Socio Cultural', - 'tag_id': '202' - }, - { - 'is_deprecated': False, - 'name': 'Economy', - 'tag_id': '203' - }, - { - 'is_deprecated': False, - 'name': 'Legal & Policy', - 'tag_id': '205' - }, - { - 'is_deprecated': False, - 'name': 'Security & Stability', - 'tag_id': '206' - }, - { - 'is_deprecated': False, - 'name': 'Politics', - 'tag_id': '207' - }, - { - 'is_deprecated': False, - 'name': 'Type And Characteristics', - 'tag_id': '208' - }, - { - 'is_deprecated': False, - 'name': 'Hazard & Threats', - 'tag_id': '210' - }, - { - 'is_deprecated': False, - 'name': 'Type/Numbers/Movements', - 'tag_id': '212' - }, - { - 'is_deprecated': False, - 'name': 'Push Factors', - 'tag_id': '213' - }, - { - 'is_deprecated': False, - 'name': 'Intentions', - 'tag_id': '215' - }, - { - 'is_deprecated': False, - 'name': 'Relief To Population', - 'tag_id': '220' - }, - { - 'is_deprecated': False, - 'name': 'Population To Relief', - 'tag_id': '221' - }, - { - 'is_deprecated': False, - 'name': 'Physical Constraints', - 'tag_id': '222' - }, - { - 'is_deprecated': False, - 'name': 'Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps', - 'tag_id': '223' - }, - { - 'is_deprecated': False, - 'name': 'Communication Means And Preferences', - 'tag_id': '224' - }, - { - 'is_deprecated': False, - 'name': 'Information Challenges And Barriers', - 'tag_id': '225' - }, - { - 'is_deprecated': False, - 'name': 'Knowledge And Info Gaps (Pop)', - 'tag_id': '226' - }, - { - 'is_deprecated': False, - 'name': 'Knowledge And Info Gaps (Hum)', - 'tag_id': '227' - }, - { - 'is_deprecated': False, - 'name': 'Cases', - 'tag_id': '228' - }, - { - 'is_deprecated': False, - 'name': 'Contact Tracing', - 'tag_id': '229' - }, - { - 'is_deprecated': False, - 'name': 'Deaths', - 'tag_id': '230' - }, - { - 'is_deprecated': False, - 'name': 'Hospitalization & Care', - 'tag_id': '231' - }, - { - 'is_deprecated': False, - 'name': 'Restriction Measures', - 'tag_id': '232' - }, - { - 'is_deprecated': False, - 'name': 'Testing', - 'tag_id': '233' - }, - { - 'is_deprecated': False, - 'name': 'Vaccination', - 'tag_id': '234' - }, - { - 'is_deprecated': False, - 'name': 'Technological', - 'tag_id': '235' - }, - { - 'is_deprecated': False, - 'name': 'Prevention campaign', - 'tag_id': '236' - }, - { - 'is_deprecated': False, - 'name': 'Research and outlook', - 'tag_id': '237' - }, - { - 'is_deprecated': False, - 'name': 'National Response', - 'tag_id': '305' - }, - { - 'is_deprecated': False, - 'name': 'Number Of People Reached/Response Gaps', - 'tag_id': '306' - }, - { - 'is_deprecated': False, - 'name': 'Coping Mechanisms', - 'tag_id': '307' - }, - { - 'is_deprecated': False, - 'name': 'Living Standards', - 'tag_id': '308' - }, - { - 'is_deprecated': False, - 'name': 'Number Of People In Need', - 'tag_id': '309' - }, - { - 'is_deprecated': False, - 'name': 'Physical And Mental Well Being', - 'tag_id': '310' - }, - { - 'is_deprecated': False, - 'name': 'Driver/Aggravating Factors', - 'tag_id': '311' - }, - { - 'is_deprecated': False, - 'name': 'Impact On People', - 'tag_id': '312' - }, - { - 'is_deprecated': False, - 'name': 'Impact On Systems, Services And Networks', - 'tag_id': '313' - }, - { - 'is_deprecated': False, - 'name': 'Number Of People Affected', - 'tag_id': '314' - }, - { - 'is_deprecated': False, - 'name': 'Humanitarian coordination', - 'tag_id': '319' - }, - { - 'is_deprecated': False, - 'name': 'People reached/response gaps', - 'tag_id': '320' - }, - { - 'is_deprecated': False, - 'name': 'Red cross/red crescent', - 'tag_id': '321' - }, - { - 'is_deprecated': False, - 'name': 'Elderly Head of Household', - 'tag_id': '403' - }, - { - 'is_deprecated': False, - 'name': 'Female Head of Household', - 'tag_id': '404' - }, - { - 'is_deprecated': False, - 'name': 'GBV survivors', - 'tag_id': '405' - }, - { - 'is_deprecated': False, - 'name': 'Indigenous people', - 'tag_id': '406' - }, - { - 'is_deprecated': False, - 'name': 'Persons with Disability', - 'tag_id': '409' - }, - { - 'is_deprecated': False, - 'name': 'Pregnant or Lactating Women', - 'tag_id': '410' - }, - { - 'is_deprecated': False, - 'name': 'Single Women (including Widows)', - 'tag_id': '411' - }, - { - 'is_deprecated': False, - 'name': 'Lgbtqia+', - 'tag_id': '413' - }, - { - 'is_deprecated': False, - 'name': 'Unaccompanied or/and separated children', - 'tag_id': '414' - }, - { - 'is_deprecated': False, - 'name': 'Infants/Toddlers (<5 years old) ', - 'tag_id': '901' - }, - { - 'is_deprecated': False, - 'name': 'Female Children/Youth (5 to 17 years old)', - 'tag_id': '902' - }, - { - 'is_deprecated': False, - 'name': 'Male Children/Youth (5 to 17 years old)', - 'tag_id': '903' - }, - { - 'is_deprecated': False, - 'name': 'Female Older Persons (60+ years old)', - 'tag_id': '906' - }, - { - 'is_deprecated': False, - 'name': 'Major', - 'tag_id': '702' - }, - { - 'is_deprecated': False, - 'name': 'Minor Problem', - 'tag_id': '703' - }, - { - 'is_deprecated': False, - 'name': 'No problem', - 'tag_id': '704' - }, - { - 'is_deprecated': False, - 'name': 'Of Concern', - 'tag_id': '705' - }, - { - 'is_deprecated': False, - 'name': 'Critical issue', - 'tag_id': '706' - }, - { - 'is_deprecated': False, - 'name': 'Issue of concern', - 'tag_id': '707' - }, - { - 'is_deprecated': False, - 'name': 'Minor issue', - 'tag_id': '708' - }, - { - 'is_deprecated': False, - 'name': 'No issue', - 'tag_id': '709' - }, - { - 'is_deprecated': False, - 'name': 'Severe issue', - 'tag_id': '710' - }, - { - 'is_deprecated': False, - 'name': 'IDP', - 'tag_id': '803' - }, - { - 'is_deprecated': False, - 'name': 'Migrants', - 'tag_id': '804' - }, - { - 'is_deprecated': False, - 'name': 'Refugees', - 'tag_id': '805' - }, - { - 'is_deprecated': False, - 'name': 'Returnees', - 'tag_id': '806' - }, - { - 'is_deprecated': False, - 'name': 'Completely reliable', - 'tag_id': '1001' - }, - { - 'is_deprecated': False, - 'name': 'Usually reliable', - 'tag_id': '1002' - }, - { - 'is_deprecated': False, - 'name': 'Fairly Reliable', - 'tag_id': '1003' - }, - { - 'is_deprecated': False, - 'name': 'Unreliable', - 'tag_id': '1004' - }, - { - 'is_deprecated': False, - 'name': 'All', - 'tag_id': '503' - }, - { - 'is_deprecated': False, - 'name': '12-17 years old', - 'tag_id': '605' - }, - { - 'is_deprecated': False, - 'name': '18-24 years old', - 'tag_id': '606' + 'name': '101', + 'tag_id': '101' }, { 'is_deprecated': False, - 'name': '18-59 years old', - 'tag_id': '607' + 'name': '102', + 'tag_id': '102' }, { 'is_deprecated': False, - 'name': '25-59 years old', - 'tag_id': '608' + 'name': '103', + 'tag_id': '103' }, { 'is_deprecated': False, - 'name': '5-11 years old', - 'tag_id': '609' + 'name': '104', + 'tag_id': '104' }, { 'is_deprecated': False, - 'name': '5-17 years old', - 'tag_id': '610' + 'name': '2', + 'tag_id': '2' }, { 'is_deprecated': False, - 'name': '<18 years', - 'tag_id': '611' + 'name': '204', + 'tag_id': '204' }, { 'is_deprecated': False, - 'name': '<18 years old', - 'tag_id': '612' + 'name': '209', + 'tag_id': '209' }, { 'is_deprecated': False, - 'name': '<5 years old', - 'tag_id': '613' + 'name': '214', + 'tag_id': '214' }, { 'is_deprecated': False, - 'name': '>60 years old', - 'tag_id': '614' + 'name': '216', + 'tag_id': '216' }, { 'is_deprecated': False, - 'name': 'Agriculture', - 'tag_id': '101' + 'name': '217', + 'tag_id': '217' }, { 'is_deprecated': False, - 'name': 'Cross', - 'tag_id': '102' + 'name': '218', + 'tag_id': '218' }, { 'is_deprecated': False, - 'name': 'Education', - 'tag_id': '103' + 'name': '219', + 'tag_id': '219' }, { 'is_deprecated': False, - 'name': 'Food Security', - 'tag_id': '104' + 'name': '3', + 'tag_id': '3' }, { 'is_deprecated': False, - 'name': 'Number Of People At Risk', + 'name': '301', 'tag_id': '301' }, { 'is_deprecated': False, - 'name': 'Risk And Vulnerabilities', + 'name': '302', 'tag_id': '302' }, { 'is_deprecated': False, - 'name': 'International Response', + 'name': '303', 'tag_id': '303' }, { 'is_deprecated': False, - 'name': 'Local Response', + 'name': '304', 'tag_id': '304' }, { 'is_deprecated': False, - 'name': 'Expressed By Humanitarian Staff', + 'name': '315', 'tag_id': '315' }, { 'is_deprecated': False, - 'name': 'Expressed By Population', + 'name': '316', 'tag_id': '316' }, { 'is_deprecated': False, - 'name': 'Expressed By Humanitarian Staff', + 'name': '317', 'tag_id': '317' }, { 'is_deprecated': False, - 'name': 'Expressed By Population', + 'name': '318', 'tag_id': '318' }, { 'is_deprecated': False, - 'name': 'Dead', - 'tag_id': '219' - }, - { - 'is_deprecated': False, - 'name': 'Injured', - 'tag_id': '217' + 'name': '4', + 'tag_id': '4' }, { 'is_deprecated': False, - 'name': 'Missing', - 'tag_id': '218' + 'name': '401', + 'tag_id': '401' }, { 'is_deprecated': False, - 'name': 'Demography', - 'tag_id': '204' + 'name': '402', + 'tag_id': '402' }, { 'is_deprecated': False, - 'name': 'Local Integration', - 'tag_id': '216' + 'name': '407', + 'tag_id': '407' }, { 'is_deprecated': False, - 'name': 'Pull Factors', - 'tag_id': '214' + 'name': '408', + 'tag_id': '408' }, { 'is_deprecated': False, - 'name': 'Underlying/Aggravating Factors', - 'tag_id': '209' + 'name': '412', + 'tag_id': '412' }, { 'is_deprecated': False, - 'name': 'Adult (18 to 59 years old)', - 'tag_id': '601' + 'name': '5', + 'tag_id': '5' }, { 'is_deprecated': False, - 'name': 'Children/Youth (5 to 17 years old)', - 'tag_id': '602' + 'name': '501', + 'tag_id': '501' }, { 'is_deprecated': False, - 'name': 'Infants/Toddlers (<5 years old)', - 'tag_id': '603' + 'name': '502', + 'tag_id': '502' }, { 'is_deprecated': False, - 'name': 'Older Persons (60+ years old)', - 'tag_id': '604' + 'name': '6', + 'tag_id': '6' }, { 'is_deprecated': False, - 'name': 'Female', - 'tag_id': '501' + 'name': '601', + 'tag_id': '601' }, { 'is_deprecated': False, - 'name': 'Male', - 'tag_id': '502' + 'name': '602', + 'tag_id': '602' }, { 'is_deprecated': False, - 'name': 'Asylum Seekers', - 'tag_id': '801' + 'name': '603', + 'tag_id': '603' }, { 'is_deprecated': False, - 'name': 'Host', - 'tag_id': '802' + 'name': '604', + 'tag_id': '604' }, { 'is_deprecated': False, - 'name': 'Child Head of Household', - 'tag_id': '401' + 'name': '7', + 'tag_id': '7' }, { 'is_deprecated': False, - 'name': 'Chronically Ill', - 'tag_id': '402' + 'name': '701', + 'tag_id': '701' }, { 'is_deprecated': False, - 'name': 'LGBTQI+', - 'tag_id': '407' + 'name': '8', + 'tag_id': '8' }, { 'is_deprecated': False, - 'name': 'Minorities', - 'tag_id': '408' + 'name': '801', + 'tag_id': '801' }, { 'is_deprecated': False, - 'name': 'Unaccompanied or Separated Children', - 'tag_id': '412' + 'name': '802', + 'tag_id': '802' }, { 'is_deprecated': False, - 'name': 'Critical', - 'tag_id': '701' + 'name': '9', + 'tag_id': '9' }, { 'is_deprecated': False, - 'name': 'Female Adult (18 to 59 years old)', + 'name': '904', 'tag_id': '904' }, { 'is_deprecated': False, - 'name': 'Male Adult (18 to 59 years old)', + 'name': '905', 'tag_id': '905' }, { 'is_deprecated': False, - 'name': 'Male Older Persons (60+ years old)', + 'name': '907', 'tag_id': '907' } ] diff --git a/apps/assisted_tagging/tests/test_query.py b/apps/assisted_tagging/tests/test_query.py index 1a376a1586..119b999bfb 100644 --- a/apps/assisted_tagging/tests/test_query.py +++ b/apps/assisted_tagging/tests/test_query.py @@ -331,6 +331,18 @@ def _query_check(**kwargs): class AssistedTaggingCallbackApiTest(TestCase, SnapShotTextCase): + factories_used = [ + UserFactory, + ProjectFactory, + LeadFactory, + AssistedTaggingModelFactory, + AssistedTaggingModelPredictionTagFactory, + AssistedTaggingModelVersionFactory, + DraftEntryFactory, + AssistedTaggingPredictionFactory, + MissingPredictionReviewFactory, + WrongPredictionReviewFactory, + ] DEEPL_CALLBACK_MOCK_DATA = { "client_id": "random-client-id", @@ -1272,13 +1284,13 @@ def _get_current_model_stats(): model_version_count=AssistedTaggingModelVersion.objects.count(), tag_count=AssistedTaggingModelPredictionTag.objects.count(), models=list( - AssistedTaggingModel.objects.values('model_id', 'name') + AssistedTaggingModel.objects.values('model_id', 'name').order_by('model_id') ), model_versions=list( - AssistedTaggingModelVersion.objects.values('model__model_id', 'version') + AssistedTaggingModelVersion.objects.values('model__model_id', 'version').order_by('model__model_id') ), tags=list( - AssistedTaggingModelPredictionTag.objects.values('name', 'tag_id', 'is_deprecated') + AssistedTaggingModelPredictionTag.objects.values('name', 'tag_id', 'is_deprecated').order_by('tag_id') ), ) diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 2c8d744cb2..846a4626b2 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -259,7 +259,8 @@ def get_tags_map(): current_tags_map = get_tags_map() # Check if new tags needs to be created new_tags = [ - tag for tag in tags + tag + for tag in tags if tag not in current_tags_map ] if new_tags: