From 739ce783395f67e42f5555900395ce9b8ba99695 Mon Sep 17 00:00:00 2001 From: IrfanUddinAhmad Date: Wed, 14 Feb 2024 17:37:54 +0500 Subject: [PATCH] feat: Optimized finalize_xblockskill_tags command for memory via chunking --- CHANGELOG.rst | 4 ++ taxonomy/__init__.py | 2 +- .../commands/finalize_xblockskill_tags.py | 48 +++++++++---------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f704ab83..17c8495a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,10 @@ Change Log Unreleased +[1.46.2] - 2024-02-14 +--------------------- +* feat: Optimized finalize_xblockskill_tags command for memory via chunking + [1.46.1] - 2024-01-05 --------------------- * feat: Modify prefetch related to select related for whitelisted product skills. diff --git a/taxonomy/__init__.py b/taxonomy/__init__.py index 00d6f85f..f7a49b10 100644 --- a/taxonomy/__init__.py +++ b/taxonomy/__init__.py @@ -15,6 +15,6 @@ # 2. MINOR version when you add functionality in a backwards compatible manner, and # 3. PATCH version when you make backwards compatible bug fixes. # More details can be found at https://semver.org/ -__version__ = '1.46.1' +__version__ = '1.46.2' default_app_config = 'taxonomy.apps.TaxonomyConfig' # pylint: disable=invalid-name diff --git a/taxonomy/management/commands/finalize_xblockskill_tags.py b/taxonomy/management/commands/finalize_xblockskill_tags.py index 232800eb..5aee7389 100644 --- a/taxonomy/management/commands/finalize_xblockskill_tags.py +++ b/taxonomy/management/commands/finalize_xblockskill_tags.py @@ -7,7 +7,6 @@ from django.conf import settings from django.core.management.base import BaseCommand -from django.db import transaction from django.utils.translation import gettext as _ from taxonomy.exceptions import InvalidCommandOptionsError @@ -99,28 +98,27 @@ def handle(self, *args, **options): options, ) - with transaction.atomic(): - unverified_skills = XBlockSkillData.objects.filter(verified=False, is_blacklisted=False) - for xblock_skill in unverified_skills: - verified_count = xblock_skill.verified_count if xblock_skill.verified_count else 0 - ignored_count = xblock_skill.ignored_count if xblock_skill.ignored_count else 0 - total_count = int(verified_count + ignored_count) - if total_count <= 0: - continue - if self._is_over_threshold(verified_count, total_count, min_verified_votes, ratio_verified_threshold): - xblock_skill.verified = True - xblock_skill.save() - LOGGER.info( - '[%s] skill tag for the xblock [%s] has been verified', - xblock_skill.skill.name, - xblock_skill.xblock.usage_key - ) - elif self._is_over_threshold(ignored_count, total_count, min_ignored_votes, ratio_ignored_threshold): - xblock_skill.is_blacklisted = True - xblock_skill.save() - LOGGER.info( - '[%s] skill tag for the xblock [%s] has been blacklisted', - xblock_skill.skill.name, - xblock_skill.xblock.usage_key - ) + for xblock_skill in XBlockSkillData.objects.filter( + verified=False, is_blacklisted=False).iterator(chunk_size=2000): + verified_count = xblock_skill.verified_count if xblock_skill.verified_count else 0 + ignored_count = xblock_skill.ignored_count if xblock_skill.ignored_count else 0 + total_count = int(verified_count + ignored_count) + if total_count <= 0: + continue + if self._is_over_threshold(verified_count, total_count, min_verified_votes, ratio_verified_threshold): + xblock_skill.verified = True + xblock_skill.save() + LOGGER.info( + '[%s] skill tag for the xblock [%s] has been verified', + xblock_skill.skill.name, + xblock_skill.xblock.usage_key + ) + elif self._is_over_threshold(ignored_count, total_count, min_ignored_votes, ratio_ignored_threshold): + xblock_skill.is_blacklisted = True + xblock_skill.save() + LOGGER.info( + '[%s] skill tag for the xblock [%s] has been blacklisted', + xblock_skill.skill.name, + xblock_skill.xblock.usage_key + ) LOGGER.info('Xblockskill tags verification task is completed')