fix: chunked data at 50000 byte in EMSI client for xblock-skills job

openedx · Sep 12, 2023 · 7675deb · 7675deb
1 parent 2c39d13
commit 7675deb
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 1 deletion.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -13,6 +13,10 @@ Change Log
 
 Unreleased
 
+[1.44.2] - 2023-09-11
+---------------------
+* fix: chunked data at 50000 byte in EMSI client for xblock-skills job
+
 [1.44.1] - 2023-08-25
 ---------------------
 * feat: add prefetch related to the whitelisted product skills

diff --git a/taxonomy/__init__.py b/taxonomy/__init__.py
@@ -15,6 +15,6 @@
 # 2. MINOR version when you add functionality in a backwards compatible manner, and
 # 3. PATCH version when you make backwards compatible bug fixes.
 # More details can be found at https://semver.org/
-__version__ = '1.44.1'
+__version__ = '1.44.2'
 
 default_app_config = 'taxonomy.apps.TaxonomyConfig'  # pylint: disable=invalid-name
diff --git a/taxonomy/emsi/client.py b/taxonomy/emsi/client.py
@@ -183,6 +183,7 @@ class EMSISkillsApiClient(JwtEMSIApiClient):
     """
 
     API_BASE_URL = urljoin(JwtEMSIApiClient.API_BASE_URL, '/skills/versions/8.9')
+    MAX_LIGHTCAST_DATA_SIZE = 50000  # Maximum 50,000-byte data is supported by LightCast
 
     def __init__(self):
         """
@@ -229,6 +230,11 @@ def get_product_skills(self, text_data):
         Returns:
             dict: A dictionary containing details of all the skills.
         """
+
+        if text_data and len(text_data) > self.MAX_LIGHTCAST_DATA_SIZE:
+            # Truncate the text_data to 50,000 bytes since only 50,000-byte data is supported by LightCast
+            text_data = text_data[:self.MAX_LIGHTCAST_DATA_SIZE]
+
         data = {
             'text': text_data
         }

diff --git a/tests/emsi/test_client.py b/tests/emsi/test_client.py
@@ -157,8 +157,16 @@ def test_get_product_skills(self):
         """
         Validate that the behavior of client while fetching product skills.
         """
+
+        max_data_size = self.client.MAX_LIGHTCAST_DATA_SIZE
+        # Chunk larger-sized data according to the maximum supported size
+        truncated_text_data = (SKILL_TEXT_DATA * max_data_size)[:max_data_size]
+
         skills = self.client.get_product_skills(SKILL_TEXT_DATA)
 
+        # Validate that the larger data has been chunked according to the maximum supported size
+        assert len(truncated_text_data) == max_data_size
+
         assert skills == SKILLS_EMSI_CLIENT_RESPONSE
 
     @mock_api_response(