adding periodic task for update the disambiguator json

dojo/problem/helper.py

@@ -9,7 +9,7 @@
 logger = logging.getLogger(__name__)
 
 CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'config.json')
-CACHED_JSON_FILE = os.path.join('/tmp', 'cached_disambiguator.json')
+CACHED_JSON_FILE = os.path.join('/app/media', 'cached_disambiguator.json')
 
 SEVERITY_ORDER = {
     'Critical': 5,
@@ -19,9 +19,6 @@
     'Info': 1
 }
 
-# Disable SSL warnings
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
 def load_config():
     with open(CONFIG_FILE, 'r') as f:
         return json.load(f)
@@ -59,21 +56,22 @@ def save_json_to_cache(data):
 
 def load_json():
     try:
+        # Disable SSL warnings
+        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
         cached_data = load_cached_json()
-        if cached_data is not None:
+        if cached_data:
             return cached_data
 
         # Cache is missing or invalid, download and validate
         config = load_config()
         json_url = config.get('json_url')
 
-        if not json_url:
-            return {}
-
-        data = download_json(json_url)
-        if validate_json(data):
-            save_json_to_cache(data)
-            return data
+        if json_url:
+            data = download_json(json_url)
+            if validate_json(data):
+                save_json_to_cache(data)
+                return data
 
         return {}
 

dojo/problem/tasks.py

@@ -0,0 +1,37 @@
+import json
+import os
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+from dojo.celery import app
+from dojo.decorators import dojo_async_task
+from dojo.problem.helper import CONFIG_FILE, validate_json, download_json, save_json_to_cache
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+@dojo_async_task
+@app.task
+def daily_cache_update(**kwargs):
+    logger.info("Starting daily cache update")
+    try:
+        # Disable SSL warnings
+        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+        if os.path.exists(CONFIG_FILE):
+            with open(CONFIG_FILE, 'r') as f:
+                config = json.load(f)
+                json_url = config.get('json_url')
+                if json_url:
+                    data = download_json(json_url)
+                    if validate_json(data):
+                        save_json_to_cache(data)
+                    else:
+                        logger.error('Disambiguator JSON is invalid')
+                else:
+                    logger.error('No JSON URL found in config')
+        else:
+            logger.error('Config file not found')
+    except (requests.RequestException, ValueError, json.JSONDecodeError) as e:
+        logger.error('Error updating cache: %s', e)

dojo/settings/settings.dist.py

@@ -1109,7 +1109,7 @@ def saml2_attrib_map_format(dict):
 if len(env("DD_CELERY_BROKER_TRANSPORT_OPTIONS")) > 0:
     CELERY_BROKER_TRANSPORT_OPTIONS = json.loads(env("DD_CELERY_BROKER_TRANSPORT_OPTIONS"))
 
-CELERY_IMPORTS = ("dojo.tools.tool_issue_updater", )
+CELERY_IMPORTS = ("dojo.tools.tool_issue_updater", "dojo.problem.tasks")
 
 # Celery beat scheduled tasks
 CELERY_BEAT_SCHEDULE = {
@@ -1147,6 +1147,10 @@ def saml2_attrib_map_format(dict):
         "task": "dojo.notifications.helper.webhook_status_cleanup",
         "schedule": timedelta(minutes=1),
     },
+    "daily-cache-update": {
+        "task": "dojo.problem.tasks.daily_cache_update",
+        "schedule": crontab(minute=0, hour=0),  # every day at midnight
+    },
     # 'jira_status_reconciliation': {
     #     'task': 'dojo.tasks.jira_status_reconciliation_task',
     #     'schedule': timedelta(hours=12),