diff --git a/api/urls.py b/api/urls.py index 02fe4eecc..467159430 100644 --- a/api/urls.py +++ b/api/urls.py @@ -6,7 +6,7 @@ from .views import (ConfigurationViewSet, ExportRunViewSet, HDXExportRegionViewSet, PartnerExportRegionViewSet, JobViewSet, permalink, get_overpass_timestamp, - cancel_run, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status) + cancel_run,sync_to_hdx_api, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status) router = DefaultRouter(trailing_slash=False) router.register(r'jobs', JobViewSet, base_name='jobs') @@ -36,6 +36,8 @@ url(r'^stats$', stats), url(r'^run_stats$', run_stats), url(r'^status$', machine_status), + url(r'^sync_to_hdx_api$', sync_to_hdx_api), url(r'^cancel_run$', cancel_run), + ] diff --git a/api/views.py b/api/views.py index 76d452360..6d5a7a32f 100644 --- a/api/views.py +++ b/api/views.py @@ -10,6 +10,7 @@ from collections import Counter import os import io +import pickle import csv import dateutil.parser import requests @@ -21,6 +22,7 @@ from django.contrib.auth.models import Group from django.contrib.gis.geos import GEOSGeometry, Polygon from django.db.models import Q +from django.core.exceptions import ObjectDoesNotExist from django.http import ( JsonResponse, HttpResponse, @@ -55,6 +57,8 @@ from hdx_exports.hdx_export_set import sync_region from rtree import index import psutil +from asgiref.sync import async_to_sync +import asyncio # Get an instance of a logger LOG = logging.getLogger(__name__) @@ -650,7 +654,7 @@ def request_geonames(request): if str(keyword).lower().startswith('osm'): lst=keyword.split(" ") - if len(lst)>=1: + if len(lst)>1: keyword=lst[1] try : osm_id= int(keyword) @@ -680,7 +684,7 @@ def request_geonames(request): if str(keyword).lower().startswith('tm'): lst=keyword.split(" ") - if len(lst)>=1: + if len(lst)>1: keyword=lst[1] if tm_url: @@ -773,6 +777,105 @@ def get_groups(request): return JsonResponse({"groups": groups}) +async def sync_to_hdx_api_async(run_uid): + try: + run = ExportRun.objects.get(uid=run_uid) + except ExportRun.DoesNotExist: + return {"error": "Invalid run UID"} + + try: + region = HDXExportRegion.objects.get(job_id=run.job_id) + except HDXExportRegion.DoesNotExist: + return JsonResponse({"error": "HDXExportRegion not found"}, status=404) + + public_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid) + LOG.debug(public_dir) + pickle_file_path = os.path.join(public_dir, 'all_zips.pkl') + + if os.path.exists(pickle_file_path): + try: + with open(pickle_file_path, 'rb') as file: + all_zips_data = file.read() + all_zips = pickle.loads(all_zips_data) + LOG.debug("Calling hdx API") + sync_region(region, all_zips, public_dir) + run.hdx_sync_status = True + except Exception as ex: + run.sync_status = False + LOG.error(ex) + return JsonResponse({"error": "Sync failed"}, status=500) + else: + return JsonResponse({"error": "No exports available"}, status=404) + + run.save() + LOG.debug('Sync Success to HDX for run: {0}'.format(run_uid)) + + + return {"success": "Sync to HDX completed successfully"} + + +@require_http_methods(["GET"]) +@async_to_sync +async def sync_to_hdx_api(request): + if not request.user.is_superuser: + return JsonResponse({"error": "Permission denied"}, status=403) + + run_uid = request.GET.get("run_uid") + if run_uid: + # loop = asyncio.get_event_loop() + # loop.create_task(sync_to_hdx_api_async(run_uid)) + await sync_to_hdx_api_async(run_uid) + # asyncio.run(sync_to_hdx_api_async(run_uid)) + return JsonResponse({"message": "Sync request received and is being processed in the background."}, status=202) + + return JsonResponse({"error": "Missing run UID"}, status=400) + +# @require_http_methods(["GET"]) +# def sync_to_hdx_api(request): +# if not request.user.is_superuser: +# return HttpResponseForbidden() + +# run_uid = request.GET.get("run_uid") +# if run_uid: +# LOG.debug('Syncing to HDX for run: {0}'.format(run_uid)) +# try: +# run = ExportRun.objects.get(uid=run_uid) +# except ExportRun.DoesNotExist: +# return JsonResponse({"error": "Invalid run UID"}, status=404) + +# try: +# region = HDXExportRegion.objects.get(job_id=run.job_id) +# except HDXExportRegion.DoesNotExist: +# return JsonResponse({"error": "HDXExportRegion not found"}, status=404) + +# public_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid) +# LOG.debug(public_dir) +# pickle_file_path = os.path.join(public_dir, 'all_zips.pkl') + +# if os.path.exists(pickle_file_path): +# try: +# with open(pickle_file_path, 'rb') as file: +# all_zips_data = file.read() +# all_zips = pickle.loads(all_zips_data) +# LOG.debug("Calling hdx API") +# sync_region(region, all_zips, public_dir) +# run.hdx_sync_status = True +# except Exception as ex: +# run.sync_status = False +# LOG.error(ex) +# return JsonResponse({"error": "Sync failed"}, status=500) +# else: +# return JsonResponse({"error": "No exports available"}, status=404) + +# run.save() +# LOG.debug('Sync Success to HDX for run: {0}'.format(run_uid)) + +# return JsonResponse({"success": "Sync to HDX completed successfully"}, status=200) + +# return JsonResponse({"error": "Missing run UID"}, status=400) + + + from dramatiq_abort import abort diff --git a/hdx_exports/hdx_export_set.py b/hdx_exports/hdx_export_set.py index cd485330d..4b873aeeb 100644 --- a/hdx_exports/hdx_export_set.py +++ b/hdx_exports/hdx_export_set.py @@ -3,6 +3,7 @@ import django.utils.text from hdx.data.dataset import Dataset from osm_export_tool.mapping import Mapping +import logging FILTER_CRITERIA = """ This theme includes all OpenStreetMap features in this area matching: @@ -27,16 +28,21 @@ def slugify(str): s = django.utils.text.slugify(str) return s.replace('-','_') -def sync_datasets(datasets,update_dataset_date=False): +def sync_datasets(datasets, update_dataset_date=False): for dataset in datasets: - exists = Dataset.read_from_hdx(dataset['name']) - if exists: - if update_dataset_date: + try: + exists = Dataset.read_from_hdx(dataset['name']) + if exists: + if update_dataset_date: + dataset.set_date_of_dataset(datetime.now()) + dataset.update_in_hdx() + else: dataset.set_date_of_dataset(datetime.now()) - dataset.update_in_hdx() - else: - dataset.set_date_of_dataset(datetime.now()) - dataset.create_in_hdx(allow_no_resources=True) + dataset.create_in_hdx(allow_no_resources=True) + except Exception as e: + # Log the error and continue with the next dataset + logging.error(f"Error syncing dataset {dataset['name']}: {str(e)}") + def sync_region(region,files=[],public_dir=''): export_set = HDXExportSet( diff --git a/tasks/task_runners.py b/tasks/task_runners.py index f37629f4e..6f2dcb89b 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -8,6 +8,7 @@ import shutil import zipfile import traceback +import pickle import django from dramatiq.middleware import TimeLimitExceeded @@ -488,8 +489,12 @@ def add_metadata(z,theme): if settings.SYNC_TO_HDX: LOG.debug('Syncing to HDX for run: {0}'.format(run_uid)) region = HDXExportRegion.objects.get(job_id=run.job_id) + all_zips_data = pickle.dumps(all_zips) + public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid) + pickle_file_path = os.path.join(download_dir,'all_zips.pkl') + with open(pickle_file_path, 'wb') as file: + file.write(all_zips_data) try: - public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid) sync_region(region,all_zips,public_dir) run.hdx_sync_status = True except Exception as ex: diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index cf4a0c4f5..3b1bdbb9b 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -225,7 +225,7 @@ class ExportRuns extends Component { - { (run.status === "SUBMITTED" || run.status == "RUNNING") ?( + { (run.status === "SUBMITTED" || run.status === "RUNNING") ?( @@ -273,7 +273,34 @@ class ExportRuns extends Component { /> - {run.hdx_sync_status ? "Uploaded" : "Not Uploaded"} + {run.hdx_sync_status ? "Uploaded " : "Not Uploaded "} + +