From 41268740ada6cf59ca2d883e234ee90fa0122058 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 23 May 2023 16:12:35 +0545 Subject: [PATCH 01/20] Feature resync --- api/urls.py | 4 ++- api/views.py | 41 ++++++++++++++++++++++++++++++ tasks/task_runners.py | 7 ++++- ui/app/components/ExportDetails.js | 23 +++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/api/urls.py b/api/urls.py index 02fe4eecc..467159430 100644 --- a/api/urls.py +++ b/api/urls.py @@ -6,7 +6,7 @@ from .views import (ConfigurationViewSet, ExportRunViewSet, HDXExportRegionViewSet, PartnerExportRegionViewSet, JobViewSet, permalink, get_overpass_timestamp, - cancel_run, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status) + cancel_run,sync_to_hdx_api, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status) router = DefaultRouter(trailing_slash=False) router.register(r'jobs', JobViewSet, base_name='jobs') @@ -36,6 +36,8 @@ url(r'^stats$', stats), url(r'^run_stats$', run_stats), url(r'^status$', machine_status), + url(r'^sync_to_hdx_api$', sync_to_hdx_api), url(r'^cancel_run$', cancel_run), + ] diff --git a/api/views.py b/api/views.py index 4fcfa6bde..0ae3f3931 100644 --- a/api/views.py +++ b/api/views.py @@ -10,6 +10,7 @@ from collections import Counter import os import io +import pickle import csv import dateutil.parser import requests @@ -21,6 +22,7 @@ from django.contrib.auth.models import Group from django.contrib.gis.geos import GEOSGeometry, Polygon from django.db.models import Q +from django.core.exceptions import ObjectDoesNotExist from django.http import ( JsonResponse, HttpResponse, @@ -772,6 +774,45 @@ def get_groups(request): return JsonResponse({"groups": groups}) +@require_http_methods(["GET"]) +def sync_to_hdx_api(request): + if not request.user.is_superuser: + return HttpResponseForbidden() + + run_uid = request.GET.get("run_uid") + if run_uid: + LOG.debug('Syncing to HDX for run: {0}'.format(run_uid)) + try: + run = ExportRun.objects.get(id=run_uid) + except ExportRun.DoesNotExist: + return JsonResponse({"error": "Invalid run UID"}, status=404) + + try: + region = HDXExportRegion.objects.get(job_id=run.job_id) + except HDXExportRegion.DoesNotExist: + return JsonResponse({"error": "HDXExportRegion not found"}, status=404) + + public_dir = settings.HOSTNAME + os.path.join(settings.EXPORT_MEDIA_ROOT, run_uid) + pickle_file_path = os.path.join(public_dir, 'all_zips.pkl') + + if os.path.exists(pickle_file_path): + try: + with open(pickle_file_path, 'rb') as file: + all_zips_data = file.read() + all_zips = pickle.loads(all_zips_data) + sync_region(region, all_zips, public_dir) + run.hdx_sync_status = True + except Exception as ex: + run.sync_status = False + LOG.error(ex) + return JsonResponse({"error": "Sync failed"}, status=500) + + run.save() + return JsonResponse({"success": "Sync to HDX completed successfully"}) + + return JsonResponse({"error": "Missing run UID"}, status=400) + + from dramatiq_abort import abort diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 125d7580f..32d7ab5bf 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -8,6 +8,7 @@ import shutil import zipfile import traceback +import pickle import django from dramatiq.middleware import TimeLimitExceeded @@ -489,8 +490,12 @@ def add_metadata(z,theme): if settings.SYNC_TO_HDX: LOG.debug('Syncing to HDX for run: {0}'.format(run_uid)) region = HDXExportRegion.objects.get(job_id=run.job_id) + all_zips_data = pickle.dumps(all_zips) + public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid) + pickle_file_path = os.path.join(public_dir,'all_zips.pkl') + with open(pickle_file_path, 'wb') as file: + file.write(all_zips_data) try: - public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid) sync_region(region,all_zips,public_dir) run.hdx_sync_status = True except Exception as ex: diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index cf4a0c4f5..59a9c0f05 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -274,6 +274,29 @@ class ExportRuns extends Component { {run.hdx_sync_status ? "Uploaded" : "Not Uploaded"} + From fa7be1c024b684c7a01b95f41cb47a98e0548367 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 23 May 2023 17:24:04 +0545 Subject: [PATCH 02/20] Only show Button if run is successful --- ui/app/components/ExportDetails.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index 59a9c0f05..5012548f2 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -273,7 +273,8 @@ class ExportRuns extends Component { /> - {run.hdx_sync_status ? "Uploaded" : "Not Uploaded"} + {run.hdx_sync_status ? "Uploaded " : "Not Uploaded "} + {run.status === "COMPLETED" && ( + )} From bd4ca532af47e0e309e7efd8539ec2e46effc773 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 23 May 2023 18:26:31 +0545 Subject: [PATCH 03/20] Added response await --- ui/app/components/ExportDetails.js | 47 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index 5012548f2..10a9b2b80 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -274,30 +274,33 @@ class ExportRuns extends Component { {run.hdx_sync_status ? "Uploaded " : "Not Uploaded "} - {run.status === "COMPLETED" && ( - - } - }}> - - )} From 07140b8cf33fe87923afd50d49af37b0649bbc87 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 23 May 2023 19:05:35 +0545 Subject: [PATCH 04/20] Fix lookup using user id --- api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index db317968f..d5d4af68c 100644 --- a/api/views.py +++ b/api/views.py @@ -784,7 +784,7 @@ def sync_to_hdx_api(request): if run_uid: LOG.debug('Syncing to HDX for run: {0}'.format(run_uid)) try: - run = ExportRun.objects.get(id=run_uid) + run = ExportRun.objects.get(uid=run_uid) except ExportRun.DoesNotExist: return JsonResponse({"error": "Invalid run UID"}, status=404) From d8fbe200ca441d0702ffc0387975e9d168c48f48 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 24 May 2023 12:43:32 +0545 Subject: [PATCH 05/20] add more debug msg --- api/views.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/api/views.py b/api/views.py index d5d4af68c..f7c2f7696 100644 --- a/api/views.py +++ b/api/views.py @@ -652,7 +652,7 @@ def request_geonames(request): if str(keyword).lower().startswith('osm'): lst=keyword.split(" ") - if len(lst)>=1: + if len(lst)>1: keyword=lst[1] try : osm_id= int(keyword) @@ -682,7 +682,7 @@ def request_geonames(request): if str(keyword).lower().startswith('tm'): lst=keyword.split(" ") - if len(lst)>=1: + if len(lst)>1: keyword=lst[1] if tm_url: @@ -801,18 +801,22 @@ def sync_to_hdx_api(request): with open(pickle_file_path, 'rb') as file: all_zips_data = file.read() all_zips = pickle.loads(all_zips_data) + LOG.debug("Calling hdx API") sync_region(region, all_zips, public_dir) run.hdx_sync_status = True except Exception as ex: run.sync_status = False LOG.error(ex) return JsonResponse({"error": "Sync failed"}, status=500) + else: + return JsonResponse({"error": "No exports available"}, status=404) run.save() - return JsonResponse({"success": "Sync to HDX completed successfully"}) - - return JsonResponse({"error": "Missing run UID"}, status=400) + LOG.debug('Sync Success to HDX for run: {0}'.format(run_uid)) + + return JsonResponse({"success": "Sync to HDX completed successfully"}, status=200) + return JsonResponse({"error": "Missing run UID"}, status=400) from dramatiq_abort import abort From a9d83ad99fdabbff9b35aa520981b4933f7738fd Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 24 May 2023 14:22:25 +0545 Subject: [PATCH 06/20] Change pickle path to stage dir --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 7e3953f29..6f2dcb89b 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -491,7 +491,7 @@ def add_metadata(z,theme): region = HDXExportRegion.objects.get(job_id=run.job_id) all_zips_data = pickle.dumps(all_zips) public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid) - pickle_file_path = os.path.join(public_dir,'all_zips.pkl') + pickle_file_path = os.path.join(download_dir,'all_zips.pkl') with open(pickle_file_path, 'wb') as file: file.write(all_zips_data) try: From c2d4acd59bd33f404d721244b1fcae688010eb16 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 24 May 2023 15:18:03 +0545 Subject: [PATCH 07/20] fix typo error --- ui/app/components/ExportDetails.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index 10a9b2b80..6445d01a4 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -225,7 +225,7 @@ class ExportRuns extends Component { - { (run.status === "SUBMITTED" || run.status == "RUNNING") ?( + { (run.status === "SUBMITTED" || run.status === "RUNNING") ?( @@ -274,7 +274,7 @@ class ExportRuns extends Component { {run.hdx_sync_status ? "Uploaded " : "Not Uploaded "} - {run.status === "COMPLETED" || run.status == "FAILED" && ( + {run.status === "COMPLETED" || run.status === "FAILED" && ( - - )} From 2dcd5d9a85d3e3ec6a77cf3fccd67633e65657f4 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 24 May 2023 15:24:38 +0545 Subject: [PATCH 09/20] add disabled button on the basis of hdx sync status --- ui/app/components/ExportDetails.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/app/components/ExportDetails.js b/ui/app/components/ExportDetails.js index eb6a02439..3b1bdbb9b 100644 --- a/ui/app/components/ExportDetails.js +++ b/ui/app/components/ExportDetails.js @@ -277,6 +277,7 @@ class ExportRuns extends Component {