From 3361d1c79d1956515070fd6f90c67e8121e03d96 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 22 Jan 2024 18:30:42 -0600 Subject: [PATCH 01/21] commenting out espa wrapper orders. orders now done through eros wrapper (m2m api) --- theia/adapters/usgs/adapter.py | 16 +++- theia/adapters/usgs/eros_wrapper.py | 115 +++++++++++++++++++++++++--- 2 files changed, 116 insertions(+), 15 deletions(-) diff --git a/theia/adapters/usgs/adapter.py b/theia/adapters/usgs/adapter.py index 9abdd2a..3850850 100644 --- a/theia/adapters/usgs/adapter.py +++ b/theia/adapters/usgs/adapter.py @@ -69,15 +69,23 @@ def __init__(self): def process_request(self, imagery_request): search = ImagerySearch.build_search(imagery_request) + print('MDY114 BEFORE EROS WRAPPER') scenes = ErosWrapper().search(search) + print('MDY114 AFTER EROS WRAPPER SCENES') + print(scenes) if imagery_request.max_results: scenes = scenes[0:imagery_request.max_results] + print('MDY114 SCENES') + print(scenes) for scene in scenes: - result = EspaWrapper.order_all(scene, 'sr') - for item in result: - req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) - wait_for_scene.delay(req.id) + print('MDY114 BEFORE ESPA') + # result = EspaWrapper.order_all(scene, 'sr') + # print('MDY114 AFTER ESPA') + # print(result) + # for item in result: + # req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) + # wait_for_scene.delay(req.id) def construct_filename(self, bundle, suffix): product = "sr" diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index a8cf7d6..25e8e90 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -1,37 +1,130 @@ from .utils import Utils +from urllib.parse import urljoin import json import requests import sys +import datetime +import time from sentry_sdk import capture_message +EROS_SERVICE_URL = "https://m2m.cr.usgs.gov/api/api/json/stable/" +MAX_THREADS = 5 +TOKEN_EXPIRY_HOURS = 2 class ErosWrapper(): + def __init__(self): + self.api_key = None + self.login_time = None - def search(self, search): - serviceUrl = "https://m2m.cr.usgs.gov/api/api/json/stable/" + def login(self): + if self.login_time is None or self.is_token_expired(): + loginParameters = { + 'username': Utils.get_username(), + 'password': Utils.get_password() + } + self.apiKey = self.send_request(EROS_SERVICE_URL + "login", loginParameters) + self.login_time = datetime.datetime.utcnow() - loginParameters = { - 'username': Utils.get_username(), - 'password': Utils.get_password() - } - apiKey = self.send_request(serviceUrl + "login", loginParameters) - scenes = self.send_request(serviceUrl + "scene-search", search, apiKey) + + def is_token_expired(self): + if self.login_time: + token_expiration_time = self.login_time + datetime.timedelta(hours=TOKEN_EXPIRY_HOURS) + # minusing 1 minute to expiry check to protect against race condition + return datetime.datetime.utcnow() - datetime.timedelta(minutes=1) >= token_expiration_time + return True + + def search(self, search): + self.login() + scenes = self.send_request(EROS_SERVICE_URL + "scene-search", search) results = self.parse_result_set(scenes['results']) return results + def add_scenes_to_order_list(self, scene_id, search): + self.login() + scene_list_add_payload = { + "listId": scene_id, + "idField": "displayId", #default is search by entityId + "entityId": scene_id, + "datasetName": search['datasetName'] + } + return self.send_request(EROS_SERVICE_URL + "scene-list-add", scene_list_add_payload, apiKey) + + def available_products(self, list_id, search): + self.login() + + download_options_payload = { + "listId": list_id, + "datasetName": search['datasetName'] + } + results = self.send_request(EROS_SERVICE_URL + "download-options", download_options_payload, apiKey) + products = [] + + for result in results: + if result["bulkAvailable"] and result['downloadSystem'] != 'folder': + products.append({"entityId": result['entityId'], "productId": result['id']}) + return products + + def download_request(self, products): + self.login() + + download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + download_request_payload = { + "downloads": products, + "label": download_request_label + } + + product_urls = [] + + results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload, apiKey) + + for result in results['availableDownloads']: + product_urls.append(result['url']) + + preparingDownloadCount = len(results['preparingDownloads']) + preparingDownloadIds = [] + if preparingDownloadCount > 0: + for result in results['preparingDownloads']: + preparingDownloadIds.append(result['downloadId']) + + payload = {"label": download_request_label} + results = self.send_request("download-retrieve", payload, apiKey) + if results != False: + for result in results['available']: + if result['downloadId'] in preparingDownloadIds: + preparingDownloadIds.remove(result['downloadId']) + product_urls.append(result['url']) + + for result in results['requested']: + if result['downloadId'] in preparingDownloadIds: + preparingDownloadIds.remove(result['downloadId']) + product_urls.append(result['url']) + # Don't get all download urls, retrieve again after 30 seconds + while len(preparingDownloadIds) > 0: + print(f"{len(preparingDownloadIds)} downloads are not available yet. Waiting for 30s to retrieve again\n") + time.sleep(30) + results = self.send_request("download-retrieve", payload, apiKey) + if results != False: + for result in results['available']: + if result['downloadId'] in preparingDownloadIds: + preparingDownloadIds.remove(result['downloadId']) + product_urls.append(result['url']) + + return product_urls + def parse_result_set(self, result_set): if not result_set: return [] return [scene.get('displayId', None) for scene in result_set if 'displayId' in scene] - def send_request(self, url, data, apiKey=None): + def send_request(self, url, data): json_data = json.dumps(data) - if apiKey == None: + if self.apiKey == None: response = requests.post(url, json_data) else: - headers = {'X-Auth-Token': apiKey} + headers = {'X-Auth-Token': self.apiKey} response = requests.post(url, json_data, headers=headers) try: From e7576b221338eb1e40ebde5a9accda08b13426c2 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Tue, 23 Jan 2024 13:50:50 -0600 Subject: [PATCH 02/21] remove unneeded apiKey parameter --- theia/adapters/usgs/eros_wrapper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index 25e8e90..20ebea4 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -47,7 +47,7 @@ def add_scenes_to_order_list(self, scene_id, search): "entityId": scene_id, "datasetName": search['datasetName'] } - return self.send_request(EROS_SERVICE_URL + "scene-list-add", scene_list_add_payload, apiKey) + return self.send_request(EROS_SERVICE_URL + "scene-list-add", scene_list_add_payload) def available_products(self, list_id, search): self.login() @@ -56,7 +56,7 @@ def available_products(self, list_id, search): "listId": list_id, "datasetName": search['datasetName'] } - results = self.send_request(EROS_SERVICE_URL + "download-options", download_options_payload, apiKey) + results = self.send_request(EROS_SERVICE_URL + "download-options", download_options_payload) products = [] for result in results: @@ -76,7 +76,7 @@ def download_request(self, products): product_urls = [] - results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload, apiKey) + results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) for result in results['availableDownloads']: product_urls.append(result['url']) @@ -88,7 +88,7 @@ def download_request(self, products): preparingDownloadIds.append(result['downloadId']) payload = {"label": download_request_label} - results = self.send_request("download-retrieve", payload, apiKey) + results = self.send_request("download-retrieve", payload) if results != False: for result in results['available']: if result['downloadId'] in preparingDownloadIds: @@ -103,7 +103,7 @@ def download_request(self, products): while len(preparingDownloadIds) > 0: print(f"{len(preparingDownloadIds)} downloads are not available yet. Waiting for 30s to retrieve again\n") time.sleep(30) - results = self.send_request("download-retrieve", payload, apiKey) + results = self.send_request("download-retrieve", payload) if results != False: for result in results['available']: if result['downloadId'] in preparingDownloadIds: From c5d2312b081810ee0c062d18a0a9638261467c3b Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Tue, 6 Feb 2024 10:33:57 -0600 Subject: [PATCH 03/21] attempt to replace espa api calls with corresponding eros calls --- theia/adapters/usgs/adapter.py | 31 +++++++--- theia/adapters/usgs/eros_wrapper.py | 89 +++++++++++++++++++++++------ theia/adapters/usgs/espa_wrapper.py | 19 ++++++ theia/adapters/usgs/tasks.py | 20 +++++-- theia/settings.py | 1 - 5 files changed, 129 insertions(+), 31 deletions(-) diff --git a/theia/adapters/usgs/adapter.py b/theia/adapters/usgs/adapter.py index 3850850..3e23bda 100644 --- a/theia/adapters/usgs/adapter.py +++ b/theia/adapters/usgs/adapter.py @@ -69,8 +69,9 @@ def __init__(self): def process_request(self, imagery_request): search = ImagerySearch.build_search(imagery_request) + eros_wrapper = ErosWrapper() print('MDY114 BEFORE EROS WRAPPER') - scenes = ErosWrapper().search(search) + scenes = eros_wrapper.search(search) print('MDY114 AFTER EROS WRAPPER SCENES') print(scenes) if imagery_request.max_results: @@ -79,13 +80,29 @@ def process_request(self, imagery_request): print(scenes) for scene in scenes: - print('MDY114 BEFORE ESPA') + print('MDY114 BEFORE EROS ADD SCENES TO ORDER LIST') + eros_wrapper.add_scenes_to_order_list(scene, search) + print('MDY114 AFTER ADDING TO LIST') + print('MDY114 AVAILABLE PRODUCTS BEFORE') + available_products = eros_wrapper.available_products(scene, search) + print('MDY114 AVAIALBLE PRODUCTS AFTER') + # result = EspaWrapper.order_all(scene, 'sr') - # print('MDY114 AFTER ESPA') - # print(result) - # for item in result: - # req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) - # wait_for_scene.delay(req.id) + print(available_products) + result = eros_wrapper.request_download(available_products) + print(result) + print('MDY114 AFTER ESPA') + for item in available_products: + print('MDY114') + print(item) + # req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) + req = models.RequestedScene.objects.create( + scene_entity_id = item['entityId'], + scene_order_id = item['productId'], + **{'imagery_request': imagery_request} + ) + print(req) + wait_for_scene.delay(req.id, available_products) def construct_filename(self, bundle, suffix): product = "sr" diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index 20ebea4..b30dec6 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -5,6 +5,7 @@ import sys import datetime import time +import threading from sentry_sdk import capture_message EROS_SERVICE_URL = "https://m2m.cr.usgs.gov/api/api/json/stable/" MAX_THREADS = 5 @@ -14,6 +15,10 @@ class ErosWrapper(): def __init__(self): self.api_key = None self.login_time = None + self.sema = threading.Semaphore(value=MAX_THREADS) + self.threads = [] + + def login(self): if self.login_time is None or self.is_token_expired(): @@ -21,7 +26,7 @@ def login(self): 'username': Utils.get_username(), 'password': Utils.get_password() } - self.apiKey = self.send_request(EROS_SERVICE_URL + "login", loginParameters) + self.api_key = self.send_request(EROS_SERVICE_URL + "login", loginParameters) self.login_time = datetime.datetime.utcnow() @@ -61,10 +66,45 @@ def available_products(self, list_id, search): for result in results: if result["bulkAvailable"] and result['downloadSystem'] != 'folder': - products.append({"entityId": result['entityId'], "productId": result['id']}) + products.append( + { + "entityId": result['entityId'], + "productId": result['id'] + }) return products - def download_request(self, products): + def request_download(self, products): + self.login() + + download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + download_request_payload = { + "downloads": products, + "label": download_request_label + } + + #returns format like + #{ + # "requestId": 1591674034, + # "version": "stable", + # "data": { + # "availableDownloads": [], + # "duplicateProducts": [], + # "preparingDownloads": [], + # "failed": [], + # "newRecords": [], + # "numInvalidScenes": 0 + # }, + # "errorCode": null, + # "errorMessage": null + # } + + # {'availableDownloads': [], 'duplicateProducts': [], 'preparingDownloads': [{'downloadId': 550754832, 'eulaCode': None, 'url': 'https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=='}, {'downloadId': 550752861, 'eulaCode': None, 'url': 'https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=='}], 'failed': [], 'newRecords': {'550754832': '20240131_143747', '550752861': '20240131_143747'}, 'numInvalidScenes': 0} + results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) + return results + + + def download_urls(self, products): self.login() download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -76,38 +116,53 @@ def download_request(self, products): product_urls = [] + #returns format like + #{ + # "requestId": 1591674034, + # "version": "stable", + # "data": { + # "availableDownloads": [], + # "duplicateProducts": [], + # "preparingDownloads": [], + # "failed": [], + # "newRecords": [], + # "numInvalidScenes": 0 + # }, + # "errorCode": null, + # "errorMessage": null + # } results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) for result in results['availableDownloads']: product_urls.append(result['url']) - preparingDownloadCount = len(results['preparingDownloads']) - preparingDownloadIds = [] - if preparingDownloadCount > 0: + preparing_download_count = len(results['preparingDownloads']) + preparing_download_ids = [] + if preparing_download_count > 0: for result in results['preparingDownloads']: - preparingDownloadIds.append(result['downloadId']) + preparing_download_ids.append(result['downloadId']) payload = {"label": download_request_label} results = self.send_request("download-retrieve", payload) if results != False: for result in results['available']: - if result['downloadId'] in preparingDownloadIds: - preparingDownloadIds.remove(result['downloadId']) + if result['downloadId'] in preparing_download_ids: + preparing_download_ids.remove(result['downloadId']) product_urls.append(result['url']) for result in results['requested']: - if result['downloadId'] in preparingDownloadIds: - preparingDownloadIds.remove(result['downloadId']) + if result['downloadId'] in preparing_download_ids: + preparing_download_ids.remove(result['downloadId']) product_urls.append(result['url']) # Don't get all download urls, retrieve again after 30 seconds - while len(preparingDownloadIds) > 0: - print(f"{len(preparingDownloadIds)} downloads are not available yet. Waiting for 30s to retrieve again\n") + while len(preparing_download_ids) > 0: + print(f"{len(preparing_download_ids)} downloads are not available yet. Waiting for 30s to retrieve again\n") time.sleep(30) results = self.send_request("download-retrieve", payload) if results != False: for result in results['available']: - if result['downloadId'] in preparingDownloadIds: - preparingDownloadIds.remove(result['downloadId']) + if result['downloadId'] in preparing_download_ids: + preparing_download_ids.remove(result['downloadId']) product_urls.append(result['url']) return product_urls @@ -121,10 +176,10 @@ def parse_result_set(self, result_set): def send_request(self, url, data): json_data = json.dumps(data) - if self.apiKey == None: + if self.api_key == None: response = requests.post(url, json_data) else: - headers = {'X-Auth-Token': self.apiKey} + headers = {'X-Auth-Token': self.api_key} response = requests.post(url, json_data, headers=headers) try: diff --git a/theia/adapters/usgs/espa_wrapper.py b/theia/adapters/usgs/espa_wrapper.py index 1ddc453..9fb6c1d 100644 --- a/theia/adapters/usgs/espa_wrapper.py +++ b/theia/adapters/usgs/espa_wrapper.py @@ -18,12 +18,18 @@ def list_orders(cls): @classmethod def locate_collections(cls, scene_id, desired_product_id): results = cls.espa_get('available-products', scene_id) + print('MDY114 ESPA AFTER GETTING AVAILABLE PRODUCTS') + print(results) return [collection for collection in results if cls._product_is_available(desired_product_id, results[collection])] @classmethod def _product_is_available(cls, product_id, result_dict): + print('MDY114 HITS PRODUCT IS AVAILABLE') + print(product_id) + print(result_dict) + print(isinstance(result_dict, dict) and result_dict['products'] and product_id in result_dict['products']) return \ isinstance(result_dict, dict) and \ result_dict['products'] and \ @@ -35,6 +41,10 @@ def order_status(cls, order_id): @classmethod def order_one(cls, collection, scene_id, product_type): + print('MDY114 ORDERING ONE IN ESPA') + print('SCENE ID') + print(scene_id) + print(product_type) res = cls.espa_post('order', { collection: { 'inputs': [scene_id], @@ -42,10 +52,16 @@ def order_one(cls, collection, scene_id, product_type): }, 'format': 'gtiff' }) + print('MDY114 AFTER POSTING ORDER') + print(res) return res['orderid'] @classmethod def order_all(cls, scene_id, product_type): + print('MDY114 SCENE ID') + print(scene_id) + print('MDY114 PRODUCT TYPE') + print(product_type) return [ { 'scene_entity_id': scene_id, @@ -77,10 +93,13 @@ def espa_get(cls, url, request_data, **kwargs): @classmethod def espa_post(cls, url, request_data, **kwargs): + print('MDY114 HITS ESPA POST') new_args = cls.espa_prepare(request_data, **kwargs) new_url = cls.api_url(url) if request_data: new_args = {**new_args, **{'json': request_data}} + print(new_url) + print(new_args) return requests.post(new_url, **new_args).json() @classmethod diff --git a/theia/adapters/usgs/tasks.py b/theia/adapters/usgs/tasks.py index 8d3c460..3f84bf5 100644 --- a/theia/adapters/usgs/tasks.py +++ b/theia/adapters/usgs/tasks.py @@ -1,27 +1,35 @@ from celery import shared_task from theia.api import models -from .espa_wrapper import EspaWrapper +# from .espa_wrapper import EspaWrapper +from .eros_wrapper import ErosWrapper from datetime import datetime, timedelta from django.utils.timezone import make_aware @shared_task(name='theia.adapters.usgs.tasks.wait_for_scene') -def wait_for_scene(requested_scene_id): +def wait_for_scene(requested_scene_id, available_products): request = models.RequestedScene.objects.get(pk=requested_scene_id) if request.status == 1: return - status = EspaWrapper.order_status(request.scene_order_id) + # TO DO CHANGES THIS TO REQUEST FROM REQUEST DOWNLOAD + #status = EspaWrapper.order_status(request.scene_order_id) + eros_wrapper = ErosWrapper() + rerequest_download_result = eros_wrapper.request_download(available_products) now = make_aware(datetime.utcnow()) request.checked_at = now request.save() - if status == 'complete': + if len(rerequest_download_result['preparingDownloads']) > 0: request.status = 1 - request.scene_url = EspaWrapper.download_urls(request.scene_order_id)[0] + if len(rerequest_download_result['availableDownloads']) > 1: + # find the download that matches the orderId. + for item in rerequest_download_result['availableDownloads']: + if item['downloadId'] == request.scene_order_id: + request.scene_url = item['url'] request.save() models.JobBundle.objects.from_requested_scene(request) else: soon = now + timedelta(minutes=5) - wait_for_scene.apply_async((requested_scene_id,), eta=soon) + wait_for_scene.apply_async((requested_scene_id,available_products), eta=soon) diff --git a/theia/settings.py b/theia/settings.py index e824c25..2169673 100644 --- a/theia/settings.py +++ b/theia/settings.py @@ -44,7 +44,6 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.getenv('DJANGO_ENV').lower() != "production" -ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '127.0.0.1').split(',') SOCIAL_AUTH_JSONFIELD_ENABLED = True SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'email'] From 26f2955263ae2588afca8ebf82e684bd239c1320 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Wed, 7 Feb 2024 21:05:17 -0600 Subject: [PATCH 04/21] update metadata parsing --- theia/adapters/usgs/adapter.py | 7 +- theia/adapters/usgs/eros_wrapper.py | 41 ++------ theia/adapters/usgs/tasks.py | 16 ++- .../floating_forest.py | 97 ++++++++++++------- 4 files changed, 84 insertions(+), 77 deletions(-) diff --git a/theia/adapters/usgs/adapter.py b/theia/adapters/usgs/adapter.py index 3e23bda..50b3b06 100644 --- a/theia/adapters/usgs/adapter.py +++ b/theia/adapters/usgs/adapter.py @@ -91,13 +91,14 @@ def process_request(self, imagery_request): print(available_products) result = eros_wrapper.request_download(available_products) print(result) + print('MDY114 AFTER ESPA') for item in available_products: print('MDY114') print(item) # req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) req = models.RequestedScene.objects.create( - scene_entity_id = item['entityId'], + scene_entity_id = item['displayId'], scene_order_id = item['productId'], **{'imagery_request': imagery_request} ) @@ -128,8 +129,10 @@ def retrieve(self, job_bundle): # get the compressed scene data if we don't have it if not os.path.isfile(zip_path): + print('MDY114 HITS HERE?') urllib.request.urlretrieve(job_bundle.requested_scene.scene_url, zip_path) - + print('MDY114 ZIP PATH') + print(zip_path) FileUtils.untar(zip_path, job_bundle.local_path) def default_extension(self): diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index b30dec6..c61aba8 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -17,6 +17,7 @@ def __init__(self): self.login_time = None self.sema = threading.Semaphore(value=MAX_THREADS) self.threads = [] + self.download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -69,18 +70,17 @@ def available_products(self, list_id, search): products.append( { "entityId": result['entityId'], - "productId": result['id'] + "productId": result['id'], + "displayId": result['displayId'] }) return products def request_download(self, products): self.login() - download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - download_request_payload = { "downloads": products, - "label": download_request_label + "label": self.download_request_label } #returns format like @@ -100,39 +100,14 @@ def request_download(self, products): # } # {'availableDownloads': [], 'duplicateProducts': [], 'preparingDownloads': [{'downloadId': 550754832, 'eulaCode': None, 'url': 'https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=='}, {'downloadId': 550752861, 'eulaCode': None, 'url': 'https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=='}], 'failed': [], 'newRecords': {'550754832': '20240131_143747', '550752861': '20240131_143747'}, 'numInvalidScenes': 0} - results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) - return results + return self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) - def download_urls(self, products): - self.login() - - download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - - download_request_payload = { - "downloads": products, - "label": download_request_label - } + def download_urls(self): + self.login() product_urls = [] - #returns format like - #{ - # "requestId": 1591674034, - # "version": "stable", - # "data": { - # "availableDownloads": [], - # "duplicateProducts": [], - # "preparingDownloads": [], - # "failed": [], - # "newRecords": [], - # "numInvalidScenes": 0 - # }, - # "errorCode": null, - # "errorMessage": null - # } - results = self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) - for result in results['availableDownloads']: product_urls.append(result['url']) @@ -142,7 +117,7 @@ def download_urls(self, products): for result in results['preparingDownloads']: preparing_download_ids.append(result['downloadId']) - payload = {"label": download_request_label} + payload = {"label": self.download_request_label} results = self.send_request("download-retrieve", payload) if results != False: for result in results['available']: diff --git a/theia/adapters/usgs/tasks.py b/theia/adapters/usgs/tasks.py index 3f84bf5..b27e2bb 100644 --- a/theia/adapters/usgs/tasks.py +++ b/theia/adapters/usgs/tasks.py @@ -9,6 +9,8 @@ @shared_task(name='theia.adapters.usgs.tasks.wait_for_scene') def wait_for_scene(requested_scene_id, available_products): + print('MDY114 AVAILABLE PRODUCTS IN WAIT') + print(available_products) request = models.RequestedScene.objects.get(pk=requested_scene_id) if request.status == 1: return @@ -17,17 +19,21 @@ def wait_for_scene(requested_scene_id, available_products): #status = EspaWrapper.order_status(request.scene_order_id) eros_wrapper = ErosWrapper() rerequest_download_result = eros_wrapper.request_download(available_products) + print('MDY114 REREQUEST DOWNLOAD') + print(rerequest_download_result) now = make_aware(datetime.utcnow()) request.checked_at = now request.save() - if len(rerequest_download_result['preparingDownloads']) > 0: + if rerequest_download_result['preparingDownloads'] != None and len(rerequest_download_result['preparingDownloads']) == 0: request.status = 1 - if len(rerequest_download_result['availableDownloads']) > 1: - # find the download that matches the orderId. + print('MDY114 BEFORE AVAIL DOWNLOADS CHECK') + print(rerequest_download_result['availableDownloads']) + print(len(rerequest_download_result['availableDownloads'])) + if len(rerequest_download_result['availableDownloads']) > 0: for item in rerequest_download_result['availableDownloads']: - if item['downloadId'] == request.scene_order_id: - request.scene_url = item['url'] + print(item) + request.scene_url = item['url'] request.save() models.JobBundle.objects.from_requested_scene(request) else: diff --git a/theia/operations/floating_forest_operations/floating_forest.py b/theia/operations/floating_forest_operations/floating_forest.py index cd90375..83624cb 100644 --- a/theia/operations/floating_forest_operations/floating_forest.py +++ b/theia/operations/floating_forest_operations/floating_forest.py @@ -58,8 +58,8 @@ def apply(self, filenames): def parse_options(filenames): ff_config.SCENE_NAME = path.dirname(filenames[0]) - ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_pixel_qa.tif") - ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + ".xml") + ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_qa_pixel.tif") + ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_mtl.xml") ff_config.INPUT_FILE = ff_config.NEW_MASK #===========SIMPLE====================== @@ -125,8 +125,8 @@ def parse_options(filenames): ff_config.SCENE_DIR = path.dirname(filenames[0]) path_components = ff_config.SCENE_DIR.split('/') ff_config.SCENE_NAME = path_components[len(path_components) - 1] - ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_pixel_qa.tif") - ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + ".xml") + ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_qa_pixel.tif") + ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_mtl.xml") ff_config.INPUT_FILE = ff_config.NEW_MASK def generate_mask_tiles(): @@ -443,8 +443,10 @@ def maybe_clean_scratch(config): #=========XML OPERATIONS============================= def get_field_text(tree, path): - nsmap = {"espa": tree.getroot().nsmap[None]} - node = tree.xpath(path, namespaces=nsmap) + # nsmap = {"espa": tree.getroot().nsmap[None]} + # node = tree.xpath(path, namespaces=nsmap) + # node = tree.xpath(path, namespaces=nsmap) + node = tree.xpath(path) if len(node) > 0: return node[0].text return '' @@ -456,41 +458,62 @@ def parse_metadata(scene, xml_filename): result = {'!scene': scene} tree = etree.parse(xml_filename) - nsmap = {"espa": tree.getroot().nsmap[None]} - - result['acquired_date'] = get_field_text(tree, "espa:global_metadata/espa:acquisition_date") - result['acquired_time'] = get_field_text(tree, "espa:global_metadata/espa:scene_center_time") - result['sensor_id'] = get_field_text(tree, "espa:global_metadata/espa:instrument") - result['spacecraft'] = get_field_text(tree, 'espa:global_metadata/espa:satellite') - + print('MDY114 XML NSMAP') + # print(tree.getroot().nsmap) + # nsmap = {"espa": tree.getroot().nsmap[None]} + + # result['acquired_date'] = get_field_text(tree, "espa:global_metadata/espa:acquisition_date") + result['acquired_date'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/DATE_ACQUIRED") + # result['acquired_time'] = get_field_text(tree, "espa:global_metadata/espa:scene_center_time") + result['acquired_time'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SPACE_CENTER_TIME") + # result['sensor_id'] = get_field_text(tree, "espa:global_metadata/espa:instrument") + result['sensor_id'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SENSOR_ID") + # result['spacecraft'] = get_field_text(tree, 'espa:global_metadata/espa:satellite') + result['spacecraft'] = get_field_text(tree, '//IMAGE_ATTRIBUTES/SPACECRAFT_ID') + + # result['!earth_sun_distance'] = get_field_text( + # tree, + # "espa:global_metadata/espa:earth_sun_distance") result['!earth_sun_distance'] = get_field_text( tree, - "espa:global_metadata/espa:earth_sun_distance") - - angles = tree.xpath("espa:global_metadata/espa:solar_angles", namespaces=nsmap) - if len(angles) > 0: - result['!sun_azimuth'] = angles[0].get("azimuth") - result['!sun_zenith'] = angles[0].get("zenith") - - covers = tree.xpath( - "espa:bands/espa:band[@name='cfmask']/espa:percent_coverage/espa:cover", - namespaces=nsmap) - for cover in covers: - if cover.get("type") == "cloud": - result['!cloud_cover'] = cover.text - if cover.get("type") == "water": - result['!water_cover'] = cover.text - + "//IMAGE_ATTRIBUTES/EARTH_SUN_DISTANCE") + + # angles = tree.xpath("espa:global_metadata/espa:solar_angles", namespaces=nsmap) + # angles = tree.xpath("espa:global_metadata/espa:solar_angles") + # if len(angles) > 0: + # result['!sun_azimuth'] = angles[0].get("azimuth") + # result['!sun_zenith'] = angles[0].get("zenith") + result['!sun_azimuth'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SUN_AZIMUTH") + result['!sun_zenith'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SUN_ELEVATION") + + # covers = tree.xpath( + # "espa:bands/espa:band[@name='cfmask']/espa:percent_coverage/espa:cover", + # namespaces=nsmap) + # for cover in covers: + # if cover.get("type") == "cloud": + # result['!cloud_cover'] = cover.text + # if cover.get("type") == "water": + # result['!water_cover'] = cover.text + result['!cloud_cover'] = get_field_text("//IMAGE_ATTRIBUTES/CLOUD_COVER") + + # result['#utm_zone'] = get_field_text( + # tree, + # "espa:global_metadata/espa:projection_information/espa:utm_proj_params/espa:zone_code") result['#utm_zone'] = get_field_text( tree, - "espa:global_metadata/espa:projection_information/espa:utm_proj_params/espa:zone_code") - - corners = tree.xpath( - "espa:global_metadata/espa:projection_information/espa:corner_point", - namespaces=nsmap) - for corner in corners: - result["#scene_corner_{0}_x".format(corner.get("location"))] = corner.get("x") - result["#scene_corner_{0}_y".format(corner.get("location"))] = corner.get("y") + "//LEVEL1_PROJECTION_PARAMETERS/UTM_ZONE") + + # TODO ASK CLIFF ABOUT THIS ON CORNERS AND THERE IS NO WATER COVER CODE IN NEW XML + # corners = tree.xpath( + # "espa:global_metadata/espa:projection_information/espa:corner_point", + # namespaces=nsmap) + # for corner in corners: + # result["#scene_corner_{0}_x".format(corner.get("location"))] = corner.get("x") + # result["#scene_corner_{0}_y".format(corner.get("location"))] = corner.get("y") + result["#scene_corner_UL_x"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_UL_PROJECTION_X_PRODUCT') + result["#scene_corner_UL_y"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_UL_PROJECTION_Y_PRODUCT') + result["#scene_corner_LR_x"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_LR_PROJECTION_X_PRODUCT') + result["#scene_corner_LR_y"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_LR_PROJECTION_Y_PRODUCT') return result From 4a553c661a26b7cb8375c02ff85be79296ba0385 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Wed, 7 Feb 2024 21:09:47 -0600 Subject: [PATCH 05/21] update typo missing arg on cloud_cover --- theia/operations/floating_forest_operations/floating_forest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/theia/operations/floating_forest_operations/floating_forest.py b/theia/operations/floating_forest_operations/floating_forest.py index 83624cb..d4c7d0a 100644 --- a/theia/operations/floating_forest_operations/floating_forest.py +++ b/theia/operations/floating_forest_operations/floating_forest.py @@ -494,7 +494,7 @@ def parse_metadata(scene, xml_filename): # result['!cloud_cover'] = cover.text # if cover.get("type") == "water": # result['!water_cover'] = cover.text - result['!cloud_cover'] = get_field_text("//IMAGE_ATTRIBUTES/CLOUD_COVER") + result['!cloud_cover'] = get_field_text(tree,"//IMAGE_ATTRIBUTES/CLOUD_COVER") # result['#utm_zone'] = get_field_text( # tree, From 9040369f15d41ddfc55b192b21f3ab3652ecaa43 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Fri, 9 Feb 2024 11:34:43 -0600 Subject: [PATCH 06/21] update Bands to match new file names --- .../floating_forest.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/theia/operations/floating_forest_operations/floating_forest.py b/theia/operations/floating_forest_operations/floating_forest.py index d4c7d0a..dbb35bb 100644 --- a/theia/operations/floating_forest_operations/floating_forest.py +++ b/theia/operations/floating_forest_operations/floating_forest.py @@ -12,11 +12,13 @@ from ..abstract_operation import AbstractOperation # https://docs.python.org/3/library/csv.html -LANDSAT = {'red': 'band5', 'green': 'band2', 'blue': 'band3', 'infrared': 'band4'} -LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'} +LANDSAT = {'red': 'B5', 'green': 'B2', 'blue': 'B3', 'infrared': 'B4'} +# LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'} +LANDSAT8 = {'red': 'B6', 'green': 'B3', 'blue': 'B4', 'infrared': 'B5'} # https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/LSDS-1822_Landsat8-9-OLI-TIRS-C2-L1-DFCB-v6.pdf # https://www.usgs.gov/faqs/what-are-band-designations-landsat-satellites -LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' } +# LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' } +LANDSAT9 = {'red': 'B6', 'green': 'B3', 'blue': 'B4', 'infrared': 'B5' } ff_config = type('Config', (object,), { @@ -67,10 +69,6 @@ def parse_options(filenames): format='[ff-import %(name)s] %(levelname)s %(asctime)-15s %(message)s' ) -LANDSAT = {'red': 'band5', 'green': 'band2', 'blue': 'band3', 'infrared': 'band4'} -LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'} -LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' } - def usage(): print(""" This script is used to process satellite imagery from LANDSAT 4, 5, 7, and 8 @@ -229,13 +227,13 @@ def run_ff(filenames, output_directory, manifest_directory): ff_config.SATELLITE = LANDSAT9 ff_config.RED_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['red'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['red'] + ".tif") ff_config.GREEN_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['green'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['green'] + ".tif") ff_config.BLUE_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['blue'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['blue'] + ".tif") ff_config.INFRARED_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['infrared'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['infrared'] + ".tif") ff_config.YOU_ARE_HERE = path.dirname(path.realpath(__file__)) @@ -254,6 +252,8 @@ def run_ff(filenames, output_directory, manifest_directory): logger.info("Processing source data to remove negative pixels") clamp = clamp_image boost = boost_image + print('MDY114 FF CONFIG BEFORE CLAMPING AND BOOSTING') + print(ff_config) clamp(ff_config.RED_CHANNEL, "red", ff_config, False) clamp(ff_config.INFRARED_CHANNEL, "green", ff_config, False) clamp(ff_config.BLUE_CHANNEL, "blue", ff_config, True) From b30968f747995be5620b91d251c1869400bc4fa2 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 15:40:49 -0600 Subject: [PATCH 07/21] remove comments from floating_forest.py xml operations --- .../floating_forest.py | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/theia/operations/floating_forest_operations/floating_forest.py b/theia/operations/floating_forest_operations/floating_forest.py index dbb35bb..4404f2f 100644 --- a/theia/operations/floating_forest_operations/floating_forest.py +++ b/theia/operations/floating_forest_operations/floating_forest.py @@ -13,11 +13,7 @@ # https://docs.python.org/3/library/csv.html LANDSAT = {'red': 'B5', 'green': 'B2', 'blue': 'B3', 'infrared': 'B4'} -# LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'} LANDSAT8 = {'red': 'B6', 'green': 'B3', 'blue': 'B4', 'infrared': 'B5'} -# https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/LSDS-1822_Landsat8-9-OLI-TIRS-C2-L1-DFCB-v6.pdf -# https://www.usgs.gov/faqs/what-are-band-designations-landsat-satellites -# LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' } LANDSAT9 = {'red': 'B6', 'green': 'B3', 'blue': 'B4', 'infrared': 'B5' } @@ -252,8 +248,6 @@ def run_ff(filenames, output_directory, manifest_directory): logger.info("Processing source data to remove negative pixels") clamp = clamp_image boost = boost_image - print('MDY114 FF CONFIG BEFORE CLAMPING AND BOOSTING') - print(ff_config) clamp(ff_config.RED_CHANNEL, "red", ff_config, False) clamp(ff_config.INFRARED_CHANNEL, "green", ff_config, False) clamp(ff_config.BLUE_CHANNEL, "blue", ff_config, True) @@ -443,9 +437,6 @@ def maybe_clean_scratch(config): #=========XML OPERATIONS============================= def get_field_text(tree, path): - # nsmap = {"espa": tree.getroot().nsmap[None]} - # node = tree.xpath(path, namespaces=nsmap) - # node = tree.xpath(path, namespaces=nsmap) node = tree.xpath(path) if len(node) > 0: return node[0].text @@ -458,58 +449,24 @@ def parse_metadata(scene, xml_filename): result = {'!scene': scene} tree = etree.parse(xml_filename) - print('MDY114 XML NSMAP') - # print(tree.getroot().nsmap) - # nsmap = {"espa": tree.getroot().nsmap[None]} - - # result['acquired_date'] = get_field_text(tree, "espa:global_metadata/espa:acquisition_date") result['acquired_date'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/DATE_ACQUIRED") - # result['acquired_time'] = get_field_text(tree, "espa:global_metadata/espa:scene_center_time") result['acquired_time'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SPACE_CENTER_TIME") - # result['sensor_id'] = get_field_text(tree, "espa:global_metadata/espa:instrument") result['sensor_id'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SENSOR_ID") - # result['spacecraft'] = get_field_text(tree, 'espa:global_metadata/espa:satellite') result['spacecraft'] = get_field_text(tree, '//IMAGE_ATTRIBUTES/SPACECRAFT_ID') - # result['!earth_sun_distance'] = get_field_text( - # tree, - # "espa:global_metadata/espa:earth_sun_distance") result['!earth_sun_distance'] = get_field_text( tree, "//IMAGE_ATTRIBUTES/EARTH_SUN_DISTANCE") - # angles = tree.xpath("espa:global_metadata/espa:solar_angles", namespaces=nsmap) - # angles = tree.xpath("espa:global_metadata/espa:solar_angles") - # if len(angles) > 0: - # result['!sun_azimuth'] = angles[0].get("azimuth") - # result['!sun_zenith'] = angles[0].get("zenith") result['!sun_azimuth'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SUN_AZIMUTH") result['!sun_zenith'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SUN_ELEVATION") - # covers = tree.xpath( - # "espa:bands/espa:band[@name='cfmask']/espa:percent_coverage/espa:cover", - # namespaces=nsmap) - # for cover in covers: - # if cover.get("type") == "cloud": - # result['!cloud_cover'] = cover.text - # if cover.get("type") == "water": - # result['!water_cover'] = cover.text result['!cloud_cover'] = get_field_text(tree,"//IMAGE_ATTRIBUTES/CLOUD_COVER") - # result['#utm_zone'] = get_field_text( - # tree, - # "espa:global_metadata/espa:projection_information/espa:utm_proj_params/espa:zone_code") result['#utm_zone'] = get_field_text( tree, "//LEVEL1_PROJECTION_PARAMETERS/UTM_ZONE") - # TODO ASK CLIFF ABOUT THIS ON CORNERS AND THERE IS NO WATER COVER CODE IN NEW XML - # corners = tree.xpath( - # "espa:global_metadata/espa:projection_information/espa:corner_point", - # namespaces=nsmap) - # for corner in corners: - # result["#scene_corner_{0}_x".format(corner.get("location"))] = corner.get("x") - # result["#scene_corner_{0}_y".format(corner.get("location"))] = corner.get("y") result["#scene_corner_UL_x"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_UL_PROJECTION_X_PRODUCT') result["#scene_corner_UL_y"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_UL_PROJECTION_Y_PRODUCT') result["#scene_corner_LR_x"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_LR_PROJECTION_X_PRODUCT') From 14c42892dabe87f4290e30834275e884cecf21df Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 15:47:41 -0600 Subject: [PATCH 08/21] Update ErosWraper to remove unused download_urls method and update comment of what requested -download result will look like --- theia/adapters/usgs/eros_wrapper.py | 74 +++++++++++------------------ 1 file changed, 27 insertions(+), 47 deletions(-) diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index c61aba8..a6b670f 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -87,61 +87,41 @@ def request_download(self, products): #{ # "requestId": 1591674034, # "version": "stable", - # "data": { - # "availableDownloads": [], - # "duplicateProducts": [], - # "preparingDownloads": [], - # "failed": [], - # "newRecords": [], - # "numInvalidScenes": 0 + # "data": # { + # "availableDownloads":[ + + # ], + # "duplicateProducts":[ + + # ], + # "preparingDownloads":[ + # { + # "downloadId":550754832, + # "eulaCode":"None", + # "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + # }, + # { + # "downloadId":550752861, + # "eulaCode":"None", + # "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + # } + # ], + # "failed":[ + + # ], + # "newRecords":{ + # "550754832":"20240131_143747", + # "550752861":"20240131_143747" + # }, + # "numInvalidScenes":0 # }, # "errorCode": null, # "errorMessage": null # } - # {'availableDownloads': [], 'duplicateProducts': [], 'preparingDownloads': [{'downloadId': 550754832, 'eulaCode': None, 'url': 'https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=='}, {'downloadId': 550752861, 'eulaCode': None, 'url': 'https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=='}], 'failed': [], 'newRecords': {'550754832': '20240131_143747', '550752861': '20240131_143747'}, 'numInvalidScenes': 0} return self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) - - def download_urls(self): - self.login() - product_urls = [] - - for result in results['availableDownloads']: - product_urls.append(result['url']) - - preparing_download_count = len(results['preparingDownloads']) - preparing_download_ids = [] - if preparing_download_count > 0: - for result in results['preparingDownloads']: - preparing_download_ids.append(result['downloadId']) - - payload = {"label": self.download_request_label} - results = self.send_request("download-retrieve", payload) - if results != False: - for result in results['available']: - if result['downloadId'] in preparing_download_ids: - preparing_download_ids.remove(result['downloadId']) - product_urls.append(result['url']) - - for result in results['requested']: - if result['downloadId'] in preparing_download_ids: - preparing_download_ids.remove(result['downloadId']) - product_urls.append(result['url']) - # Don't get all download urls, retrieve again after 30 seconds - while len(preparing_download_ids) > 0: - print(f"{len(preparing_download_ids)} downloads are not available yet. Waiting for 30s to retrieve again\n") - time.sleep(30) - results = self.send_request("download-retrieve", payload) - if results != False: - for result in results['available']: - if result['downloadId'] in preparing_download_ids: - preparing_download_ids.remove(result['downloadId']) - product_urls.append(result['url']) - - return product_urls - def parse_result_set(self, result_set): if not result_set: return [] From fea5e84c09628c0d4f43d7e5b65c667bfdd99a19 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 15:49:20 -0600 Subject: [PATCH 09/21] remove unused threading on eros wrapper --- theia/adapters/usgs/eros_wrapper.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index a6b670f..69f4c41 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -1,22 +1,16 @@ from .utils import Utils -from urllib.parse import urljoin import json import requests import sys import datetime -import time -import threading from sentry_sdk import capture_message EROS_SERVICE_URL = "https://m2m.cr.usgs.gov/api/api/json/stable/" -MAX_THREADS = 5 TOKEN_EXPIRY_HOURS = 2 class ErosWrapper(): def __init__(self): self.api_key = None self.login_time = None - self.sema = threading.Semaphore(value=MAX_THREADS) - self.threads = [] self.download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") From 16ce486f07b9017eacb4013d46dd4e007dbf7343 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 15:52:08 -0600 Subject: [PATCH 10/21] remove unused code from tasks.py wait_for_scene --- theia/adapters/usgs/tasks.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/theia/adapters/usgs/tasks.py b/theia/adapters/usgs/tasks.py index b27e2bb..3f76bdf 100644 --- a/theia/adapters/usgs/tasks.py +++ b/theia/adapters/usgs/tasks.py @@ -1,6 +1,5 @@ from celery import shared_task from theia.api import models -# from .espa_wrapper import EspaWrapper from .eros_wrapper import ErosWrapper from datetime import datetime, timedelta @@ -9,30 +8,20 @@ @shared_task(name='theia.adapters.usgs.tasks.wait_for_scene') def wait_for_scene(requested_scene_id, available_products): - print('MDY114 AVAILABLE PRODUCTS IN WAIT') - print(available_products) request = models.RequestedScene.objects.get(pk=requested_scene_id) if request.status == 1: return - # TO DO CHANGES THIS TO REQUEST FROM REQUEST DOWNLOAD - #status = EspaWrapper.order_status(request.scene_order_id) eros_wrapper = ErosWrapper() rerequest_download_result = eros_wrapper.request_download(available_products) - print('MDY114 REREQUEST DOWNLOAD') - print(rerequest_download_result) now = make_aware(datetime.utcnow()) request.checked_at = now request.save() if rerequest_download_result['preparingDownloads'] != None and len(rerequest_download_result['preparingDownloads']) == 0: request.status = 1 - print('MDY114 BEFORE AVAIL DOWNLOADS CHECK') - print(rerequest_download_result['availableDownloads']) - print(len(rerequest_download_result['availableDownloads'])) if len(rerequest_download_result['availableDownloads']) > 0: for item in rerequest_download_result['availableDownloads']: - print(item) request.scene_url = item['url'] request.save() models.JobBundle.objects.from_requested_scene(request) From 858725c9e1a3578daab2d00766d49a3f095d94a0 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 15:55:12 -0600 Subject: [PATCH 11/21] undo change in settings.py on ALLOWED_HOSTS --- theia/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/theia/settings.py b/theia/settings.py index 2169673..e824c25 100644 --- a/theia/settings.py +++ b/theia/settings.py @@ -44,6 +44,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.getenv('DJANGO_ENV').lower() != "production" +ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '127.0.0.1').split(',') SOCIAL_AUTH_JSONFIELD_ENABLED = True SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'email'] From 93e6dd8f255f3936e2b906b29ed182ad1a047534 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 16:03:46 -0600 Subject: [PATCH 12/21] update adapter.py to remove unused code --- theia/adapters/usgs/adapter.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/theia/adapters/usgs/adapter.py b/theia/adapters/usgs/adapter.py index 50b3b06..b65b0ba 100644 --- a/theia/adapters/usgs/adapter.py +++ b/theia/adapters/usgs/adapter.py @@ -1,6 +1,5 @@ import os.path import platform -import re import urllib.request import numpy as np @@ -8,7 +7,6 @@ from theia.utils import FileUtils from .eros_wrapper import ErosWrapper -from .espa_wrapper import EspaWrapper from .imagery_search import ImagerySearch from .tasks import wait_for_scene from .xml_helper import XmlHelper @@ -70,39 +68,23 @@ def __init__(self): def process_request(self, imagery_request): search = ImagerySearch.build_search(imagery_request) eros_wrapper = ErosWrapper() - print('MDY114 BEFORE EROS WRAPPER') scenes = eros_wrapper.search(search) - print('MDY114 AFTER EROS WRAPPER SCENES') - print(scenes) + if imagery_request.max_results: scenes = scenes[0:imagery_request.max_results] - print('MDY114 SCENES') - print(scenes) for scene in scenes: - print('MDY114 BEFORE EROS ADD SCENES TO ORDER LIST') eros_wrapper.add_scenes_to_order_list(scene, search) - print('MDY114 AFTER ADDING TO LIST') - print('MDY114 AVAILABLE PRODUCTS BEFORE') available_products = eros_wrapper.available_products(scene, search) - print('MDY114 AVAIALBLE PRODUCTS AFTER') - # result = EspaWrapper.order_all(scene, 'sr') - print(available_products) - result = eros_wrapper.request_download(available_products) - print(result) + eros_wrapper.request_download(available_products) - print('MDY114 AFTER ESPA') for item in available_products: - print('MDY114') - print(item) - # req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) req = models.RequestedScene.objects.create( scene_entity_id = item['displayId'], scene_order_id = item['productId'], **{'imagery_request': imagery_request} ) - print(req) wait_for_scene.delay(req.id, available_products) def construct_filename(self, bundle, suffix): @@ -129,10 +111,7 @@ def retrieve(self, job_bundle): # get the compressed scene data if we don't have it if not os.path.isfile(zip_path): - print('MDY114 HITS HERE?') urllib.request.urlretrieve(job_bundle.requested_scene.scene_url, zip_path) - print('MDY114 ZIP PATH') - print(zip_path) FileUtils.untar(zip_path, job_bundle.local_path) def default_extension(self): From 383dabdf441d27c2eee0e3eb3f4f1658f90b004e Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 16:06:25 -0600 Subject: [PATCH 13/21] remove logs from espa wrapper. will keep in case we need to make any future callouts to ESPA API --- theia/adapters/usgs/espa_wrapper.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/theia/adapters/usgs/espa_wrapper.py b/theia/adapters/usgs/espa_wrapper.py index 9fb6c1d..1ddc453 100644 --- a/theia/adapters/usgs/espa_wrapper.py +++ b/theia/adapters/usgs/espa_wrapper.py @@ -18,18 +18,12 @@ def list_orders(cls): @classmethod def locate_collections(cls, scene_id, desired_product_id): results = cls.espa_get('available-products', scene_id) - print('MDY114 ESPA AFTER GETTING AVAILABLE PRODUCTS') - print(results) return [collection for collection in results if cls._product_is_available(desired_product_id, results[collection])] @classmethod def _product_is_available(cls, product_id, result_dict): - print('MDY114 HITS PRODUCT IS AVAILABLE') - print(product_id) - print(result_dict) - print(isinstance(result_dict, dict) and result_dict['products'] and product_id in result_dict['products']) return \ isinstance(result_dict, dict) and \ result_dict['products'] and \ @@ -41,10 +35,6 @@ def order_status(cls, order_id): @classmethod def order_one(cls, collection, scene_id, product_type): - print('MDY114 ORDERING ONE IN ESPA') - print('SCENE ID') - print(scene_id) - print(product_type) res = cls.espa_post('order', { collection: { 'inputs': [scene_id], @@ -52,16 +42,10 @@ def order_one(cls, collection, scene_id, product_type): }, 'format': 'gtiff' }) - print('MDY114 AFTER POSTING ORDER') - print(res) return res['orderid'] @classmethod def order_all(cls, scene_id, product_type): - print('MDY114 SCENE ID') - print(scene_id) - print('MDY114 PRODUCT TYPE') - print(product_type) return [ { 'scene_entity_id': scene_id, @@ -93,13 +77,10 @@ def espa_get(cls, url, request_data, **kwargs): @classmethod def espa_post(cls, url, request_data, **kwargs): - print('MDY114 HITS ESPA POST') new_args = cls.espa_prepare(request_data, **kwargs) new_url = cls.api_url(url) if request_data: new_args = {**new_args, **{'json': request_data}} - print(new_url) - print(new_args) return requests.post(new_url, **new_args).json() @classmethod From 355c9f03edd98f815c3c422c60325fc16a12a017 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Mon, 12 Feb 2024 17:05:40 -0600 Subject: [PATCH 14/21] fix tests in Adapter and set default dataset to Landsat c2 l2 --- tests/adapters/usgs/test_adapter.py | 27 ++++++++++++++++----------- theia/adapters/usgs/eros_wrapper.py | 18 ++++++++++-------- theia/adapters/usgs/imagery_search.py | 3 ++- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/tests/adapters/usgs/test_adapter.py b/tests/adapters/usgs/test_adapter.py index 2b1477d..b17a2df 100644 --- a/tests/adapters/usgs/test_adapter.py +++ b/tests/adapters/usgs/test_adapter.py @@ -8,6 +8,9 @@ import tarfile +EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}] +DEFAULT_SEARCH = {'datasetName': 'LANDSAT_OT_C2_L2'} + class TestUsgsAdapter: def test_resolve_relative_image(self): request = ImageryRequest(adapter_name='usgs', dataset_name='LANDSAT_8_C1') @@ -18,20 +21,22 @@ def test_construct_filename(self): bundle = JobBundle(scene_entity_id='LC08', local_path='tmp/') assert(Adapter().construct_filename(bundle, 'aerosol')=='LC08_sr_aerosol.tif') - @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value={}) + @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE) + @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH) @patch('theia.adapters.usgs.ErosWrapper.search', return_value=['some scene id']) - @patch('theia.adapters.usgs.EspaWrapper.order_all', return_value=[{}]) + @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3)) @patch('theia.adapters.usgs.tasks.wait_for_scene.delay') - def test_process_request(self, mock_wait, mock_rso, mock_order_all, mock_search, mock_build): + def test_process_request(self, mock_wait, mock_requested_scene_creation, mock_add_scene_to_order, mock_search, mock_build, mock_avail_prods): request = ImageryRequest() Adapter().process_request(request) mock_build.assert_called_once_with(request) - mock_search.assert_called_once_with({}) - mock_order_all.assert_called_once_with('some scene id', 'sr') - mock_rso.assert_called_once() - mock_wait.assert_called_once_with(3) + mock_search.assert_called_once_with(DEFAULT_SEARCH) + mock_add_scene_to_order.assert_called_once_with('some scene id', DEFAULT_SEARCH) + mock_avail_prods.assert_called_once_with('some scene id', DEFAULT_SEARCH) + mock_requested_scene_creation.assert_called_once() + mock_wait.assert_called_once_with(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE) @patch('os.path.isfile', return_value=False) @patch('platform.uname_result.node', new_callable=PropertyMock, return_value='testhostname') @@ -57,16 +62,16 @@ def test_remap_pixel(self): assert(remap.tolist()==[0, 0, 125, 250, 255]) assert(remap.dtype==np.uint8) - @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value={}) + @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH) @patch('theia.adapters.usgs.ErosWrapper.search', return_value=[1, 2, 3, 4, 5]) - @patch('theia.adapters.usgs.EspaWrapper.order_all', return_value=[{}]) + @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3)) @patch('theia.adapters.usgs.tasks.wait_for_scene.delay') - def test_limit_scenes(self, mock_wait, mock_rso, mock_order_all, mock_search, mock_build): + def test_limit_scenes(self, mock_wait, mock_rso, mock_add_scenes_to_order, mock_search, mock_build): request = ImageryRequest(max_results=3) Adapter().process_request(request) - assert(mock_order_all.call_count==3) + assert(mock_add_scenes_to_order.call_count==3) @patch('theia.adapters.usgs.XmlHelper.get_tree', autospec=True) @patch('theia.adapters.usgs.XmlHelper.resolve') diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index 69f4c41..3a25112 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -41,6 +41,7 @@ def search(self, search): def add_scenes_to_order_list(self, scene_id, search): self.login() + scene_list_add_payload = { "listId": scene_id, "idField": "displayId", #default is search by entityId @@ -59,14 +60,15 @@ def available_products(self, list_id, search): results = self.send_request(EROS_SERVICE_URL + "download-options", download_options_payload) products = [] - for result in results: - if result["bulkAvailable"] and result['downloadSystem'] != 'folder': - products.append( - { - "entityId": result['entityId'], - "productId": result['id'], - "displayId": result['displayId'] - }) + if results is not None: + for result in results: + if result["bulkAvailable"] and result['downloadSystem'] != 'folder': + products.append( + { + "entityId": result['entityId'], + "productId": result['id'], + "displayId": result['displayId'] + }) return products def request_download(self, products): diff --git a/theia/adapters/usgs/imagery_search.py b/theia/adapters/usgs/imagery_search.py index d492228..c401497 100644 --- a/theia/adapters/usgs/imagery_search.py +++ b/theia/adapters/usgs/imagery_search.py @@ -8,8 +8,9 @@ class ImagerySearch: @classmethod def build_search(cls, imagery_request): search = {} - search['datasetName'] = imagery_request.dataset_name + if imagery_request.dataset_name is None: + search['datasetName'] = 'LANDSAT_OT_C2_L2' if imagery_request.wgs_row and imagery_request.wgs_path: cls.add_wgs_row_and_path(search, row=imagery_request.wgs_row, path=imagery_request.wgs_path) From 2e221fa44e0bd716a78b876274d017a8e32a7006 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Tue, 13 Feb 2024 09:58:15 -0600 Subject: [PATCH 15/21] update eros scene-search test to not take in the expected response as an argument to expected call --- tests/adapters/usgs/test_eros_wrapper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/adapters/usgs/test_eros_wrapper.py b/tests/adapters/usgs/test_eros_wrapper.py index 62318dc..044b4ed 100644 --- a/tests/adapters/usgs/test_eros_wrapper.py +++ b/tests/adapters/usgs/test_eros_wrapper.py @@ -17,8 +17,7 @@ def test_send_search_request(self, mockRequestSend, mockPassword, mockUsername): ), call( 'https://m2m.cr.usgs.gov/api/api/json/stable/scene-search', - {'datasetName': 'LANDSAT_BAND'}, - {'results': ['LC01_FAKESCENE_007']} + {'datasetName': 'LANDSAT_BAND'} ) ], any_order=False) From 662807949eed1b44024f557b0fa18da9400f669b Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Tue, 13 Feb 2024 10:39:01 -0600 Subject: [PATCH 16/21] remove unused vars on tests and update wait_for_scene tests to expect eros calls instead of ESPA calls --- tests/adapters/usgs/test_adapter.py | 5 +-- tests/adapters/usgs/test_usgs_tasks.py | 48 ++++++++++++++++++-------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/adapters/usgs/test_adapter.py b/tests/adapters/usgs/test_adapter.py index b17a2df..20847b2 100644 --- a/tests/adapters/usgs/test_adapter.py +++ b/tests/adapters/usgs/test_adapter.py @@ -1,12 +1,9 @@ -import pytest -from unittest import mock from unittest.mock import patch, PropertyMock import numpy as np -from theia.adapters.usgs import Adapter, ErosWrapper, EspaWrapper, ImagerySearch +from theia.adapters.usgs import Adapter from theia.api.models import ImageryRequest, JobBundle, RequestedScene -import tarfile EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}] DEFAULT_SEARCH = {'datasetName': 'LANDSAT_OT_C2_L2'} diff --git a/tests/adapters/usgs/test_usgs_tasks.py b/tests/adapters/usgs/test_usgs_tasks.py index fed223e..bb1e145 100644 --- a/tests/adapters/usgs/test_usgs_tasks.py +++ b/tests/adapters/usgs/test_usgs_tasks.py @@ -1,39 +1,59 @@ -import pytest from unittest.mock import patch import theia.adapters.usgs.tasks +EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}] +EROS_REQUEST_DOWNLOAD_PENDING_DOWNLOADS_RESULT = { + "availableDownloads":[], + "duplicateProducts":[], + "preparingDownloads":[ + { + "downloadId":550754832, + "eulaCode":"None", + "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + } + ] +} + +EROS_REQUEST_DOWNLOAD_DOWNLOADS_AVAILABLE_RESULT = { + "availableDownloads":[{ + "downloadId":550752861, + "eulaCode":"None", + "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + }], + "duplicateProducts":[], + "preparingDownloads":[] +} + @patch('theia.api.models.RequestedScene.objects.get', autospec=True) -@patch('theia.adapters.usgs.EspaWrapper.order_status', autospec=True) +@patch('theia.adapters.usgs.ErosWrapper.request_download', autospec=True) def test_wait_for_scene_already_done(mockStatus, mockGetScene): mockGetScene.return_value.status=1 - theia.adapters.usgs.tasks.wait_for_scene(1) + theia.adapters.usgs.tasks.wait_for_scene(1, EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockStatus.assert_not_called() @patch('theia.api.models.RequestedScene.objects.get', autospec=True) -@patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='whatever') -@patch('theia.adapters.usgs.EspaWrapper.download_urls', autospec=True) +@patch('theia.adapters.usgs.ErosWrapper.request_download', return_value=EROS_REQUEST_DOWNLOAD_PENDING_DOWNLOADS_RESULT) @patch('theia.adapters.usgs.tasks.wait_for_scene.apply_async', autospec=True) -def test_wait_for_scene_pending(mockWait, mockUrls, mockStatus, mockGetScene): +def test_wait_for_scene_pending(mockWait, mockStatus, mockGetScene): mockGetScene.return_value.status=0 mockGetScene.return_value.scene_order_id='order id' - theia.adapters.usgs.tasks.wait_for_scene(3) + theia.adapters.usgs.tasks.wait_for_scene(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.assert_called_once_with(pk=3) - mockStatus.assert_called_once_with('order id') + mockStatus.assert_called_once_with(EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.return_value.save.assert_called_once() mockWait.assert_called_once() @patch('theia.api.models.RequestedScene.objects.get', autospec=True) -@patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='complete') -@patch('theia.adapters.usgs.EspaWrapper.download_urls') +@patch('theia.adapters.usgs.ErosWrapper.request_download', return_value=EROS_REQUEST_DOWNLOAD_DOWNLOADS_AVAILABLE_RESULT) @patch('theia.api.models.JobBundleManager.from_requested_scene', autospec=True) -def test_wait_for_scene_ready(mockConstruct, mockUrls, mockStatus, mockGetScene): +def test_wait_for_scene_ready(mockConstruct, mockStatus, mockGetScene): mockGetScene.return_value.status=0 mockGetScene.return_value.scene_order_id='order id' - theia.adapters.usgs.tasks.wait_for_scene(3) + theia.adapters.usgs.tasks.wait_for_scene(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.assert_called_once_with(pk=3) - mockStatus.assert_called_once_with('order id') + mockStatus.assert_called_once_with(EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.return_value.save.assert_called() - mockConstruct.assert_called_once() + mockConstruct.assert_called_once() \ No newline at end of file From 1fdd88812168039bc1aa23c53edc4add96442164 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Wed, 14 Feb 2024 08:13:01 -0600 Subject: [PATCH 17/21] adding tests to add_scene_to_order_list new method --- tests/adapters/usgs/test_eros_wrapper.py | 28 ++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/adapters/usgs/test_eros_wrapper.py b/tests/adapters/usgs/test_eros_wrapper.py index 044b4ed..d146ddd 100644 --- a/tests/adapters/usgs/test_eros_wrapper.py +++ b/tests/adapters/usgs/test_eros_wrapper.py @@ -1,13 +1,14 @@ from theia.adapters.usgs import ErosWrapper -from unittest.mock import patch, PropertyMock, call +from unittest.mock import patch, call +import datetime class TestErosWrapper: @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value={'results': ['LC01_FAKESCENE_007']}) - def test_send_search_request(self, mockRequestSend, mockPassword, mockUsername): + def test_send_search_request(self, mockRequestSend, *_): fakeSearch = {'datasetName' : 'LANDSAT_BAND'} ErosWrapper().search(search=fakeSearch) mockRequestSend.assert_has_calls([ @@ -53,3 +54,26 @@ def test_parse_result_set(self): ) assert result == ['a'] + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_add_scene_to_order_list(self, mock_send_request): + fakeSearch = {'datasetName' : 'LANDSAT_BAND'} + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + eros_wrapper.add_scenes_to_order_list(scene_id=1, search=fakeSearch) + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/scene-list-add', + {'listId': 1, 'idField': 'displayId', 'entityId': 1, 'datasetName': 'LANDSAT_BAND'} + ) + ], + any_order=False) + + @patch('theia.adapters.usgs.ErosWrapper.login') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_add_scene_to_order_list_logs_in(self, _, mock_login): + fakeSearch = {'datasetName' : 'LANDSAT_BAND'} + eros_wrapper = ErosWrapper() + eros_wrapper.add_scenes_to_order_list(scene_id=1, search=fakeSearch) + mock_login.assert_called_once() + + From 29dd53e67e4dad3c6e93154a6c1dde373cfb3952 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Thu, 15 Feb 2024 14:01:53 -0600 Subject: [PATCH 18/21] add login and token expiration check tests --- tests/adapters/usgs/test_eros_wrapper.py | 49 +++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/adapters/usgs/test_eros_wrapper.py b/tests/adapters/usgs/test_eros_wrapper.py index d146ddd..83a4205 100644 --- a/tests/adapters/usgs/test_eros_wrapper.py +++ b/tests/adapters/usgs/test_eros_wrapper.py @@ -1,10 +1,57 @@ from theia.adapters.usgs import ErosWrapper - from unittest.mock import patch, call import datetime +THREE_HOURS_AGO = datetime.datetime.utcnow() - datetime.timedelta(hours=3) class TestErosWrapper: + def test_is_token_expired_no_login_time(self): + is_expired = ErosWrapper().is_token_expired() + assert is_expired == True + + def test_is_token_expired_login_time_past_expiry(self): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = THREE_HOURS_AGO + assert eros_wrapper.is_token_expired() == True + + def test_is_token_expired_login_time_not_expired(self): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + assert eros_wrapper.is_token_expired() == False + + @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') + @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_login_no_login_time(self, mock_send_request, *_): + eros_wrapper = ErosWrapper() + eros_wrapper.login() + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/login', + {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'} + ) + ]) + + @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') + @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_login_token_expired(self, mock_send_request, *_): + eros_wrapper = ErosWrapper() + eros_wrapper.login() + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/login', + {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'} + ) + ]) + + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_login_not_called_when_token_not_expired(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + eros_wrapper.login() + mock_send_request.assert_not_called() + @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value={'results': ['LC01_FAKESCENE_007']}) From 346247a59cb5477663e6f4e2ac5cb4822ea5a7a1 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Thu, 15 Feb 2024 14:25:07 -0600 Subject: [PATCH 19/21] add tests for requesting available products --- tests/adapters/usgs/test_eros_wrapper.py | 57 +++++++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/adapters/usgs/test_eros_wrapper.py b/tests/adapters/usgs/test_eros_wrapper.py index 83a4205..1824933 100644 --- a/tests/adapters/usgs/test_eros_wrapper.py +++ b/tests/adapters/usgs/test_eros_wrapper.py @@ -2,9 +2,9 @@ from unittest.mock import patch, call import datetime THREE_HOURS_AGO = datetime.datetime.utcnow() - datetime.timedelta(hours=3) +FAKE_SEARCH = {'datasetName' : 'LANDSAT_BAND'} class TestErosWrapper: - def test_is_token_expired_no_login_time(self): is_expired = ErosWrapper().is_token_expired() assert is_expired == True @@ -56,8 +56,7 @@ def test_login_not_called_when_token_not_expired(self, mock_send_request): @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value={'results': ['LC01_FAKESCENE_007']}) def test_send_search_request(self, mockRequestSend, *_): - fakeSearch = {'datasetName' : 'LANDSAT_BAND'} - ErosWrapper().search(search=fakeSearch) + ErosWrapper().search(search=FAKE_SEARCH) mockRequestSend.assert_has_calls([ call( 'https://m2m.cr.usgs.gov/api/api/json/stable/login', @@ -103,10 +102,9 @@ def test_parse_result_set(self): @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) def test_add_scene_to_order_list(self, mock_send_request): - fakeSearch = {'datasetName' : 'LANDSAT_BAND'} eros_wrapper = ErosWrapper() eros_wrapper.login_time = datetime.datetime.utcnow() - eros_wrapper.add_scenes_to_order_list(scene_id=1, search=fakeSearch) + eros_wrapper.add_scenes_to_order_list(scene_id=1, search=FAKE_SEARCH) mock_send_request.assert_has_calls([ call( 'https://m2m.cr.usgs.gov/api/api/json/stable/scene-list-add', @@ -118,9 +116,54 @@ def test_add_scene_to_order_list(self, mock_send_request): @patch('theia.adapters.usgs.ErosWrapper.login') @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) def test_add_scene_to_order_list_logs_in(self, _, mock_login): - fakeSearch = {'datasetName' : 'LANDSAT_BAND'} eros_wrapper = ErosWrapper() - eros_wrapper.add_scenes_to_order_list(scene_id=1, search=fakeSearch) + eros_wrapper.add_scenes_to_order_list(scene_id=1, search=FAKE_SEARCH) + mock_login.assert_called_once() + + @patch('theia.adapters.usgs.ErosWrapper.login') + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_available_products_logs_in(self, _, mock_login): + eros_wrapper = ErosWrapper() + eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) mock_login.assert_called_once() + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': True, 'downloadSystem': 'dds', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}]) + def test_available_products(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/download-options', + {'listId': 1, 'datasetName': 'LANDSAT_BAND'} + ) + ], + any_order=False) + assert len(avail_products) == 1 + assert avail_products[0] == { + 'entityId': 'LC01_FAKESCENE_007', + 'productId': 1, + 'displayId': 'LC01FAKESCENE007' + } + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[]) + def test_available_products_has_no_results(self, _): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + assert avail_products == [] + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': False, 'downloadSystem': 'dds', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}]) + def test_avail_products_result_not_available(self, _): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + assert avail_products == [] + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': True, 'downloadSystem': 'folder', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}]) + def test_avail_products_result_download_system_incorrect(self, _): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + assert avail_products == [] From 353d2b7a128380acbe665fade2104cf8f6504620 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Thu, 15 Feb 2024 14:41:53 -0600 Subject: [PATCH 20/21] add tests for requesting downloads --- tests/adapters/usgs/test_eros_wrapper.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/adapters/usgs/test_eros_wrapper.py b/tests/adapters/usgs/test_eros_wrapper.py index 1824933..19658d6 100644 --- a/tests/adapters/usgs/test_eros_wrapper.py +++ b/tests/adapters/usgs/test_eros_wrapper.py @@ -167,3 +167,32 @@ def test_avail_products_result_download_system_incorrect(self, _): avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) assert avail_products == [] + @patch('theia.adapters.usgs.ErosWrapper.login') + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_request_download_logs_in(self, _, mock_login): + eros_wrapper = ErosWrapper() + eros_wrapper.request_download([{ + 'entityId': 'LC01_FAKESCENE_007', + 'productId': 1, + 'displayId': 'LC01FAKESCENE007' + }]) + mock_login.assert_called_once() + + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_request_download(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = [{ + 'entityId': 'LC01_FAKESCENE_007', + 'productId': 1, + 'displayId': 'LC01FAKESCENE007' + }] + eros_wrapper.request_download(avail_products) + + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/download-request', + {'downloads': avail_products, 'label': eros_wrapper.download_request_label} + ) + ] + ) From dd14e8c92554b01b9d22bfd791c30c6f0ad5ec08 Mon Sep 17 00:00:00 2001 From: yuenmichelle1 Date: Thu, 15 Feb 2024 14:56:03 -0600 Subject: [PATCH 21/21] update test_adapter.py to fake user login --- tests/adapters/usgs/test_adapter.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/adapters/usgs/test_adapter.py b/tests/adapters/usgs/test_adapter.py index 20847b2..f282b3f 100644 --- a/tests/adapters/usgs/test_adapter.py +++ b/tests/adapters/usgs/test_adapter.py @@ -1,6 +1,6 @@ from unittest.mock import patch, PropertyMock import numpy as np - +import datetime from theia.adapters.usgs import Adapter from theia.api.models import ImageryRequest, JobBundle, RequestedScene @@ -18,13 +18,14 @@ def test_construct_filename(self): bundle = JobBundle(scene_entity_id='LC08', local_path='tmp/') assert(Adapter().construct_filename(bundle, 'aerosol')=='LC08_sr_aerosol.tif') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE) @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH) @patch('theia.adapters.usgs.ErosWrapper.search', return_value=['some scene id']) @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3)) @patch('theia.adapters.usgs.tasks.wait_for_scene.delay') - def test_process_request(self, mock_wait, mock_requested_scene_creation, mock_add_scene_to_order, mock_search, mock_build, mock_avail_prods): + def test_process_request(self, mock_wait, mock_requested_scene_creation, mock_add_scene_to_order, mock_search, mock_build, mock_avail_prods, _): request = ImageryRequest() Adapter().process_request(request) @@ -59,12 +60,14 @@ def test_remap_pixel(self): assert(remap.tolist()==[0, 0, 125, 250, 255]) assert(remap.dtype==np.uint8) + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE) @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH) @patch('theia.adapters.usgs.ErosWrapper.search', return_value=[1, 2, 3, 4, 5]) - @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3)) @patch('theia.adapters.usgs.tasks.wait_for_scene.delay') - def test_limit_scenes(self, mock_wait, mock_rso, mock_add_scenes_to_order, mock_search, mock_build): + @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) + def test_limit_scenes(self, mock_add_scenes_to_order, *_): request = ImageryRequest(max_results=3) Adapter().process_request(request)