diff --git a/tests/adapters/usgs/test_adapter.py b/tests/adapters/usgs/test_adapter.py index 2b1477d..f282b3f 100644 --- a/tests/adapters/usgs/test_adapter.py +++ b/tests/adapters/usgs/test_adapter.py @@ -1,12 +1,12 @@ -import pytest -from unittest import mock from unittest.mock import patch, PropertyMock import numpy as np - -from theia.adapters.usgs import Adapter, ErosWrapper, EspaWrapper, ImagerySearch +import datetime +from theia.adapters.usgs import Adapter from theia.api.models import ImageryRequest, JobBundle, RequestedScene -import tarfile + +EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}] +DEFAULT_SEARCH = {'datasetName': 'LANDSAT_OT_C2_L2'} class TestUsgsAdapter: def test_resolve_relative_image(self): @@ -18,20 +18,23 @@ def test_construct_filename(self): bundle = JobBundle(scene_entity_id='LC08', local_path='tmp/') assert(Adapter().construct_filename(bundle, 'aerosol')=='LC08_sr_aerosol.tif') - @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value={}) + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE) + @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH) @patch('theia.adapters.usgs.ErosWrapper.search', return_value=['some scene id']) - @patch('theia.adapters.usgs.EspaWrapper.order_all', return_value=[{}]) + @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3)) @patch('theia.adapters.usgs.tasks.wait_for_scene.delay') - def test_process_request(self, mock_wait, mock_rso, mock_order_all, mock_search, mock_build): + def test_process_request(self, mock_wait, mock_requested_scene_creation, mock_add_scene_to_order, mock_search, mock_build, mock_avail_prods, _): request = ImageryRequest() Adapter().process_request(request) mock_build.assert_called_once_with(request) - mock_search.assert_called_once_with({}) - mock_order_all.assert_called_once_with('some scene id', 'sr') - mock_rso.assert_called_once() - mock_wait.assert_called_once_with(3) + mock_search.assert_called_once_with(DEFAULT_SEARCH) + mock_add_scene_to_order.assert_called_once_with('some scene id', DEFAULT_SEARCH) + mock_avail_prods.assert_called_once_with('some scene id', DEFAULT_SEARCH) + mock_requested_scene_creation.assert_called_once() + mock_wait.assert_called_once_with(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE) @patch('os.path.isfile', return_value=False) @patch('platform.uname_result.node', new_callable=PropertyMock, return_value='testhostname') @@ -57,16 +60,18 @@ def test_remap_pixel(self): assert(remap.tolist()==[0, 0, 125, 250, 255]) assert(remap.dtype==np.uint8) - @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value={}) + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE) + @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH) @patch('theia.adapters.usgs.ErosWrapper.search', return_value=[1, 2, 3, 4, 5]) - @patch('theia.adapters.usgs.EspaWrapper.order_all', return_value=[{}]) @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3)) @patch('theia.adapters.usgs.tasks.wait_for_scene.delay') - def test_limit_scenes(self, mock_wait, mock_rso, mock_order_all, mock_search, mock_build): + @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}]) + def test_limit_scenes(self, mock_add_scenes_to_order, *_): request = ImageryRequest(max_results=3) Adapter().process_request(request) - assert(mock_order_all.call_count==3) + assert(mock_add_scenes_to_order.call_count==3) @patch('theia.adapters.usgs.XmlHelper.get_tree', autospec=True) @patch('theia.adapters.usgs.XmlHelper.resolve') diff --git a/tests/adapters/usgs/test_eros_wrapper.py b/tests/adapters/usgs/test_eros_wrapper.py index 62318dc..19658d6 100644 --- a/tests/adapters/usgs/test_eros_wrapper.py +++ b/tests/adapters/usgs/test_eros_wrapper.py @@ -1,15 +1,62 @@ from theia.adapters.usgs import ErosWrapper - -from unittest.mock import patch, PropertyMock, call +from unittest.mock import patch, call +import datetime +THREE_HOURS_AGO = datetime.datetime.utcnow() - datetime.timedelta(hours=3) +FAKE_SEARCH = {'datasetName' : 'LANDSAT_BAND'} class TestErosWrapper: + def test_is_token_expired_no_login_time(self): + is_expired = ErosWrapper().is_token_expired() + assert is_expired == True + + def test_is_token_expired_login_time_past_expiry(self): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = THREE_HOURS_AGO + assert eros_wrapper.is_token_expired() == True + + def test_is_token_expired_login_time_not_expired(self): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + assert eros_wrapper.is_token_expired() == False + + @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') + @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_login_no_login_time(self, mock_send_request, *_): + eros_wrapper = ErosWrapper() + eros_wrapper.login() + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/login', + {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'} + ) + ]) + + @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') + @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_login_token_expired(self, mock_send_request, *_): + eros_wrapper = ErosWrapper() + eros_wrapper.login() + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/login', + {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'} + ) + ]) + + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_login_not_called_when_token_not_expired(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + eros_wrapper.login() + mock_send_request.assert_not_called() @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman') @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd') @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value={'results': ['LC01_FAKESCENE_007']}) - def test_send_search_request(self, mockRequestSend, mockPassword, mockUsername): - fakeSearch = {'datasetName' : 'LANDSAT_BAND'} - ErosWrapper().search(search=fakeSearch) + def test_send_search_request(self, mockRequestSend, *_): + ErosWrapper().search(search=FAKE_SEARCH) mockRequestSend.assert_has_calls([ call( 'https://m2m.cr.usgs.gov/api/api/json/stable/login', @@ -17,8 +64,7 @@ def test_send_search_request(self, mockRequestSend, mockPassword, mockUsername): ), call( 'https://m2m.cr.usgs.gov/api/api/json/stable/scene-search', - {'datasetName': 'LANDSAT_BAND'}, - {'results': ['LC01_FAKESCENE_007']} + {'datasetName': 'LANDSAT_BAND'} ) ], any_order=False) @@ -54,3 +100,99 @@ def test_parse_result_set(self): ) assert result == ['a'] + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_add_scene_to_order_list(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + eros_wrapper.add_scenes_to_order_list(scene_id=1, search=FAKE_SEARCH) + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/scene-list-add', + {'listId': 1, 'idField': 'displayId', 'entityId': 1, 'datasetName': 'LANDSAT_BAND'} + ) + ], + any_order=False) + + @patch('theia.adapters.usgs.ErosWrapper.login') + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200) + def test_add_scene_to_order_list_logs_in(self, _, mock_login): + eros_wrapper = ErosWrapper() + eros_wrapper.add_scenes_to_order_list(scene_id=1, search=FAKE_SEARCH) + mock_login.assert_called_once() + + @patch('theia.adapters.usgs.ErosWrapper.login') + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_available_products_logs_in(self, _, mock_login): + eros_wrapper = ErosWrapper() + eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + mock_login.assert_called_once() + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': True, 'downloadSystem': 'dds', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}]) + def test_available_products(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/download-options', + {'listId': 1, 'datasetName': 'LANDSAT_BAND'} + ) + ], + any_order=False) + assert len(avail_products) == 1 + assert avail_products[0] == { + 'entityId': 'LC01_FAKESCENE_007', + 'productId': 1, + 'displayId': 'LC01FAKESCENE007' + } + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[]) + def test_available_products_has_no_results(self, _): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + assert avail_products == [] + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': False, 'downloadSystem': 'dds', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}]) + def test_avail_products_result_not_available(self, _): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + assert avail_products == [] + + @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': True, 'downloadSystem': 'folder', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}]) + def test_avail_products_result_download_system_incorrect(self, _): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH) + assert avail_products == [] + + @patch('theia.adapters.usgs.ErosWrapper.login') + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_request_download_logs_in(self, _, mock_login): + eros_wrapper = ErosWrapper() + eros_wrapper.request_download([{ + 'entityId': 'LC01_FAKESCENE_007', + 'productId': 1, + 'displayId': 'LC01FAKESCENE007' + }]) + mock_login.assert_called_once() + + @patch('theia.adapters.usgs.ErosWrapper.send_request') + def test_request_download(self, mock_send_request): + eros_wrapper = ErosWrapper() + eros_wrapper.login_time = datetime.datetime.utcnow() + avail_products = [{ + 'entityId': 'LC01_FAKESCENE_007', + 'productId': 1, + 'displayId': 'LC01FAKESCENE007' + }] + eros_wrapper.request_download(avail_products) + + mock_send_request.assert_has_calls([ + call( + 'https://m2m.cr.usgs.gov/api/api/json/stable/download-request', + {'downloads': avail_products, 'label': eros_wrapper.download_request_label} + ) + ] + ) diff --git a/tests/adapters/usgs/test_usgs_tasks.py b/tests/adapters/usgs/test_usgs_tasks.py index fed223e..bb1e145 100644 --- a/tests/adapters/usgs/test_usgs_tasks.py +++ b/tests/adapters/usgs/test_usgs_tasks.py @@ -1,39 +1,59 @@ -import pytest from unittest.mock import patch import theia.adapters.usgs.tasks +EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}] +EROS_REQUEST_DOWNLOAD_PENDING_DOWNLOADS_RESULT = { + "availableDownloads":[], + "duplicateProducts":[], + "preparingDownloads":[ + { + "downloadId":550754832, + "eulaCode":"None", + "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + } + ] +} + +EROS_REQUEST_DOWNLOAD_DOWNLOADS_AVAILABLE_RESULT = { + "availableDownloads":[{ + "downloadId":550752861, + "eulaCode":"None", + "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + }], + "duplicateProducts":[], + "preparingDownloads":[] +} + @patch('theia.api.models.RequestedScene.objects.get', autospec=True) -@patch('theia.adapters.usgs.EspaWrapper.order_status', autospec=True) +@patch('theia.adapters.usgs.ErosWrapper.request_download', autospec=True) def test_wait_for_scene_already_done(mockStatus, mockGetScene): mockGetScene.return_value.status=1 - theia.adapters.usgs.tasks.wait_for_scene(1) + theia.adapters.usgs.tasks.wait_for_scene(1, EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockStatus.assert_not_called() @patch('theia.api.models.RequestedScene.objects.get', autospec=True) -@patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='whatever') -@patch('theia.adapters.usgs.EspaWrapper.download_urls', autospec=True) +@patch('theia.adapters.usgs.ErosWrapper.request_download', return_value=EROS_REQUEST_DOWNLOAD_PENDING_DOWNLOADS_RESULT) @patch('theia.adapters.usgs.tasks.wait_for_scene.apply_async', autospec=True) -def test_wait_for_scene_pending(mockWait, mockUrls, mockStatus, mockGetScene): +def test_wait_for_scene_pending(mockWait, mockStatus, mockGetScene): mockGetScene.return_value.status=0 mockGetScene.return_value.scene_order_id='order id' - theia.adapters.usgs.tasks.wait_for_scene(3) + theia.adapters.usgs.tasks.wait_for_scene(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.assert_called_once_with(pk=3) - mockStatus.assert_called_once_with('order id') + mockStatus.assert_called_once_with(EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.return_value.save.assert_called_once() mockWait.assert_called_once() @patch('theia.api.models.RequestedScene.objects.get', autospec=True) -@patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='complete') -@patch('theia.adapters.usgs.EspaWrapper.download_urls') +@patch('theia.adapters.usgs.ErosWrapper.request_download', return_value=EROS_REQUEST_DOWNLOAD_DOWNLOADS_AVAILABLE_RESULT) @patch('theia.api.models.JobBundleManager.from_requested_scene', autospec=True) -def test_wait_for_scene_ready(mockConstruct, mockUrls, mockStatus, mockGetScene): +def test_wait_for_scene_ready(mockConstruct, mockStatus, mockGetScene): mockGetScene.return_value.status=0 mockGetScene.return_value.scene_order_id='order id' - theia.adapters.usgs.tasks.wait_for_scene(3) + theia.adapters.usgs.tasks.wait_for_scene(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.assert_called_once_with(pk=3) - mockStatus.assert_called_once_with('order id') + mockStatus.assert_called_once_with(EROS_AVAILABLE_PRODUCTS_EXAMPLE) mockGetScene.return_value.save.assert_called() - mockConstruct.assert_called_once() + mockConstruct.assert_called_once() \ No newline at end of file diff --git a/theia/adapters/usgs/adapter.py b/theia/adapters/usgs/adapter.py index 9abdd2a..b65b0ba 100644 --- a/theia/adapters/usgs/adapter.py +++ b/theia/adapters/usgs/adapter.py @@ -1,6 +1,5 @@ import os.path import platform -import re import urllib.request import numpy as np @@ -8,7 +7,6 @@ from theia.utils import FileUtils from .eros_wrapper import ErosWrapper -from .espa_wrapper import EspaWrapper from .imagery_search import ImagerySearch from .tasks import wait_for_scene from .xml_helper import XmlHelper @@ -69,15 +67,25 @@ def __init__(self): def process_request(self, imagery_request): search = ImagerySearch.build_search(imagery_request) - scenes = ErosWrapper().search(search) + eros_wrapper = ErosWrapper() + scenes = eros_wrapper.search(search) + if imagery_request.max_results: scenes = scenes[0:imagery_request.max_results] for scene in scenes: - result = EspaWrapper.order_all(scene, 'sr') - for item in result: - req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}}) - wait_for_scene.delay(req.id) + eros_wrapper.add_scenes_to_order_list(scene, search) + available_products = eros_wrapper.available_products(scene, search) + + eros_wrapper.request_download(available_products) + + for item in available_products: + req = models.RequestedScene.objects.create( + scene_entity_id = item['displayId'], + scene_order_id = item['productId'], + **{'imagery_request': imagery_request} + ) + wait_for_scene.delay(req.id, available_products) def construct_filename(self, bundle, suffix): product = "sr" @@ -104,7 +112,6 @@ def retrieve(self, job_bundle): # get the compressed scene data if we don't have it if not os.path.isfile(zip_path): urllib.request.urlretrieve(job_bundle.requested_scene.scene_url, zip_path) - FileUtils.untar(zip_path, job_bundle.local_path) def default_extension(self): diff --git a/theia/adapters/usgs/eros_wrapper.py b/theia/adapters/usgs/eros_wrapper.py index a8cf7d6..3a25112 100644 --- a/theia/adapters/usgs/eros_wrapper.py +++ b/theia/adapters/usgs/eros_wrapper.py @@ -2,36 +2,135 @@ import json import requests import sys +import datetime from sentry_sdk import capture_message +EROS_SERVICE_URL = "https://m2m.cr.usgs.gov/api/api/json/stable/" +TOKEN_EXPIRY_HOURS = 2 class ErosWrapper(): + def __init__(self): + self.api_key = None + self.login_time = None + self.download_request_label = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - def search(self, search): - serviceUrl = "https://m2m.cr.usgs.gov/api/api/json/stable/" - loginParameters = { - 'username': Utils.get_username(), - 'password': Utils.get_password() - } - apiKey = self.send_request(serviceUrl + "login", loginParameters) - scenes = self.send_request(serviceUrl + "scene-search", search, apiKey) + + def login(self): + if self.login_time is None or self.is_token_expired(): + loginParameters = { + 'username': Utils.get_username(), + 'password': Utils.get_password() + } + self.api_key = self.send_request(EROS_SERVICE_URL + "login", loginParameters) + self.login_time = datetime.datetime.utcnow() + + + def is_token_expired(self): + if self.login_time: + token_expiration_time = self.login_time + datetime.timedelta(hours=TOKEN_EXPIRY_HOURS) + # minusing 1 minute to expiry check to protect against race condition + return datetime.datetime.utcnow() - datetime.timedelta(minutes=1) >= token_expiration_time + return True + + def search(self, search): + self.login() + scenes = self.send_request(EROS_SERVICE_URL + "scene-search", search) results = self.parse_result_set(scenes['results']) return results + def add_scenes_to_order_list(self, scene_id, search): + self.login() + + scene_list_add_payload = { + "listId": scene_id, + "idField": "displayId", #default is search by entityId + "entityId": scene_id, + "datasetName": search['datasetName'] + } + return self.send_request(EROS_SERVICE_URL + "scene-list-add", scene_list_add_payload) + + def available_products(self, list_id, search): + self.login() + + download_options_payload = { + "listId": list_id, + "datasetName": search['datasetName'] + } + results = self.send_request(EROS_SERVICE_URL + "download-options", download_options_payload) + products = [] + + if results is not None: + for result in results: + if result["bulkAvailable"] and result['downloadSystem'] != 'folder': + products.append( + { + "entityId": result['entityId'], + "productId": result['id'], + "displayId": result['displayId'] + }) + return products + + def request_download(self, products): + self.login() + + download_request_payload = { + "downloads": products, + "label": self.download_request_label + } + + #returns format like + #{ + # "requestId": 1591674034, + # "version": "stable", + # "data": # { + # "availableDownloads":[ + + # ], + # "duplicateProducts":[ + + # ], + # "preparingDownloads":[ + # { + # "downloadId":550754832, + # "eulaCode":"None", + # "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + # }, + # { + # "downloadId":550752861, + # "eulaCode":"None", + # "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ==" + # } + # ], + # "failed":[ + + # ], + # "newRecords":{ + # "550754832":"20240131_143747", + # "550752861":"20240131_143747" + # }, + # "numInvalidScenes":0 + # }, + # "errorCode": null, + # "errorMessage": null + # } + + return self.send_request(EROS_SERVICE_URL + "download-request", download_request_payload) + + def parse_result_set(self, result_set): if not result_set: return [] return [scene.get('displayId', None) for scene in result_set if 'displayId' in scene] - def send_request(self, url, data, apiKey=None): + def send_request(self, url, data): json_data = json.dumps(data) - if apiKey == None: + if self.api_key == None: response = requests.post(url, json_data) else: - headers = {'X-Auth-Token': apiKey} + headers = {'X-Auth-Token': self.api_key} response = requests.post(url, json_data, headers=headers) try: diff --git a/theia/adapters/usgs/imagery_search.py b/theia/adapters/usgs/imagery_search.py index d492228..c401497 100644 --- a/theia/adapters/usgs/imagery_search.py +++ b/theia/adapters/usgs/imagery_search.py @@ -8,8 +8,9 @@ class ImagerySearch: @classmethod def build_search(cls, imagery_request): search = {} - search['datasetName'] = imagery_request.dataset_name + if imagery_request.dataset_name is None: + search['datasetName'] = 'LANDSAT_OT_C2_L2' if imagery_request.wgs_row and imagery_request.wgs_path: cls.add_wgs_row_and_path(search, row=imagery_request.wgs_row, path=imagery_request.wgs_path) diff --git a/theia/adapters/usgs/tasks.py b/theia/adapters/usgs/tasks.py index 8d3c460..3f76bdf 100644 --- a/theia/adapters/usgs/tasks.py +++ b/theia/adapters/usgs/tasks.py @@ -1,27 +1,30 @@ from celery import shared_task from theia.api import models -from .espa_wrapper import EspaWrapper +from .eros_wrapper import ErosWrapper from datetime import datetime, timedelta from django.utils.timezone import make_aware @shared_task(name='theia.adapters.usgs.tasks.wait_for_scene') -def wait_for_scene(requested_scene_id): +def wait_for_scene(requested_scene_id, available_products): request = models.RequestedScene.objects.get(pk=requested_scene_id) if request.status == 1: return - status = EspaWrapper.order_status(request.scene_order_id) + eros_wrapper = ErosWrapper() + rerequest_download_result = eros_wrapper.request_download(available_products) now = make_aware(datetime.utcnow()) request.checked_at = now request.save() - if status == 'complete': + if rerequest_download_result['preparingDownloads'] != None and len(rerequest_download_result['preparingDownloads']) == 0: request.status = 1 - request.scene_url = EspaWrapper.download_urls(request.scene_order_id)[0] + if len(rerequest_download_result['availableDownloads']) > 0: + for item in rerequest_download_result['availableDownloads']: + request.scene_url = item['url'] request.save() models.JobBundle.objects.from_requested_scene(request) else: soon = now + timedelta(minutes=5) - wait_for_scene.apply_async((requested_scene_id,), eta=soon) + wait_for_scene.apply_async((requested_scene_id,available_products), eta=soon) diff --git a/theia/operations/floating_forest_operations/floating_forest.py b/theia/operations/floating_forest_operations/floating_forest.py index cd90375..4404f2f 100644 --- a/theia/operations/floating_forest_operations/floating_forest.py +++ b/theia/operations/floating_forest_operations/floating_forest.py @@ -12,11 +12,9 @@ from ..abstract_operation import AbstractOperation # https://docs.python.org/3/library/csv.html -LANDSAT = {'red': 'band5', 'green': 'band2', 'blue': 'band3', 'infrared': 'band4'} -LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'} -# https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/LSDS-1822_Landsat8-9-OLI-TIRS-C2-L1-DFCB-v6.pdf -# https://www.usgs.gov/faqs/what-are-band-designations-landsat-satellites -LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' } +LANDSAT = {'red': 'B5', 'green': 'B2', 'blue': 'B3', 'infrared': 'B4'} +LANDSAT8 = {'red': 'B6', 'green': 'B3', 'blue': 'B4', 'infrared': 'B5'} +LANDSAT9 = {'red': 'B6', 'green': 'B3', 'blue': 'B4', 'infrared': 'B5' } ff_config = type('Config', (object,), { @@ -58,8 +56,8 @@ def apply(self, filenames): def parse_options(filenames): ff_config.SCENE_NAME = path.dirname(filenames[0]) - ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_pixel_qa.tif") - ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + ".xml") + ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_qa_pixel.tif") + ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_mtl.xml") ff_config.INPUT_FILE = ff_config.NEW_MASK #===========SIMPLE====================== @@ -67,10 +65,6 @@ def parse_options(filenames): format='[ff-import %(name)s] %(levelname)s %(asctime)-15s %(message)s' ) -LANDSAT = {'red': 'band5', 'green': 'band2', 'blue': 'band3', 'infrared': 'band4'} -LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'} -LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' } - def usage(): print(""" This script is used to process satellite imagery from LANDSAT 4, 5, 7, and 8 @@ -125,8 +119,8 @@ def parse_options(filenames): ff_config.SCENE_DIR = path.dirname(filenames[0]) path_components = ff_config.SCENE_DIR.split('/') ff_config.SCENE_NAME = path_components[len(path_components) - 1] - ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_pixel_qa.tif") - ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + ".xml") + ff_config.NEW_MASK = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_qa_pixel.tif") + ff_config.METADATA_SRC = path.join(ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_mtl.xml") ff_config.INPUT_FILE = ff_config.NEW_MASK def generate_mask_tiles(): @@ -229,13 +223,13 @@ def run_ff(filenames, output_directory, manifest_directory): ff_config.SATELLITE = LANDSAT9 ff_config.RED_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['red'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['red'] + ".tif") ff_config.GREEN_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['green'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['green'] + ".tif") ff_config.BLUE_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['blue'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['blue'] + ".tif") ff_config.INFRARED_CHANNEL = path.join( - ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['infrared'] + ".tif") + ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_SR_" + ff_config.SATELLITE['infrared'] + ".tif") ff_config.YOU_ARE_HERE = path.dirname(path.realpath(__file__)) @@ -443,8 +437,7 @@ def maybe_clean_scratch(config): #=========XML OPERATIONS============================= def get_field_text(tree, path): - nsmap = {"espa": tree.getroot().nsmap[None]} - node = tree.xpath(path, namespaces=nsmap) + node = tree.xpath(path) if len(node) > 0: return node[0].text return '' @@ -456,41 +449,28 @@ def parse_metadata(scene, xml_filename): result = {'!scene': scene} tree = etree.parse(xml_filename) - nsmap = {"espa": tree.getroot().nsmap[None]} - - result['acquired_date'] = get_field_text(tree, "espa:global_metadata/espa:acquisition_date") - result['acquired_time'] = get_field_text(tree, "espa:global_metadata/espa:scene_center_time") - result['sensor_id'] = get_field_text(tree, "espa:global_metadata/espa:instrument") - result['spacecraft'] = get_field_text(tree, 'espa:global_metadata/espa:satellite') + result['acquired_date'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/DATE_ACQUIRED") + result['acquired_time'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SPACE_CENTER_TIME") + result['sensor_id'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SENSOR_ID") + result['spacecraft'] = get_field_text(tree, '//IMAGE_ATTRIBUTES/SPACECRAFT_ID') result['!earth_sun_distance'] = get_field_text( tree, - "espa:global_metadata/espa:earth_sun_distance") - - angles = tree.xpath("espa:global_metadata/espa:solar_angles", namespaces=nsmap) - if len(angles) > 0: - result['!sun_azimuth'] = angles[0].get("azimuth") - result['!sun_zenith'] = angles[0].get("zenith") - - covers = tree.xpath( - "espa:bands/espa:band[@name='cfmask']/espa:percent_coverage/espa:cover", - namespaces=nsmap) - for cover in covers: - if cover.get("type") == "cloud": - result['!cloud_cover'] = cover.text - if cover.get("type") == "water": - result['!water_cover'] = cover.text + "//IMAGE_ATTRIBUTES/EARTH_SUN_DISTANCE") + + result['!sun_azimuth'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SUN_AZIMUTH") + result['!sun_zenith'] = get_field_text(tree, "//IMAGE_ATTRIBUTES/SUN_ELEVATION") + + result['!cloud_cover'] = get_field_text(tree,"//IMAGE_ATTRIBUTES/CLOUD_COVER") result['#utm_zone'] = get_field_text( tree, - "espa:global_metadata/espa:projection_information/espa:utm_proj_params/espa:zone_code") - - corners = tree.xpath( - "espa:global_metadata/espa:projection_information/espa:corner_point", - namespaces=nsmap) - for corner in corners: - result["#scene_corner_{0}_x".format(corner.get("location"))] = corner.get("x") - result["#scene_corner_{0}_y".format(corner.get("location"))] = corner.get("y") + "//LEVEL1_PROJECTION_PARAMETERS/UTM_ZONE") + + result["#scene_corner_UL_x"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_UL_PROJECTION_X_PRODUCT') + result["#scene_corner_UL_y"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_UL_PROJECTION_Y_PRODUCT') + result["#scene_corner_LR_x"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_LR_PROJECTION_X_PRODUCT') + result["#scene_corner_LR_y"] = get_field_text(tree, '//PROJECTION_ATTRIBUTES/CORNER_LR_PROJECTION_Y_PRODUCT') return result