Theia Update USGS API Callouts to reflect USGS API changes and Landsa…

…t Collection 1 Data Shutdown (#189)

* commenting out espa wrapper orders. orders now done through eros wrapper (m2m api)

* remove unneeded apiKey parameter

* attempt to replace espa api calls with corresponding eros calls

* update metadata parsing

* update typo missing arg on cloud_cover

* update Bands to match new file names

* remove comments from floating_forest.py xml operations

* Update ErosWraper to remove unused download_urls method and update comment of what requested -download result will look like

* remove unused threading on eros wrapper

* remove unused code from tasks.py wait_for_scene

* undo change in settings.py on ALLOWED_HOSTS

* update adapter.py to remove unused code

* remove logs from espa wrapper. will keep in case we need to make any future callouts to ESPA API

* fix tests in Adapter and set default dataset to Landsat c2 l2

* update eros scene-search test to not take in the expected response as an argument to expected call

* remove unused vars on tests and update wait_for_scene tests to expect eros calls instead of ESPA calls

* adding tests to add_scene_to_order_list new method

* add login and token expiration check tests

* add tests for requesting available products

* add tests for requesting downloads

* update test_adapter.py to fake user login

tests/adapters/usgs/test_adapter.py

@@ -1,12 +1,12 @@
-import pytest
-from unittest import mock
 from unittest.mock import patch, PropertyMock
 import numpy as np
-
-from theia.adapters.usgs import Adapter, ErosWrapper, EspaWrapper, ImagerySearch
+import datetime
+from theia.adapters.usgs import Adapter
 from theia.api.models import ImageryRequest, JobBundle, RequestedScene
 
-import tarfile
+
+EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}]
+DEFAULT_SEARCH = {'datasetName': 'LANDSAT_OT_C2_L2'}
 
 class TestUsgsAdapter:
     def test_resolve_relative_image(self):
@@ -18,20 +18,23 @@ def test_construct_filename(self):
         bundle = JobBundle(scene_entity_id='LC08', local_path='tmp/')
         assert(Adapter().construct_filename(bundle, 'aerosol')=='LC08_sr_aerosol.tif')
 
-    @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value={})
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200)
+    @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE)
+    @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH)
     @patch('theia.adapters.usgs.ErosWrapper.search', return_value=['some scene id'])
-    @patch('theia.adapters.usgs.EspaWrapper.order_all', return_value=[{}])
+    @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}])
     @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3))
     @patch('theia.adapters.usgs.tasks.wait_for_scene.delay')
-    def test_process_request(self, mock_wait, mock_rso, mock_order_all, mock_search, mock_build):
+    def test_process_request(self, mock_wait, mock_requested_scene_creation, mock_add_scene_to_order, mock_search, mock_build, mock_avail_prods, _):
         request = ImageryRequest()
         Adapter().process_request(request)
 
         mock_build.assert_called_once_with(request)
-        mock_search.assert_called_once_with({})
-        mock_order_all.assert_called_once_with('some scene id', 'sr')
-        mock_rso.assert_called_once()
-        mock_wait.assert_called_once_with(3)
+        mock_search.assert_called_once_with(DEFAULT_SEARCH)
+        mock_add_scene_to_order.assert_called_once_with('some scene id', DEFAULT_SEARCH)
+        mock_avail_prods.assert_called_once_with('some scene id', DEFAULT_SEARCH)
+        mock_requested_scene_creation.assert_called_once()
+        mock_wait.assert_called_once_with(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE)
 
     @patch('os.path.isfile', return_value=False)
     @patch('platform.uname_result.node', new_callable=PropertyMock, return_value='testhostname')
@@ -57,16 +60,18 @@ def test_remap_pixel(self):
         assert(remap.tolist()==[0, 0, 125, 250, 255])
         assert(remap.dtype==np.uint8)
 
-    @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value={})
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200)
+    @patch('theia.adapters.usgs.ErosWrapper.available_products', return_value=EROS_AVAILABLE_PRODUCTS_EXAMPLE)
+    @patch('theia.adapters.usgs.ImagerySearch.build_search', return_value=DEFAULT_SEARCH)
     @patch('theia.adapters.usgs.ErosWrapper.search', return_value=[1, 2, 3, 4, 5])
-    @patch('theia.adapters.usgs.EspaWrapper.order_all', return_value=[{}])
     @patch('theia.api.models.RequestedScene.objects.create', return_value=RequestedScene(id=3))
     @patch('theia.adapters.usgs.tasks.wait_for_scene.delay')
-    def test_limit_scenes(self, mock_wait, mock_rso, mock_order_all, mock_search, mock_build):
+    @patch('theia.adapters.usgs.ErosWrapper.add_scenes_to_order_list', return_value=[{}])
+    def test_limit_scenes(self, mock_add_scenes_to_order, *_):
         request = ImageryRequest(max_results=3)
         Adapter().process_request(request)
 
-        assert(mock_order_all.call_count==3)
+        assert(mock_add_scenes_to_order.call_count==3)
 
     @patch('theia.adapters.usgs.XmlHelper.get_tree', autospec=True)
     @patch('theia.adapters.usgs.XmlHelper.resolve')

tests/adapters/usgs/test_eros_wrapper.py

@@ -1,24 +1,70 @@
 from theia.adapters.usgs import ErosWrapper
-
-from unittest.mock import patch, PropertyMock, call
+from unittest.mock import patch, call
+import datetime
+THREE_HOURS_AGO = datetime.datetime.utcnow() - datetime.timedelta(hours=3)
+FAKE_SEARCH = {'datasetName' : 'LANDSAT_BAND'}
 
 class TestErosWrapper:
+    def test_is_token_expired_no_login_time(self):
+        is_expired = ErosWrapper().is_token_expired()
+        assert is_expired == True
+
+    def test_is_token_expired_login_time_past_expiry(self):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = THREE_HOURS_AGO
+        assert eros_wrapper.is_token_expired() == True
+
+    def test_is_token_expired_login_time_not_expired(self):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        assert eros_wrapper.is_token_expired() == False
+
+    @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman')
+    @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd')
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200)
+    def test_login_no_login_time(self, mock_send_request, *_):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login()
+        mock_send_request.assert_has_calls([
+            call(
+                'https://m2m.cr.usgs.gov/api/api/json/stable/login',
+                 {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'}
+                 )
+        ])
+
+    @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman')
+    @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd')
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200)
+    def test_login_token_expired(self, mock_send_request, *_):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login()
+        mock_send_request.assert_has_calls([
+            call(
+                'https://m2m.cr.usgs.gov/api/api/json/stable/login',
+                 {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'}
+                 )
+        ])
+
+    @patch('theia.adapters.usgs.ErosWrapper.send_request')
+    def test_login_not_called_when_token_not_expired(self, mock_send_request):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        eros_wrapper.login()
+        mock_send_request.assert_not_called()
 
     @patch('theia.adapters.usgs.Utils.get_username', return_value='Richard Feynman')
     @patch('theia.adapters.usgs.Utils.get_password', return_value='feynmansSuperSecurePassowrd')
     @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value={'results': ['LC01_FAKESCENE_007']})
-    def test_send_search_request(self, mockRequestSend, mockPassword, mockUsername):
-        fakeSearch = {'datasetName' : 'LANDSAT_BAND'}
-        ErosWrapper().search(search=fakeSearch)
+    def test_send_search_request(self, mockRequestSend, *_):
+        ErosWrapper().search(search=FAKE_SEARCH)
         mockRequestSend.assert_has_calls([
             call(
                 'https://m2m.cr.usgs.gov/api/api/json/stable/login',
                  {'username': 'Richard Feynman', 'password': 'feynmansSuperSecurePassowrd'}
                  ),
             call(
                 'https://m2m.cr.usgs.gov/api/api/json/stable/scene-search',
-                {'datasetName': 'LANDSAT_BAND'},
-                {'results': ['LC01_FAKESCENE_007']}
+                {'datasetName': 'LANDSAT_BAND'}
             )
         ],
         any_order=False)
@@ -54,3 +100,99 @@ def test_parse_result_set(self):
         )
         assert result == ['a']
 
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200)
+    def test_add_scene_to_order_list(self, mock_send_request):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        eros_wrapper.add_scenes_to_order_list(scene_id=1, search=FAKE_SEARCH)
+        mock_send_request.assert_has_calls([
+            call(
+                'https://m2m.cr.usgs.gov/api/api/json/stable/scene-list-add',
+                 {'listId': 1, 'idField': 'displayId', 'entityId': 1, 'datasetName': 'LANDSAT_BAND'}
+            )
+        ],
+        any_order=False)
+
+    @patch('theia.adapters.usgs.ErosWrapper.login')
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=200)
+    def test_add_scene_to_order_list_logs_in(self, _, mock_login):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.add_scenes_to_order_list(scene_id=1, search=FAKE_SEARCH)
+        mock_login.assert_called_once()
+
+    @patch('theia.adapters.usgs.ErosWrapper.login')
+    @patch('theia.adapters.usgs.ErosWrapper.send_request')
+    def test_available_products_logs_in(self, _, mock_login):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH)
+        mock_login.assert_called_once()
+
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': True, 'downloadSystem': 'dds', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}])
+    def test_available_products(self, mock_send_request):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH)
+        mock_send_request.assert_has_calls([
+            call(
+                'https://m2m.cr.usgs.gov/api/api/json/stable/download-options',
+                 {'listId': 1, 'datasetName': 'LANDSAT_BAND'}
+            )
+        ],
+        any_order=False)
+        assert len(avail_products) == 1
+        assert avail_products[0] == {
+            'entityId': 'LC01_FAKESCENE_007',
+            'productId': 1,
+            'displayId': 'LC01FAKESCENE007'
+        }
+
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[])
+    def test_available_products_has_no_results(self, _):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH)
+        assert avail_products == []
+
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': False, 'downloadSystem': 'dds', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}])
+    def test_avail_products_result_not_available(self, _):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH)
+        assert avail_products == []
+
+    @patch('theia.adapters.usgs.ErosWrapper.send_request', return_value=[{'bulkAvailable': True, 'downloadSystem': 'folder', 'entityId': 'LC01_FAKESCENE_007', 'displayId': 'LC01FAKESCENE007', 'id': 1}])
+    def test_avail_products_result_download_system_incorrect(self, _):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        avail_products = eros_wrapper.available_products(list_id=1, search=FAKE_SEARCH)
+        assert avail_products == []
+
+    @patch('theia.adapters.usgs.ErosWrapper.login')
+    @patch('theia.adapters.usgs.ErosWrapper.send_request')
+    def test_request_download_logs_in(self, _, mock_login):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.request_download([{
+            'entityId': 'LC01_FAKESCENE_007',
+            'productId': 1,
+            'displayId': 'LC01FAKESCENE007'
+        }])
+        mock_login.assert_called_once()
+
+    @patch('theia.adapters.usgs.ErosWrapper.send_request')
+    def test_request_download(self, mock_send_request):
+        eros_wrapper = ErosWrapper()
+        eros_wrapper.login_time = datetime.datetime.utcnow()
+        avail_products = [{
+            'entityId': 'LC01_FAKESCENE_007',
+            'productId': 1,
+            'displayId': 'LC01FAKESCENE007'
+        }]
+        eros_wrapper.request_download(avail_products)
+
+        mock_send_request.assert_has_calls([
+            call(
+                'https://m2m.cr.usgs.gov/api/api/json/stable/download-request',
+                 {'downloads': avail_products, 'label': eros_wrapper.download_request_label}
+            )
+        ]
+       )

tests/adapters/usgs/test_usgs_tasks.py

@@ -1,39 +1,59 @@
-import pytest
 from unittest.mock import patch
 import theia.adapters.usgs.tasks
 
+EROS_AVAILABLE_PRODUCTS_EXAMPLE = [{'entityId': 'LANDSATIMAGE123', 'productId': 123, 'displayId': 'LANDSAT_IMAGE_123'}]
+EROS_REQUEST_DOWNLOAD_PENDING_DOWNLOADS_RESULT = {
+    "availableDownloads":[],
+    "duplicateProducts":[],
+    "preparingDownloads":[
+        {
+            "downloadId":550754832,
+            "eulaCode":"None",
+            "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzU0ODMyLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=="
+        }
+    ]
+}
+
+EROS_REQUEST_DOWNLOAD_DOWNLOADS_AVAILABLE_RESULT = {
+    "availableDownloads":[{
+            "downloadId":550752861,
+            "eulaCode":"None",
+            "url":"https://dds.cr.usgs.gov/download-staging/eyJpZCI6NTUwNzUyODYxLCJjb250YWN0SWQiOjI2MzY4NDQ1fQ=="
+        }],
+    "duplicateProducts":[],
+    "preparingDownloads":[]
+}
+
 @patch('theia.api.models.RequestedScene.objects.get', autospec=True)
-@patch('theia.adapters.usgs.EspaWrapper.order_status', autospec=True)
+@patch('theia.adapters.usgs.ErosWrapper.request_download', autospec=True)
 def test_wait_for_scene_already_done(mockStatus, mockGetScene):
     mockGetScene.return_value.status=1
-    theia.adapters.usgs.tasks.wait_for_scene(1)
+    theia.adapters.usgs.tasks.wait_for_scene(1, EROS_AVAILABLE_PRODUCTS_EXAMPLE)
     mockStatus.assert_not_called()
 
 @patch('theia.api.models.RequestedScene.objects.get', autospec=True)
-@patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='whatever')
-@patch('theia.adapters.usgs.EspaWrapper.download_urls', autospec=True)
+@patch('theia.adapters.usgs.ErosWrapper.request_download', return_value=EROS_REQUEST_DOWNLOAD_PENDING_DOWNLOADS_RESULT)
 @patch('theia.adapters.usgs.tasks.wait_for_scene.apply_async', autospec=True)
-def test_wait_for_scene_pending(mockWait, mockUrls, mockStatus, mockGetScene):
+def test_wait_for_scene_pending(mockWait, mockStatus, mockGetScene):
     mockGetScene.return_value.status=0
     mockGetScene.return_value.scene_order_id='order id'
 
-    theia.adapters.usgs.tasks.wait_for_scene(3)
+    theia.adapters.usgs.tasks.wait_for_scene(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE)
 
     mockGetScene.assert_called_once_with(pk=3)
-    mockStatus.assert_called_once_with('order id')
+    mockStatus.assert_called_once_with(EROS_AVAILABLE_PRODUCTS_EXAMPLE)
     mockGetScene.return_value.save.assert_called_once()
     mockWait.assert_called_once()
 
 @patch('theia.api.models.RequestedScene.objects.get', autospec=True)
-@patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='complete')
-@patch('theia.adapters.usgs.EspaWrapper.download_urls')
+@patch('theia.adapters.usgs.ErosWrapper.request_download', return_value=EROS_REQUEST_DOWNLOAD_DOWNLOADS_AVAILABLE_RESULT)
 @patch('theia.api.models.JobBundleManager.from_requested_scene', autospec=True)
-def test_wait_for_scene_ready(mockConstruct, mockUrls, mockStatus, mockGetScene):
+def test_wait_for_scene_ready(mockConstruct, mockStatus, mockGetScene):
     mockGetScene.return_value.status=0
     mockGetScene.return_value.scene_order_id='order id'
-    theia.adapters.usgs.tasks.wait_for_scene(3)
+    theia.adapters.usgs.tasks.wait_for_scene(3, EROS_AVAILABLE_PRODUCTS_EXAMPLE)
 
     mockGetScene.assert_called_once_with(pk=3)
-    mockStatus.assert_called_once_with('order id')
+    mockStatus.assert_called_once_with(EROS_AVAILABLE_PRODUCTS_EXAMPLE)
     mockGetScene.return_value.save.assert_called()
-    mockConstruct.assert_called_once()
+    mockConstruct.assert_called_once()

theia/adapters/usgs/adapter.py

@@ -1,14 +1,12 @@
 import os.path
 import platform
-import re
 import urllib.request
 import numpy as np
 
 from theia.api import models
 from theia.utils import FileUtils
 
 from .eros_wrapper import ErosWrapper
-from .espa_wrapper import EspaWrapper
 from .imagery_search import ImagerySearch
 from .tasks import wait_for_scene
 from .xml_helper import XmlHelper
@@ -69,15 +67,25 @@ def __init__(self):
 
     def process_request(self, imagery_request):
         search = ImagerySearch.build_search(imagery_request)
-        scenes = ErosWrapper().search(search)
+        eros_wrapper = ErosWrapper()
+        scenes = eros_wrapper.search(search)
+
         if imagery_request.max_results:
             scenes = scenes[0:imagery_request.max_results]
 
         for scene in scenes:
-            result = EspaWrapper.order_all(scene, 'sr')
-            for item in result:
-                req = models.RequestedScene.objects.create(**{**item, **{'imagery_request': imagery_request}})
-                wait_for_scene.delay(req.id)
+            eros_wrapper.add_scenes_to_order_list(scene, search)
+            available_products = eros_wrapper.available_products(scene, search)
+
+            eros_wrapper.request_download(available_products)
+
+            for item in available_products:
+                req = models.RequestedScene.objects.create(
+                    scene_entity_id = item['displayId'],
+                    scene_order_id = item['productId'],
+                    **{'imagery_request': imagery_request}
+                )
+                wait_for_scene.delay(req.id, available_products)
 
     def construct_filename(self, bundle, suffix):
         product = "sr"
@@ -104,7 +112,6 @@ def retrieve(self, job_bundle):
             # get the compressed scene data if we don't have it
             if not os.path.isfile(zip_path):
                 urllib.request.urlretrieve(job_bundle.requested_scene.scene_url, zip_path)
-
             FileUtils.untar(zip_path, job_bundle.local_path)
 
     def default_extension(self):