Skip to content

Commit

Permalink
More merge conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
doug-newman-nasa committed Apr 8, 2024
1 parent 5cabd1e commit ecb6484
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 57 deletions.
52 changes: 4 additions & 48 deletions earthaccess/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,8 @@
from typing import Any, List, Optional, Tuple, Type, Union

import dateutil.parser as parser # type: ignore
from cmr import CollectionQuery, GranuleQuery
<<<<<<< HEAD
import requests
=======
from requests import exceptions, get, session
>>>>>>> 5ba65ca (Add VCR tests.)
from cmr import CollectionQuery, GranuleQuery

# type: ignore
from .auth import Auth
Expand All @@ -20,7 +16,6 @@ def get_results(request, limit: int = 2000) -> list: # type: ignore
"""
Get all results up to some limit, even if spanning multiple pages.
<<<<<<< HEAD
???+ Tip
The default page size is 2000, if the supplied value is greater then the Search-After header
will be used to iterate across multiple requests until either the limit has been reached
Expand All @@ -40,36 +35,16 @@ def get_results(request, limit: int = 2000) -> list: # type: ignore
while more_results:
# Only get what we need
page_size = min(limit - len(results), page_size)
response = requests.get(url, headers=request.headers, params={"page_size": page_size})
response = requests.get(
url, headers=request.headers, params={"page_size": page_size}
)
if request.headers is None:
request.headers = {}
request.headers["cmr-search-after"] = response.headers["cmr-search-after"]

try:
response.raise_for_status()
except requests.exceptions.HTTPError as ex:
=======
:limit: The number of results to return
:returns: query results as a list
"""
page_size = min(limit, 2000)
url = request._build_url()
results: List = []
more_results = True
while more_results:
# Only get what we need
page_size = min(limit - len(results), page_size)
response = get(url, headers=request.headers, params={"page_size": page_size})
if request.headers is None:
request.headers = {}
request.headers["cmr-search-after"] = response.headers["cmr-search-after"]
try:
response.raise_for_status()
except exceptions.HTTPError as ex:
>>>>>>> 5ba65ca (Add VCR tests.)
raise RuntimeError(ex.response.text)

latest = response.json()["items"]
Expand Down Expand Up @@ -145,15 +120,6 @@ def get(self, limit: int = 2000) -> list:
Returns:
query results as a list of `DataCollection` instances.
"""
<<<<<<< HEAD
=======
response = get_results(self, limit)
results = list(
DataCollection(collection, self._fields) for collection in response
)
>>>>>>> 5ba65ca (Add VCR tests.)

return list(
DataCollection(collection, self._fields)
Expand Down Expand Up @@ -436,16 +402,6 @@ def get(self, limit: int = 2000) -> list:
query results as a list of `DataGranules` instances.
"""
response = get_results(self, limit)
<<<<<<< HEAD
=======
if self._is_cloud_hosted(response[0]):
cloud = True
else:
cloud = False
results = list(DataGranule(granule, cloud_hosted=cloud) for granule in response)
>>>>>>> 5ba65ca (Add VCR tests.)

cloud = self._is_cloud_hosted(response[0])

Expand Down
24 changes: 15 additions & 9 deletions tests/unit/test_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@ def test_data_links(self):
# When `access` and `in_region` are both specified, `access` takes priority
assert g.data_links(access="direct", in_region=True)[0].startswith("s3://")
assert g.data_links(access="direct", in_region=False)[0].startswith("s3://")
assert g.data_links(access="external", in_region=True)[0].startswith(
"https://"
)
assert g.data_links(access="external", in_region=True)[0].startswith("https://")
assert g.data_links(access="external", in_region=False)[0].startswith(
"https://"
)
Expand All @@ -54,7 +52,8 @@ def test_get_more_than_2000(self):
to not fetch back more results than we ask for
"""
with my_vcr.use_cassette(
"tests/unit/fixtures/vcr_cassettes/MOD02QKM.yaml", filter_headers=['authorization', 'Set-Cookie']
"tests/unit/fixtures/vcr_cassettes/MOD02QKM.yaml",
filter_headers=["authorization", "Set-Cookie"],
) as cass:
granules = earthaccess.search_data(short_name="MOD02QKM", count=3000)

Expand All @@ -72,7 +71,8 @@ def test_get(self):
in a single request
"""
with my_vcr.use_cassette(
"tests/unit/fixtures/vcr_cassettes/MOD02QKM_2000.yaml", filter_headers=['authorization', 'Set-Cookie']
"tests/unit/fixtures/vcr_cassettes/MOD02QKM_2000.yaml",
filter_headers=["authorization", "Set-Cookie"],
) as cass:
granules = earthaccess.search_data(short_name="MOD02QKM", count=2000)

Expand All @@ -90,7 +90,8 @@ def test_get_all_less_than_2k(self):
to not fetch back more results than we ask for
"""
with my_vcr.use_cassette(
"tests/unit/fixtures/vcr_cassettes/TELLUS_GRAC.yaml", filter_headers=['authorization', 'Set-Cookie']
"tests/unit/fixtures/vcr_cassettes/TELLUS_GRAC.yaml",
filter_headers=["authorization", "Set-Cookie"],
) as cass:
granules = earthaccess.search_data(
short_name="TELLUS_GRAC_L3_JPL_RL06_LND_v04", count=2000
Expand All @@ -110,7 +111,8 @@ def test_get_all_more_than_2k(self):
to not fetch back more results than we ask for
"""
with my_vcr.use_cassette(
"tests/unit/fixtures/vcr_cassettes/CYGNSS.yaml", filter_headers=['authorization', 'Set-Cookie']
"tests/unit/fixtures/vcr_cassettes/CYGNSS.yaml",
filter_headers=["authorization", "Set-Cookie"],
) as cass:
granules = earthaccess.search_data(
short_name="CYGNSS_NOAA_L2_SWSP_25KM_V1.2", count=3000
Expand All @@ -130,7 +132,8 @@ def test_collections_less_than_2k(self):
to not fetch back more results than we ask for
"""
with my_vcr.use_cassette(
"tests/unit/fixtures/vcr_cassettes/PODAAC.yaml", filter_headers=['authorization', 'Set-Cookie']
"tests/unit/fixtures/vcr_cassettes/PODAAC.yaml",
filter_headers=["authorization", "Set-Cookie"],
) as cass:
query = DataCollections().daac("PODAAC").cloud_hosted(True)
collections = query.get(20)
Expand All @@ -150,7 +153,10 @@ def test_collections_more_than_2k(self):
invocations of a cmr granule search and
to not fetch back more results than we ask for
"""
with my_vcr.use_cassette("tests/unit/fixtures/vcr_cassettes/ALL.yaml", filter_headers=['authorization', 'Set-Cookie']) as cass:
with my_vcr.use_cassette(
"tests/unit/fixtures/vcr_cassettes/ALL.yaml",
filter_headers=["authorization", "Set-Cookie"],
) as cass:
query = DataCollections()
collections = query.get(3000)

Expand Down

0 comments on commit ecb6484

Please sign in to comment.