From 458e5281370da5795cf927ab6ae770a7e3698d25 Mon Sep 17 00:00:00 2001 From: Sherwin-14 Date: Mon, 8 Jul 2024 12:19:33 +0530 Subject: [PATCH 1/2] Added Fixtures to conftest.py --- tests/integration/conftest.py | 62 +++++++++++++++++++++++ tests/integration/test_cloud_download.py | 52 +++---------------- tests/integration/test_cloud_open.py | 54 +++----------------- tests/integration/test_kerchunk.py | 22 -------- tests/integration/test_onprem_download.py | 52 +++---------------- tests/integration/test_onprem_open.py | 52 +++---------------- 6 files changed, 87 insertions(+), 207 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8be3de59..edd93a7b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,4 +1,11 @@ +import logging +import os +import random +import unittest + +import earthaccess import pytest +from earthaccess import Auth, Store ACCEPTABLE_FAILURE_RATE = 10 @@ -10,3 +17,58 @@ def pytest_sessionfinish(session, exitstatus): failure_rate = (100.0 * session.testsfailed) / session.testscollected if failure_rate <= ACCEPTABLE_FAILURE_RATE: session.exitstatus = 0 + + +@pytest.fixture(scope="module") +def authenticated_store(): + logger = logging.getLogger(__name__) + assertions = unittest.TestCase("__init__") + + # we need to use a valid EDL credential + + assert "EARTHDATA_USERNAME" in os.environ is True + assert "EARTHDATA_PASSWORD" in os.environ is True + + auth = Auth().login(strategy="environment") + assert auth.authenticated is True + logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") + logger.info(f"earthaccess version: {earthaccess.__version__}") + + store = Store(auth) + + return store, logger, assertions + + +@pytest.fixture() +def get_sample_granules(granules, sample_size, max_granule_size): + """Returns a list with sample granules and their size in MB if + the total size is less than the max_granule_size. + """ + files_to_download = [] + total_size = 0 + max_tries = sample_size * 2 + tries = 0 + + while tries <= max_tries: + g = random.sample(granules, 1)[0] + if g.size() > max_granule_size: + # print(f"G: {g['meta']['concept-id']} exceded max size: {g.size()}") + tries += 1 + continue + else: + # print(f"Adding : {g['meta']['concept-id']} size: {g.size()}") + files_to_download.append(g) + total_size += g.size() + if len(files_to_download) >= sample_size: + break + return files_to_download, round(total_size) + + +@pytest.fixture() +def granules(): + granules = earthaccess.search_data( + count=2, + short_name="SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL2205", + cloud_hosted=True, + ) + return granules diff --git a/tests/integration/test_cloud_download.py b/tests/integration/test_cloud_download.py index 4e8f9519..4b61fcff 100644 --- a/tests/integration/test_cloud_download.py +++ b/tests/integration/test_cloud_download.py @@ -1,17 +1,11 @@ # package imports -import logging -import os import random import shutil -import unittest from pathlib import Path import earthaccess import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store - -logger = logging.getLogger(__name__) - +from earthaccess import DataCollections, DataGranules daac_list = [ { @@ -56,47 +50,11 @@ }, ] -assertions = unittest.TestCase("__init__") - -# we need to use a valid EDL credential - -assertions.assertTrue("EARTHDATA_USERNAME" in os.environ) -assertions.assertTrue("EARTHDATA_PASSWORD" in os.environ) - -auth = Auth().login(strategy="environment") -assertions.assertTrue(auth.authenticated) -logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") -logger.info(f"earthaccess version: {earthaccess.__version__}") - -store = Store(auth) - - -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - # print(f"G: {g['meta']['concept-id']} exceded max size: {g.size()}") - tries += 1 - continue - else: - # print(f"Adding : {g['meta']['concept-id']} size: {g.size()}") - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size) - @pytest.mark.parametrize("daac", daac_list) -def test_earthaccess_can_download_cloud_collection_granules(daac): +def test_earthaccess_can_download_cloud_collection_granules( + authenticated_store, get_sample_granules, daac +): """Tests that we can download cloud collections using HTTPS links.""" daac_shortname = daac["short_name"] collections_count = daac["collections_count"] @@ -105,6 +63,8 @@ def test_earthaccess_can_download_cloud_collection_granules(daac): granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] + store, logger, assertions = authenticated_store + collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(True) hits = collection_query.hits() logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") diff --git a/tests/integration/test_cloud_open.py b/tests/integration/test_cloud_open.py index b69eba15..542b915b 100644 --- a/tests/integration/test_cloud_open.py +++ b/tests/integration/test_cloud_open.py @@ -1,16 +1,10 @@ # package imports -import logging -import os import random -import unittest import earthaccess import magic import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store - -logger = logging.getLogger(__name__) - +from earthaccess import DataCollections, DataGranules daacs_list = [ { @@ -55,44 +49,6 @@ }, ] -assertions = unittest.TestCase("__init__") - -# we need to use a valid EDL credential - -assertions.assertTrue("EARTHDATA_USERNAME" in os.environ) -assertions.assertTrue("EARTHDATA_PASSWORD" in os.environ) - -auth = Auth().login(strategy="environment") -assertions.assertTrue(auth.authenticated) -logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") -logger.info(f"earthaccess version: {earthaccess.__version__}") - -store = Store(auth) - - -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - # print(f"G: {g['meta']['concept-id']} exceded max size: {g.size()}") - tries += 1 - continue - else: - # print(f"Adding : {g['meta']['concept-id']} size: {g.size()}") - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size, 2) - def supported_collection(data_links): for url in data_links: @@ -102,7 +58,9 @@ def supported_collection(data_links): @pytest.mark.parametrize("daac", daacs_list) -def test_earthaccess_can_open_onprem_collection_granules(daac): +def test_earthaccess_can_open_onprem_collection_granules( + authenticated_store, get_sample_granules, daac +): """Tests that we can download cloud collections using HTTPS links.""" daac_shortname = daac["short_name"] collections_count = daac["collections_count"] @@ -111,6 +69,8 @@ def test_earthaccess_can_open_onprem_collection_granules(daac): granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] + store, logger, assertions = authenticated_store + collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(True) hits = collection_query.hits() logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") @@ -156,7 +116,7 @@ def test_earthaccess_can_open_onprem_collection_granules(daac): logger.warning(f"File could not be open: {file}") -def test_multi_file_granule(): +def test_multi_file_granule(authenticated_store): # Ensure granules that contain multiple files are handled correctly granules = earthaccess.search_data(short_name="HLSL30", count=1) assert len(granules) == 1 diff --git a/tests/integration/test_kerchunk.py b/tests/integration/test_kerchunk.py index 2e981cce..8c4abada 100644 --- a/tests/integration/test_kerchunk.py +++ b/tests/integration/test_kerchunk.py @@ -1,6 +1,3 @@ -import logging -import os -import unittest from pathlib import Path import earthaccess @@ -10,25 +7,6 @@ kerchunk = pytest.importorskip("kerchunk") pytest.importorskip("dask") -logger = logging.getLogger(__name__) -assertions = unittest.TestCase("__init__") - -assertions.assertTrue("EARTHDATA_USERNAME" in os.environ) -assertions.assertTrue("EARTHDATA_PASSWORD" in os.environ) - -logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") -logger.info(f"earthaccess version: {earthaccess.__version__}") - - -@pytest.fixture(scope="module") -def granules(): - granules = earthaccess.search_data( - count=2, - short_name="SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL2205", - cloud_hosted=True, - ) - return granules - @pytest.mark.parametrize("protocol", ["", "file://"]) def test_consolidate_metadata_outfile(tmp_path, granules, protocol): diff --git a/tests/integration/test_onprem_download.py b/tests/integration/test_onprem_download.py index 242a3c26..7c4c6157 100644 --- a/tests/integration/test_onprem_download.py +++ b/tests/integration/test_onprem_download.py @@ -1,17 +1,11 @@ # package imports -import logging -import os import random import shutil -import unittest from pathlib import Path import earthaccess import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store - -logger = logging.getLogger(__name__) - +from earthaccess import DataCollections, DataGranules daacs_list = [ { @@ -48,44 +42,6 @@ }, ] -assertions = unittest.TestCase("__init__") - -# we need to use a valid EDL credential - -assertions.assertTrue("EARTHDATA_USERNAME" in os.environ) -assertions.assertTrue("EARTHDATA_PASSWORD" in os.environ) - -auth = Auth().login(strategy="environment") -assertions.assertTrue(auth.authenticated) -logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") -logger.info(f"earthaccess version: {earthaccess.__version__}") - -store = Store(auth) - - -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - # print(f"G: {g['meta']['concept-id']} exceded max size: {g.size()}") - tries += 1 - continue - else: - # print(f"Adding : {g['meta']['concept-id']} size: {g.size()}") - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size, 2) - def supported_collection(data_links): for url in data_links: @@ -95,7 +51,9 @@ def supported_collection(data_links): @pytest.mark.parametrize("daac", daacs_list) -def test_earthaccess_can_download_onprem_collection_granules(daac): +def test_earthaccess_can_download_onprem_collection_granules( + authenticated_store, get_sample_granules, daac +): """Tests that we can download cloud collections using HTTPS links.""" daac_shortname = daac["short_name"] collections_count = daac["collections_count"] @@ -104,6 +62,8 @@ def test_earthaccess_can_download_onprem_collection_granules(daac): granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] + store, logger, assertions = authenticated_store + collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(False) hits = collection_query.hits() logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") diff --git a/tests/integration/test_onprem_open.py b/tests/integration/test_onprem_open.py index 2a455c44..9e84870e 100644 --- a/tests/integration/test_onprem_open.py +++ b/tests/integration/test_onprem_open.py @@ -1,16 +1,10 @@ # package imports -import logging -import os import random -import unittest import earthaccess import magic import pytest -from earthaccess import Auth, DataCollections, DataGranules, Store - -logger = logging.getLogger(__name__) - +from earthaccess import DataCollections, DataGranules daacs_list = [ { @@ -47,44 +41,6 @@ }, ] -assertions = unittest.TestCase("__init__") - -# we need to use a valid EDL credential - -assertions.assertTrue("EARTHDATA_USERNAME" in os.environ) -assertions.assertTrue("EARTHDATA_PASSWORD" in os.environ) - -auth = Auth().login(strategy="environment") -assertions.assertTrue(auth.authenticated) -logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") -logger.info(f"earthaccess version: {earthaccess.__version__}") - -store = Store(auth) - - -def get_sample_granules(granules, sample_size, max_granule_size): - """Returns a list with sample granules and their size in MB if - the total size is less than the max_granule_size. - """ - files_to_download = [] - total_size = 0 - max_tries = sample_size * 2 - tries = 0 - - while tries <= max_tries: - g = random.sample(granules, 1)[0] - if g.size() > max_granule_size: - # print(f"G: {g['meta']['concept-id']} exceded max size: {g.size()}") - tries += 1 - continue - else: - # print(f"Adding : {g['meta']['concept-id']} size: {g.size()}") - files_to_download.append(g) - total_size += g.size() - if len(files_to_download) >= sample_size: - break - return files_to_download, round(total_size, 2) - def supported_collection(data_links): for url in data_links: @@ -94,7 +50,9 @@ def supported_collection(data_links): @pytest.mark.parametrize("daac", daacs_list) -def test_earthaccess_can_open_onprem_collection_granules(daac): +def test_earthaccess_can_open_onprem_collection_granules( + authenticated_store, get_sample_granules, daac +): """Tests that we can download cloud collections using HTTPS links.""" daac_shortname = daac["short_name"] collections_count = daac["collections_count"] @@ -103,6 +61,8 @@ def test_earthaccess_can_open_onprem_collection_granules(daac): granules_sample_size = daac["granules_sample_size"] granules_max_size = daac["granules_max_size_mb"] + store, logger, assertions = authenticated_store + collection_query = DataCollections().data_center(daac_shortname).cloud_hosted(False) hits = collection_query.hits() logger.info(f"Cloud hosted collections for {daac_shortname}: {hits}") From 2f7d6a5f59e7d1c2e91639fef68b19fdbd8181dd Mon Sep 17 00:00:00 2001 From: Sherwin-14 Date: Mon, 8 Jul 2024 15:24:00 +0530 Subject: [PATCH 2/2] Added Fixtures to conftest.py --- tests/integration/conftest.py | 4 ++-- tests/integration/test_api.py | 10 +++++----- tests/integration/test_auth.py | 2 +- tests/integration/test_cloud_open.py | 2 +- tests/integration/test_onprem_download.py | 6 +++--- tests/integration/test_onprem_open.py | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index edd93a7b..aaf6bf8c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -26,8 +26,8 @@ def authenticated_store(): # we need to use a valid EDL credential - assert "EARTHDATA_USERNAME" in os.environ is True - assert "EARTHDATA_PASSWORD" in os.environ is True + assert "EARTHDATA_USERNAME" in os.environ + assert "EARTHDATA_PASSWORD" in os.environ auth = Auth().login(strategy="environment") assert auth.authenticated is True diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 8fd45489..5264bf50 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -11,8 +11,8 @@ assertions = unittest.TestCase("__init__") -assertions.assertTrue("EARTHDATA_USERNAME" in os.environ) -assertions.assertTrue("EARTHDATA_PASSWORD" in os.environ) +assert "EARTHDATA_USERNAME" in os.environ +assert "EARTHDATA_PASSWORD" in os.environ logger.info(f"Current username: {os.environ['EARTHDATA_USERNAME']}") logger.info(f"earthaccess version: {earthaccess.__version__}") @@ -46,13 +46,13 @@ def test_auth_returns_valid_auth_class(): auth = earthaccess.login(strategy="environment") assertions.assertIsInstance(auth, earthaccess.Auth) assertions.assertIsInstance(earthaccess.__auth__, earthaccess.Auth) - assertions.assertTrue(earthaccess.__auth__.authenticated) + assert earthaccess.__auth__.authenticated is True def test_dataset_search_returns_none_with_no_parameters(): results = earthaccess.search_datasets() assertions.assertIsInstance(results, list) - assertions.assertTrue(len(results) == 0) + assert (len(results) == 0) is True @pytest.mark.parametrize("kwargs", dataset_valid_params) @@ -66,7 +66,7 @@ def test_dataset_search_returns_valid_results(kwargs): def test_granules_search_returns_valid_results(kwargs): results = earthaccess.search_data(count=10, **kwargs) assertions.assertIsInstance(results, list) - assertions.assertTrue(len(results) <= 10) + assert (len(results) <= 10) is True @pytest.mark.parametrize("selection", [0, slice(None)]) diff --git a/tests/integration/test_auth.py b/tests/integration/test_auth.py index aef60299..c02a228f 100644 --- a/tests/integration/test_auth.py +++ b/tests/integration/test_auth.py @@ -82,7 +82,7 @@ def test_auth_can_create_authenticated_requests_sessions(): def test_auth_can_fetch_s3_credentials(): activate_environment() auth = earthaccess.login(strategy="environment") - assertions.assertTrue(auth.authenticated) + assert auth.authenticated is True for daac in earthaccess.daac.DAACS: if len(daac["s3-credentials"]) > 0: try: diff --git a/tests/integration/test_cloud_open.py b/tests/integration/test_cloud_open.py index 542b915b..001ddfc5 100644 --- a/tests/integration/test_cloud_open.py +++ b/tests/integration/test_cloud_open.py @@ -106,7 +106,7 @@ def test_earthaccess_can_open_onprem_collection_granules( # We are testing this method fileset = store.open(granules_to_open) - assertions.assertTrue(isinstance(fileset, list)) + assert isinstance(fileset, list) is True # we test that we can read some bytes and get the file type for file in fileset: diff --git a/tests/integration/test_onprem_download.py b/tests/integration/test_onprem_download.py index 7c4c6157..3fead82f 100644 --- a/tests/integration/test_onprem_download.py +++ b/tests/integration/test_onprem_download.py @@ -99,11 +99,11 @@ def test_earthaccess_can_download_onprem_collection_granules( # We are testing this method downloaded_results = store.get(granules_to_download, local_path=local_path) - assertions.assertTrue(isinstance(downloaded_results, list)) - assertions.assertTrue(len(downloaded_results) == granules_sample_size) + assert isinstance(downloaded_results, list) is True + assert (len(downloaded_results) == granules_sample_size) is True path = Path(local_path) - assertions.assertTrue(path.is_dir()) + assert path.is_dir() is True # test that we downloaded the mb reported by CMR total_mb_downloaded = round( (sum(file.stat().st_size for file in path.rglob("*")) / 1024**2), 2 diff --git a/tests/integration/test_onprem_open.py b/tests/integration/test_onprem_open.py index 9e84870e..e1621266 100644 --- a/tests/integration/test_onprem_open.py +++ b/tests/integration/test_onprem_open.py @@ -77,7 +77,7 @@ def test_earthaccess_can_open_onprem_collection_granules( total_granules = granule_query.hits() granules = granule_query.get(granules_count) assertions.assertTrue(len(granules) > 0, "Could not fetch granules") - assertions.assertTrue(isinstance(granules[0], earthaccess.results.DataGranule)) + assert isinstance(granules[0], earthaccess.results.DataGranule) is True data_links = granules[0].data_links() if not supported_collection(data_links): logger.warning(f"PODAAC DRIVE is not supported at the moment: {data_links}") @@ -98,7 +98,7 @@ def test_earthaccess_can_open_onprem_collection_granules( # We are testing this method fileset = store.open(granules_to_open) - assertions.assertTrue(isinstance(fileset, list)) + assert isinstance(fileset, list) is True # we test that we can read some bytes and get the file type for file in fileset: