diff --git a/.gitattributes b/.gitattributes index 01fe5009..affcecd7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ notebooks/** linguist-documentation docs/** linguist-documentation -tests/unit/fixtures/vcr_cassettes/** linguist-documentation \ No newline at end of file +tests/unit/fixtures/vcr_cassettes/** linguist-documentation diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..60c23104 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,59 @@ + + + + +
Pull Request (PR) draft checklist - click to expand + +- [ ] Please review our + [contributing documentation](https://earthaccess.readthedocs.io/en/latest/contributing/) + before getting started. +- [ ] Ensure an issue exists representing the problem being solved in this PR. +- [ ] Populate a descriptive title. For example, instead of "Updated README.md", use a + title such as "Add testing details to the contributor section of the README". +- [ ] Populate the body of the pull request with: + - A clear description of the change you are proposing. + - Links to any issues resolved by this PR with text in the PR description, for + example `closes #1`. See + [GitHub docs - Linking a pull request to an issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue). +- [ ] Update `CHANGELOG.md` with details about your change in a section titled + `## Unreleased`. If such a section does not exist, please create one. +- [ ] Update the documentation and/or the `README.md` with details of changes to the + earthaccess interface, if any. Consider new environment variables, function names, + decorators, etc. + +Click the "Ready for review" button at the bottom of the "Conversation" tab in GitHub +once these requirements are fulfilled. Don't worry if you see any test failures in +GitHub at this point! + +
+ +
Pull Request (PR) merge checklist - click to expand + +Please do your best to complete these requirements! If you need help with any of these +requirements, you can ping the `@nsidc/earthaccess-support` team in a comment and we +will help you out! + +- [ ] Add unit tests for any new features. +- [ ] Apply formatting and linting autofixes. You can add a GitHub comment in this Pull + Request containing "pre-commit.ci autofix" to automate this. +- [ ] Ensure all automated PR checks (seen at the bottom of the "conversation" tab) pass. +- [ ] Get at least one approving review. + +
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e596ad29..9506aa4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,10 +7,25 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: + # Validate format - id: check-yaml - - id: trailing-whitespace - id: check-toml - id: check-json + # Check for common mistakes + - id: check-added-large-files + - id: check-case-conflict + # - id: check-illegal-windows-names # TODO: Enable in next release + - id: check-merge-conflict + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-vcs-permalinks + - id: destroyed-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + - id: no-commit-to-branch # protects `main` by default + - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.5.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index c5ee95ac..cf364141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## Unreleased ### Changed @@ -39,6 +39,8 @@ ([#508](https://github.com/nsidc/earthaccess/issues/508)) * Create destination directory prior to direct S3 downloads, if it doesn't already exist ([#562](https://github.com/nsidc/earthaccess/issues/562)) +* Fix broken image link in sea level rise tutorial + (([#427](https://github.com/nsidc/earthaccess/issues/427)) ## [v0.9.0] 2024-02-28 diff --git a/binder/postBuild b/binder/postBuild old mode 100644 new mode 100755 index 254815af..bcf7589f --- a/binder/postBuild +++ b/binder/postBuild @@ -4,4 +4,3 @@ set -ex poetry config virtualenvs.create false poetry install --no-dev - diff --git a/docs/contributing/index.md b/docs/contributing/index.md index adabeda6..6ac83fd1 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -89,20 +89,9 @@ To set up pre-commit, follow these steps: We have included type stubs for the untyped `python-cmr` library, which we intend to eventually upstream. Since `python-cmr` exposes the `cmr` package, the stubs appear under `stubs/cmr`. -### Requirements to merge code (Pull Request Process) - -- you must include test coverage -- you must update the documentation -- you must format and lint - ## Pull Request process -1. Ensure you include test coverage for all changes -1. Ensure your code is formatted properly following this document -1. Update the documentation and the `README.md` with details of changes to the - interface, this includes new environment variables, function names,decorators, etc. -1. Update `CHANGELOG.md` with details about your change in a section titled - `Unreleased`. If one does not exist, please create one. -1. You may merge the Pull Request once you have the sign-off of another - developer, or if you do not have permission to do that, you may request the - reviewer to merge it for you. +Fork the repository using the "Fork" button on the [repository +homepage](https://github.com/nsidc/earthaccess), create a branch with your changes in the fork, then open +a draft pull request from your fork. Starting a pull request provides additional instructions and requirements, and +there is no harm in starting a draft pull request while still developing. \ No newline at end of file diff --git a/docs/contributing/maintainers-guide.md b/docs/contributing/maintainers-guide.md index 1ac2fc79..2d54a354 100644 --- a/docs/contributing/maintainers-guide.md +++ b/docs/contributing/maintainers-guide.md @@ -53,6 +53,3 @@ The GitHub Actions CI services handle the project's building, testing, and manag ## Continuous Documentation [ReadTheDocs](https://readthedocs.org/projects/earthaccess/) is used to generate and host [our documentation website](https://earthaccess.readthedocs.io/) as well as the preview for documentation changes made in pull requests. This service uses a configuration file in the root of the project, `.readthedocs.yml`. - - - diff --git a/docs/howto/access-data.md b/docs/howto/access-data.md index 22f19521..f87a87f2 100644 --- a/docs/howto/access-data.md +++ b/docs/howto/access-data.md @@ -41,5 +41,3 @@ And that's it in just one line of code! This same piece of code will also work f > More examples coming soon! - - diff --git a/docs/howto/cloud.md b/docs/howto/cloud.md index d3557f72..ebfccf03 100644 --- a/docs/howto/cloud.md +++ b/docs/howto/cloud.md @@ -1,3 +1,3 @@ # Direct S3 access for cloud-based datasets -Coming soon \ No newline at end of file +Coming soon diff --git a/docs/howto/edl.ipynb b/docs/howto/edl.ipynb index d26503dc..5949ddb5 100644 --- a/docs/howto/edl.ipynb +++ b/docs/howto/edl.ipynb @@ -17,17 +17,7 @@ "execution_count": 1, "id": "ce723d6f-8b5f-43e1-9531-19ee08a056fe", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "You're now authenticated with NASA Earthdata Login\n", - "Using token with expiration date: 09/25/2023\n", - "Using environment variables for EDL\n" - ] - } - ], + "outputs": [], "source": [ "import earthaccess\n", "\n", @@ -40,32 +30,17 @@ "metadata": {}, "source": [ "### Data in AWS\n", - "If the data we want to access is on AWS, we can use earthaccess to generate temporary S3 credentials for any of the DAACs" + "If the data we want to access is on AWS, we can use earthaccess to generate temporary S3 credentials for any of the DAACs. This line is commented out for security reasons. " ] }, { "cell_type": "code", - "execution_count": 3, - "id": "3108ff21-1ea2-449b-b2f5-b442dd3b61f5", + "execution_count": 2, + "id": "f1f45172-c9c3-4f25-ad7e-acf94fba8b1d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'accessKeyId': 'ASIA2D3OGJNTE2UAIDHR',\n", - " 'secretAccessKey': '9+6D3m76HTuNi3qvbgUMBFlJECsCgPyJ4ur2CfZL',\n", - " 'sessionToken': 'FwoGZXIvYXdzEPL//////////wEaDIVnMBd6zBaEKmG58CLpAXrPN52vpRHvs+0y960kNwGOdFHPD81WVsdeUciP/vQvsU/xX4Cg+iLH/OXDlH/PBWCmSgfk4vgXD9k6r2JRmnpkwyF4v4K4+ciIuUHNG43ZegSFtkzZoLHShLtEnNu/OIT0TlcrOLJIY+TnjEDcKkXiLSsLYB7qDKZIeg654oPCkjeXyDuoWOumJa7utBEwHmakOGdGJTthOfbfJ6RJ9sbPbmsEXg0bHF5PY9h3/Eoq0rjyRmlb1WYuopw5YK8dL5N9IwSKlKiu19Cj0nd04XVNuiB02+fLDubsnKK1b3j0jR53qAU3FxobKLHgj6YGMi0kDN7xWP6yqI7Ry69Lf5EEcDRVp4UVz6llGYoDpsKr9s3cXaBljkhEjH7N5dQ=',\n", - " 'expiration': '2023-07-28 17:43:29+00:00'}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "s3_credentials = auth.get_s3_credentials(\"NSIDC\")\n", - "s3_credentials" + "#s3_credentials = auth.get_s3_credentials(\"NSIDC\")" ] }, { @@ -88,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 3, "id": "a7bb1a93-6f85-40cd-93b4-ce821a03a8b5", "metadata": {}, "outputs": [], @@ -102,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 4, "id": "c0f4e286-8ce9-4cde-b3a2-631b361a0d51", "metadata": {}, "outputs": [ @@ -112,7 +87,7 @@ "'�HDF\\r\\n\\x1a\\n\\x00\\x00\\x00\\x00\\x00\\x08\\x08\\x00\\x04\\x00\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00���������HUn\\x00\\x00\\x00\\x00��������\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00`\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00OHDR\\x02'" ] }, - "execution_count": 15, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -133,7 +108,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "id": "1e5f5f7a-0643-42cb-b70c-4ac982824ff9", "metadata": {}, "outputs": [], @@ -143,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 6, "id": "fbc7bc5f-b984-4324-b095-3648a437fba2", "metadata": {}, "outputs": [ @@ -153,7 +128,7 @@ "b'\\x89HDF\\r\\n\\x1a\\n\\x00\\x00\\x00\\x00\\x00\\x08\\x08\\x00\\x04\\x00\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xd7HUn\\x00\\x00\\x00\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00`\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00OHDR'" ] }, - "execution_count": 11, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -166,19 +141,19 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "id": "07433a9f-f276-4d2b-9ed3-f556df2884af", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f68d214c1c4b4d0fb0b44d878a003a33", + "model_id": "c76e3afc58a84118a04144f2f08ff6bc", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "QUEUEING TASKS | : 0it [00:00, ?it/s]" + "QUEUEING TASKS | : 0%| | 0/1 [00:00
<xarray.Dataset>\n",
+       "
<xarray.Dataset> Size: 3kB\n",
        "Dimensions:           (bands: 285)\n",
        "Dimensions without coordinates: bands\n",
        "Data variables:\n",
-       "    wavelengths       (bands) float32 ...\n",
-       "    fwhm              (bands) float32 ...\n",
-       "    good_wavelengths  (bands) float32 ...
"
+       "    wavelengths       (bands) float32 1kB ...\n",
+       "    fwhm              (bands) float32 1kB ...\n",
+       "    good_wavelengths  (bands) float32 1kB ...
" ], "text/plain": [ - "\n", + " Size: 3kB\n", "Dimensions: (bands: 285)\n", "Dimensions without coordinates: bands\n", "Data variables:\n", - " wavelengths (bands) float32 ...\n", - " fwhm (bands) float32 ...\n", - " good_wavelengths (bands) float32 ..." + " wavelengths (bands) float32 1kB ...\n", + " fwhm (bands) float32 1kB ...\n", + " good_wavelengths (bands) float32 1kB ..." ] }, - "execution_count": 13, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -638,7 +613,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/docs/howto/search-collections.md b/docs/howto/search-collections.md index 9b4ee282..517b8629 100644 --- a/docs/howto/search-collections.md +++ b/docs/howto/search-collections.md @@ -1,3 +1,3 @@ # how to search for data collections using spatial, temporal, keyword filters -Coming soon \ No newline at end of file +Coming soon diff --git a/docs/howto/search-granules.md b/docs/howto/search-granules.md index 06286eb9..2926dfa2 100644 --- a/docs/howto/search-granules.md +++ b/docs/howto/search-granules.md @@ -32,5 +32,3 @@ results = earthaccess.search_data( ``` Now that we have our results we can do multiple things: We can iterate over them to get HTTP (or S3) links, we can download the files to a local folder, or we can open these files and stream their content directly to other libraries e.g. xarray. - - diff --git a/docs/howto/storage-location.md b/docs/howto/storage-location.md index f68773d1..83eac95a 100644 --- a/docs/howto/storage-location.md +++ b/docs/howto/storage-location.md @@ -1,3 +1,3 @@ # Determine if a data set is on-prem or in the cloud -Coming soon \ No newline at end of file +Coming soon diff --git a/docs/resources.md b/docs/resources.md index 2421c7c2..341d8d7d 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -5,5 +5,3 @@ * [earthaccess @ AGU poster](https://docs.google.com/presentation/d/1OOSZzbHv6Ck4lzOE01FQdI4kX0VCfcVBS4pWCc7dtBo/edit?usp=sharing) * [NSIDC Cloud Data Access Guide](https://nsidc.org/data/user-resources/help-center/nasa-earthdata-cloud-data-access-guide) - - diff --git a/docs/tutorials/SSL.ipynb b/docs/tutorials/SSL.ipynb index 75182d4b..265f1fd7 100644 --- a/docs/tutorials/SSL.ipynb +++ b/docs/tutorials/SSL.ipynb @@ -10,7 +10,7 @@ "\n", "### This notebook is entirely based on Jinbo Wang's [tutorial](https://github.com/betolink/the-coding-club/blob/main/notebooks/Earthdata_webinar_20220727.ipynb)\n", "\n", - "\n", + "\"Sea\n", "\n", "--- \n", "\n", @@ -157,11 +157,11 @@ " * not fully tested with Dask distributed\n", "* Data is gridded\n", " * xarray works better with homogeneous coordinates, working with swath data will be cumbersome.\n", - "* Data is chunked using reasonable large sizes(1MB or more)\n", + "* Data is chunked using reasonable large sizes (1MB or more)\n", " * If our files are chunked in small pieces the access time will be orders of magnitude bigger than just downloading the data and accessing it locally.\n", " \n", - "Opening a year of SSH (SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL1812) data (1.1 GB approx) can take up to 5 minutes streaming the data out of region(not in AWS)\n", - "The reason for this is not that the data transfer is order of magintude slower but due the client libraries not fetching data concurrently and the metadata of the files in HDF is usually not consolidated like in Zaar, hence h5netcdf has to issue a lot of requests to get the info it needs.\n", + "Opening a year of SSH (SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL1812) data (1.1 GB approx) can take up to 5 minutes streaming the data out of region (not in AWS)\n", + "The reason for this is not that the data transfer is order of magnitude slower but due the client libraries not fetching data concurrently and the metadata of the files in HDF is usually not consolidated like in Zaar, hence h5netcdf has to issue a lot of requests to get the info it needs.\n", "\n", "> Note: we are looping through each year and getting the metadata for the first granule in May" ] @@ -195,7 +195,7 @@ "id": "8b63ca2f-c94c-4d4a-a620-a086ee66137f", "metadata": {}, "source": [ - "### What `earthaccess.open()` do?\n", + "### What does `earthaccess.open()` do?\n", "\n", "`earthaccess.open()` takes a list of results from `earthaccess.search_data()` or a list of URLs and creates a list of Python File-like objects that can be used in our code as if the remote files were local. When executed in AWS the file system used is [S3FS](https://github.com/fsspec/s3fs) when we open files outside of AWS we get a regular HTTPS file session. \n" ] diff --git a/docs/user-reference/collections/collections-query.md b/docs/user-reference/collections/collections-query.md index e554278f..a59e6926 100644 --- a/docs/user-reference/collections/collections-query.md +++ b/docs/user-reference/collections/collections-query.md @@ -7,4 +7,3 @@ show_source: false inherited_members: true show_source: false - diff --git a/docs/user-reference/collections/collections.md b/docs/user-reference/collections/collections.md index c877360b..e83628b9 100644 --- a/docs/user-reference/collections/collections.md +++ b/docs/user-reference/collections/collections.md @@ -5,4 +5,3 @@ inherited_members: true show_root_heading: true show_source: false - diff --git a/docs/user-reference/granules/granules-query.md b/docs/user-reference/granules/granules-query.md index 78120c0d..75f1adb2 100644 --- a/docs/user-reference/granules/granules-query.md +++ b/docs/user-reference/granules/granules-query.md @@ -8,4 +8,3 @@ inherited_members: true show_root_heading: true show_source: false - diff --git a/docs/user-reference/granules/granules.md b/docs/user-reference/granules/granules.md index 788949ef..e0bbd2c2 100644 --- a/docs/user-reference/granules/granules.md +++ b/docs/user-reference/granules/granules.md @@ -5,6 +5,3 @@ inherited_members: true show_root_heading: true show_source: false - - - diff --git a/poetry.lock b/poetry.lock index 00d277a3..10e0260f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiobotocore" @@ -462,13 +462,13 @@ test = ["coverage", "freezegun", "pre-commit", "pytest", "pytest-cov", "pytest-m [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]]