evetion · evetion · Sep 6, 2023 · Aug 19, 2023 · Aug 22, 2023 · Aug 25, 2023
diff --git a/Project.toml b/Project.toml
@@ -1,11 +1,12 @@
 name = "SpaceLiDAR"
 uuid = "29bf0bfc-420f-4fa5-b441-811fd9e6e11d"
 authors = ["Maarten Pronk", "Deltares"]
-version = "0.3.0"
+version = "0.4.0"
 
 [deps]
 AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
@@ -28,10 +29,11 @@ TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 [compat]
 AWSS3 = "0.10, 0.11"
 CategoricalArrays = "^0.9, 0.10"
+DataAPI = "1"
 DataFrames = "1"
 Distances = "^0.10"
 Extents = "^0.1"
-FillArrays = "1"
+FillArrays = "0.2, 1"
 GeoFormatTypes = "0.4"
 GeoInterface = "1"
 HDF5 = "^0.15, 0.16"

diff --git a/README.md b/README.md
@@ -14,10 +14,10 @@ Currently supports the following data products:
 |--- |--- |--- |--- |
 |ICESat| GLAH06 v34 | [UG](https://nsidc.org/sites/nsidc.org/files/MULTI-GLAH01-V033-V034-UserGuide.pdf) | [ATBD](https://eospso.nasa.gov/sites/default/files/atbd/ATBD-GLAS-02.pdf) |
 |ICESat| GLAH14 v34 | [UG](https://nsidc.org/sites/nsidc.org/files/MULTI-GLAH01-V033-V034-UserGuide.pdf) | [ATBD](https://eospso.nasa.gov/sites/default/files/atbd/ATBD-GLAS-02.pdf) |
-|ICESat-2| ATL03 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL03-V005-UserGuide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL03_ATBD_r005.pdf) |
-|ICESat-2| ATL06 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL03-V005-UserGuide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL06_ATBD_r005.pdf) |
-|ICESat-2| ATL08 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL08-V005-UserGuide.pdf) | [ATBD](https://nsidc.org/sites/default/files/icesat2_atl08_atbd_r005_1.pdf) |
-|ICESat-2| ATL12 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL12-V005-UserGuide.pdf) | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL12_ATBD_r005.pdf) |
+|ICESat-2| ATL03 v6 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl03-v006-userguide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL03_ATBD_r006.pdf) |
+|ICESat-2| ATL06 v5 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl06-v006-userguide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL06_ATBD_r006.pdf) |
+|ICESat-2| ATL08 v6 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl08-v006-userguide.pdf) | [ATBD](https://nsidc.org/sites/default/files/documents/technical-reference/icesat2_atl08_atbd_v006_0.pdf) |
+|ICESat-2| ATL12 v5 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl12-v006-userguide.pdf) | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL12_ATBD_r006.pdf) |
 |GEDI| L2A v2 | [UG](https://lpdaac.usgs.gov/documents/998/GEDI02_UserGuide_V21.pdf) | [ATBD](https://lpdaac.usgs.gov/documents/581/GEDI_WF_ATBD_v1.0.pdf) |
 
 For an overview with code examples, see the FOSS4G Pluto notebook [here](https://www.evetion.nl/SpaceLiDAR.jl/dev/tutorial/foss4g_2021.jl.html)
@@ -26,20 +26,20 @@ If you use SpaceLiDAR.jl in your research, please consider [citing it](https://z
 
 # Install
 ```julia
-] add SpaceLiDAR
+]add SpaceLiDAR
 ```
 
 # Usage
 Search for data
 ```julia
 using SpaceLiDAR
 using Extents
-# Find all ATL08 granules
+# Find all ATL08 granules ever
 granules = search(:ICESat2, :ATL08)
 
 # Find only ATL03 granules in a part of Vietnam
 vietnam = Extent(X = (102., 107.0), Y = (8.0, 12.0))
-granules = search(:ICESat2, :ATL08; bbox=vietnam, version=5)
+granules = search(:ICESat2, :ATL08; extent=vietnam, version=6)
 
 # Find GEDI granules in the same way
 granules = search(:GEDI, :GEDI02_A)
@@ -58,10 +58,10 @@ SpaceLiDAR.netrc!(username, password)  # replace with your credentials
 fn = SpaceLiDAR.download!(granule)
 
 # You can also load a granule from disk
-granule = granule_from_file(fn)
+granule = granule(fn)
 
 # Or from a folder
-local_granules = granules_from_folder(folder)
+local_granules = granules(folder)
 
 # Instantiate search results locally (useful for GEDI location indexing)
 local_granules = instantiate(granules, folder)
@@ -71,7 +71,7 @@ Derive points
 ```julia
 using DataFrames
 fn = "GEDI02_A_2019242104318_O04046_01_T02343_02_003_02_V002.h5"
-g = SpaceLiDAR.granule_from_file(fn)
+g = SpaceLiDAR.granule(fn)
 df = DataFrame(g)
 149680×15 DataFrame
     Row │ longitude  latitude  height    height_error  datetime                 intensity  sensitivity  surface  quality  nmo ⋯
@@ -92,7 +92,7 @@ Derive linestrings
 ```julia
 using DataFrames
 fn = "ATL03_20181110072251_06520101_003_01.h5"
-g = SpaceLiDAR.granule_from_file(fn)
+g = SpaceLiDAR.granule(fn)
 tlines = DataFrame(SpaceLiDAR.lines(g, step=10000))
 Table with 4 columns and 6 rows:
      geom                       sun_angle  track        datetime

diff --git a/docs/src/changelog.md b/docs/src/changelog.md
@@ -1,7 +1,30 @@
-## v0.3.0
+## Unreleased
 
-!!! danger
-    This is a **breaking** release
+- Working on generic retrieval of parameters as a Table from a granule, instead of the hardcoded choices made now for each product. Would result in methods like `points(granule, [vara, varb])`.
+
+### New features
+- New types `Table` and `PartitionedTable`, which support the Tables.jl interface. This prevents allocating code like `reduce(vcat, DataFrame.(points(granule)))` to get a DataFrame. You can now just call `DataFrame(table)`.
+- Reduced allocations in retrieving point data.
+- Introduced `before` and `after` keywords in `search`, to search by date(ranges).
+
+### Fixed
+- Empty (filtered) granules could result in `Vector{BitVector}` columns, which have been changed to `Vector{Bool}`.
+
+### Breaking
+- `points` now return either a `Table` or a `PartitionedTable` instead of `NamedTuple` or `Vector{NamedTuple}`. The old behaviour can be regained by calling `parent` on these tables.
+- Removed `number_of_returns` and `return_number` from ICESat-2 ATL08 canopy output.
+
+### Deprecated
+- Renamed `granule_from_file` to `granule`
+- Renamed `granules_from_file` to `granules`
+- Renamed `write_granule_urls!` to `write_urls`
+
+### Changed
+- Most of the search functionality has been moved out to the more generic [EarthData.jl](https://github.com/evetion/EarthData.jl)
+- Updated ICESat-2 from version 5 to version 6
+
+
+## v0.3.0
 
 - GeoInterface, Extents support
 - Bounding box using Extent subsetting on all `points` functions
@@ -25,7 +48,7 @@
 
 ## v0.2.0
 
-!!! danger
+!!! warning
     This is a **breaking** release
 
 - Many of the column names have changed to be more descriptive.

diff --git a/docs/src/guides/downloads.md b/docs/src/guides/downloads.md
@@ -7,7 +7,7 @@ In such cases it's useful to export a list of granules to a text file and use an
 
 ```julia
 granules = find(:ICESat2, "ATL08")
-SpaceLiDAR.write_granule_urls!("atl08_world.txt", granules)
+SpaceLiDAR.write_urls("atl08_world.txt", granules)
 ```
 
 In my case, I use [aria2c](https://aria2.github.io/manual/en/html/aria2c.html).

diff --git a/docs/src/index.md b/docs/src/index.md
@@ -11,10 +11,10 @@ Currently supports the following data products:
 |--- |--- |--- |--- |
 |ICESat| GLAH06 v34 | [UG](https://nsidc.org/sites/nsidc.org/files/MULTI-GLAH01-V033-V034-UserGuide.pdf) | [ATBD](https://eospso.nasa.gov/sites/default/files/atbd/ATBD-GLAS-02.pdf) |
 |ICESat| GLAH14 v34 | [UG](https://nsidc.org/sites/nsidc.org/files/MULTI-GLAH01-V033-V034-UserGuide.pdf) | [ATBD](https://eospso.nasa.gov/sites/default/files/atbd/ATBD-GLAS-02.pdf) |
-|ICESat-2| ATL03 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL03-V005-UserGuide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL03_ATBD_r005.pdf) |
-|ICESat-2| ATL06 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL03-V005-UserGuide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL06_ATBD_r005.pdf) |
-|ICESat-2| ATL08 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL08-V005-UserGuide.pdf) | [ATBD](https://nsidc.org/sites/default/files/icesat2_atl08_atbd_r005_1.pdf) |
-|ICESat-2| ATL12 v5 | [UG](https://nsidc.org/sites/nsidc.org/files/ATL12-V005-UserGuide.pdf) | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL12_ATBD_r005.pdf) |
+|ICESat-2| ATL03 v6 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl03-v006-userguide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL03_ATBD_r006.pdf) |
+|ICESat-2| ATL06 v5 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl06-v006-userguide.pdf)  | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL06_ATBD_r006.pdf) |
+|ICESat-2| ATL08 v6 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl08-v006-userguide.pdf) | [ATBD](https://nsidc.org/sites/default/files/documents/technical-reference/icesat2_atl08_atbd_v006_0.pdf) |
+|ICESat-2| ATL12 v5 | [UG](https://nsidc.org/sites/default/files/documents/user-guide/atl12-v006-userguide.pdf) | [ATBD](https://icesat-2.gsfc.nasa.gov/sites/default/files/page_files/ICESat2_ATL12_ATBD_r006.pdf) |
 |GEDI| L2A v2 | [UG](https://lpdaac.usgs.gov/documents/998/GEDI02_UserGuide_V21.pdf) | [ATBD](https://lpdaac.usgs.gov/documents/581/GEDI_WF_ATBD_v1.0.pdf) |
 
 

diff --git a/docs/src/tutorial/usage.md b/docs/src/tutorial/usage.md
@@ -7,7 +7,7 @@ granules = search(:ICESat2, :ATL08)
 
 # Find only ATL03 granules in a part of Vietnam
 vietnam = Extent(X = (102., 107.0), Y = (8.0, 12.0))
-granules = search(:ICESat2, :ATL08; bbox=vietnam, version=5)
+granules = search(:ICESat2, :ATL08; extent=vietnam, version=6)
 
 # Find GEDI granules in the same way
 granules = search(:GEDI, :GEDI02_A)
@@ -26,10 +26,10 @@ SpaceLiDAR.netrc!(username, password)  # replace with your credentials
 fn = SpaceLiDAR.download!(granule)
 
 # You can also load a granule from disk
-granule = granule_from_file(fn)
+granule = granule(fn)
 
 # Or from a folder
-local_granules = granules_from_folder(folder)
+local_granules = granules(folder)
 
 # Instantiate search results locally (useful for GEDI location indexing)
 local_granules = instantiate(granules, folder)
@@ -40,7 +40,7 @@ local_granules = instantiate(granules, folder)
 ```julia
 using DataFrames
 fn = "GEDI02_A_2019242104318_O04046_01_T02343_02_003_02_V002.h5"
-g = SpaceLiDAR.granule_from_file(fn)
+g = SpaceLiDAR.granule(fn)
 df = DataFrame(g)
 149680×15 DataFrame
     Row │ longitude  latitude  height    height_error  datetime                 intensity  sensitivity  surface  quality  nmo ⋯
@@ -61,7 +61,7 @@ df = DataFrame(g)
 ```julia
 using DataFrames
 fn = "ATL03_20181110072251_06520101_003_01.h5"
-g = SpaceLiDAR.granule_from_file(fn)
+g = SpaceLiDAR.granule(fn)
 tlines = DataFrame(SpaceLiDAR.lines(g, step=10000))
 Table with 4 columns and 6 rows:
      geom                       sun_angle  track        datetime

diff --git a/src/GEDI/GEDI.jl b/src/GEDI/GEDI.jl
@@ -35,7 +35,7 @@ function info(g::GEDI_Granule)
 end
 
 function gedi_info(filename)
-    id, _ = splitext(filename)
+    id, _ = splitext(basename(filename))
     if endswith(id, "V002")
         type, name, datetime, orbit, sub_orbit, track, ppds, pge_version, revision, version = Base.split(id, "_")
         version = version[2:end]