NetCDF and N-Dimensional Variable Support

README.md

@@ -17,9 +17,7 @@
 
 For a full list of which parts of the HDF5 specification **h5coro** implements, see the [compatibility](#compatibility) section at the end of this readme.  The major limitations currently present in the package are:
 * The code only implements a subset of the HDF5 specification.  **h5coro** has been shown to work on a number of different datasets, but depending on the version of the HDF5 C library used to write the file, and what options were used during its creation, it is very possible that some part of **h5coro** will need to be updated to support reading it.  Hopefully, over time as more of the spec is implemented, this will become less of a problem.
-* The code is not optimized for local file access.  If you are reading data that is local, you will get much better performance from `h5py`.
 * It is a read-only library and has no functionality to write HDF5 data.
-* It targets one dimensional datasets and is not optimized for high dimensionality data.  In practice this means that if you subset a dataset that is two or more dimensions, only the first dimension will be subsetted with all of the other dimensions collapsed into one serial array of elements.  This limitation will be addressed in future releases.
 
 ## Installation
 
@@ -59,8 +57,8 @@ from h5coro import h5coro, s3driver
 h5obj = h5coro.H5Coro(f'{my_bucket}/{path_to_hdf5_file}', s3driver.S3Driver)
 
 # (3) read
-datasets = [{'dataset': '/path/to/dataset1', 'startrow': 0, 'numrows': h5coro.ALL_ROWS},
-            {'dataset': '/path/to/dataset2', 'startrow': 324, 'numrows': 50}]
+datasets = [{'dataset': '/path/to/dataset1', 'hyperslice': []},
+            {'dataset': '/path/to/dataset2', 'hyperslice': [324, 374]}]
 promise = h5obj.readDatasets(datasets=datasets, block=True)
 
 # (4) display
@@ -154,7 +152,7 @@ We follow a standard Forking Workflow for code changes and additions. Submitted
 | ___Bogus Message___  | <span style="color:red">No</span> | | Unversioned |
 | ___Group Info Message___  | <span style="color:red">No</span> | | Version 0 |
 | ___Filter Pipeline Message___  | <span style="color:green">Yes</span> | Version 1, 2 | |
-| ___Attribute Message___  | <span style="color:blue">Partial</span> | Version 1 | Version 2, 3 |
+| ___Attribute Message___  | <span style="color:blue">Partial</span> | Version 1, 2, 3 | Shared message support for v3 |
 | ___Object Comment Message___  | <span style="color:red">No</span> | | Unversioned |
 | ___Object Modification Time (Old) Message___  | <span style="color:red">No</span> | | Unversioned |
 | ___Shared Message Table Message___  | <span style="color:red">No</span> | | Version 0 |

benchmarks/aos_profiler.py

@@ -0,0 +1,134 @@
+# imports
+import time
+import pathlib
+import s3fs
+import h5py
+import boto3
+import h5coro
+from h5coro import s3driver, logger
+
+# configure h5coro
+import logging
+logger.config(logLevel=logging.CRITICAL)
+
+# h5py test
+def eso_h5py_test(hyperslice, with_s3fs):
+
+    # test parameters
+    granule = 'OR_ABI-L1b-RadF-M6C14_G16_s20192950450355_e20192950500042_c20192950500130.nc'
+    bucketname = 'eso-west2-curated'
+    objectname = 'AOS/PoR/geo/GOES-16-ABI-L1B-FULLD/2019/295/04/' + granule
+    objecturl = 's3://' + bucketname + '/' + objectname
+    filename = '/data/AOS/' + granule
+    variable = '/Rad'
+
+    # remove file first
+    try:
+        pathlib.Path.unlink(filename)
+    except:
+        pass
+
+    # read dataset
+    start = time.perf_counter()
+    if with_s3fs:
+        s3 = s3fs.S3FileSystem()
+        fp = h5py.File(s3.open(objecturl, 'rb'), mode='r')
+    else:
+        s3 = boto3.client('s3')
+        with open(filename, 'wb') as f:
+            s3.download_fileobj(bucketname, objectname, f)
+        fp = h5py.File(filename, mode='r')
+    duration1 = time.perf_counter() - start
+
+    # sum dataset
+    start = time.perf_counter()
+    total = 0
+    if len(hyperslice) > 0:
+        for row in fp[variable][hyperslice[0][0]:hyperslice[0][1]]:
+            total += sum(row[hyperslice[1][0]:hyperslice[1][1]])
+    else:
+        for row in fp[variable]:
+            total += sum(row)
+    duration2 = time.perf_counter() - start
+
+    # return results
+    return duration1, duration2, total
+
+# h5coro test
+def eso_h5coro_test(hyperslice):
+
+    # test parameters
+    granule = 'eso-west2-curated/AOS/PoR/geo/GOES-16-ABI-L1B-FULLD/2019/295/04/OR_ABI-L1b-RadF-M6C14_G16_s20192950450355_e20192950500042_c20192950500130.nc'
+    variable = '/Rad'
+    datasets = [ {"dataset": variable, "hyperslice": hyperslice } ]
+    credentials = {"profile":"default"}
+
+    # read dataset
+    start = time.perf_counter()
+    h5obj = h5coro.H5Coro(granule, s3driver.S3Driver, errorChecking=True, verbose=False, credentials=credentials, multiProcess=False)
+    promise = h5obj.readDatasets(datasets, block=True, enableAttributes=False)
+    duration1 = time.perf_counter() - start
+
+    # sum dataset
+    start = time.perf_counter()
+    total = 0
+    for row in promise[variable]:
+        total += sum(row)
+    duration2 = time.perf_counter() - start
+
+    # return results
+    return duration1, duration2, total
+
+# h5py - hypersliced - with s3fs
+hyperslice=[(0,10), (0,10)]
+request_time, sum_time, result = eso_h5py_test(hyperslice, True)
+print(f'\ns3fs: {hyperslice}\n======================')
+print(f'Result = {result}')
+print(f'Opening Time = {request_time:.3f} secs')
+print(f'Summing Time = {sum_time:.3f} secs')
+print(f'Total Time = {sum_time + request_time:.3f} secs')
+
+# h5py - full dataset - with s3fs
+hyperslice=[]
+request_time, sum_time, result = eso_h5py_test(hyperslice, True)
+print(f'\ns3fs: {hyperslice}\n======================')
+print(f'Result = {result}')
+print(f'Opening Time = {request_time:.3f} secs')
+print(f'Summing Time = {sum_time:.3f} secs')
+print(f'Total Time = {sum_time + request_time:.3f} secs')
+
+# h5py - hypersliced - download
+hyperslice=[(0,10), (0,10)]
+request_time, sum_time, result = eso_h5py_test(hyperslice, False)
+print(f'\nh5py: {hyperslice}\n======================')
+print(f'Result = {result}')
+print(f'Opening Time = {request_time:.3f} secs')
+print(f'Summing Time = {sum_time:.3f} secs')
+print(f'Total Time = {sum_time + request_time:.3f} secs')
+
+# h5py - full dataset - download
+hyperslice=[]
+request_time, sum_time, result = eso_h5py_test(hyperslice, True)
+print(f'\nh5py: {hyperslice}\n======================')
+print(f'Result = {result}')
+print(f'Opening Time = {request_time:.3f} secs')
+print(f'Summing Time = {sum_time:.3f} secs')
+print(f'Total Time = {sum_time + request_time:.3f} secs')
+
+# h5coro - hypersliced
+hyperslice=[(0,10), (0,10)]
+request_time, sum_time, result = eso_h5coro_test(hyperslice=hyperslice)
+print(f'\nh5coro: {hyperslice}\n======================')
+print(f'Result = {result}')
+print(f'Opening Time = {request_time:.3f} secs')
+print(f'Summing Time = {sum_time:.3f} secs')
+print(f'Total Time = {sum_time + request_time:.3f} secs')
+
+# h5coro - full dataset
+hyperslice=[]
+request_time, sum_time, result = eso_h5coro_test(hyperslice=hyperslice)
+print(f'\nh5coro: {hyperslice}\n======================')
+print(f'Result = {result}')
+print(f'Opening Time = {request_time:.3f} secs')
+print(f'Summing Time = {sum_time:.3f} secs')
+print(f'Total Time = {sum_time + request_time:.3f} secs')