Run tests with h5py in workflow

.github/workflows/main.yml

@@ -14,18 +14,18 @@ env:
   USER2_PASSWORD: test
   HSDS_USERNAME: test_user1
   HSDS_PASSWORD: test
-  HSDS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock
+  HSDS_ENDPOINT: http://127.0.0.1:5101
   ROOT_DIR: ${{github.workspace}}/hsds/hsds_root
   BUCKET_NAME: hsds_bucket
-  HS_ENDPOINT: http+unix://%2Ftmp%2Fhs%2Fsn_1.sock
+  HS_ENDPOINT: http://127.0.0.1:5101
   H5PYD_TEST_FOLDER: /home/test_user1/h5pyd_tests/
 
 jobs:
   build-and-test:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest,]
+        os: [ubuntu-latest, windows-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     runs-on: ${{ matrix.os }}
@@ -62,29 +62,64 @@ jobs:
           path: ${{github.workspace}}/hsds
 
       - name: Install HSDS
+        working-directory: ${{github.workspace}}/hsds
         shell: bash
         run: |
-          cd ${{github.workspace}}/hsds
           pip install -e .
 
       - name: Start HSDS
         shell: bash
+        working-directory: ${{github.workspace}}/hsds
         run: |
-          cd ${{github.workspace}}/hsds
           mkdir hsds_root
           mkdir hsds_root/hsds_bucket
           cp admin/config/groups.default admin/config/groups.txt
           cp admin/config/passwd.default admin/config/passwd.txt
-          ./runall.sh --no-docker 1 &
-          sleep 11 # let the nodes get ready
+          hsds --root_dir hsds_root --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 &
+
+      - name: Wait for node startup
+        shell: bash
+        run: |
+          sleep 30
+
+      - name: HSDS Setup
+        shell: bash
+        working-directory: ${{github.workspace}}/hsds
+        run: |
           python tests/integ/setup_test.py
 
       - name: Create h5pyd test folder
-        shell: bash
+        env:
+          HS_USERNAME: test_user1
+          HS_PASSWORD: test
+          TEST2_USERNAME: test_user1
+          TEST2_PASSWORD: test
         run: |
-          HS_USERNAME=test_user1 HS_PASSWORD=test TEST2_USERNAME=test_user1 TEST2_PASSWORD=test hstouch -v /home/test_user1/h5pyd_tests/
+          hstouch -v /home/test_user1/h5pyd_tests/
 
       - name: Run h5pyd tests
-        shell: bash
+        id: h5pyd-tests
+        env:
+          HS_USERNAME: test_user1
+          HS_PASSWORD: test
+          TEST2_USERNAME: test_user1
+          TEST2_PASSWORD: test
+        run: |
+          python testall.py
+
+      - name: Show HSDS Logs on Fail
+        working-directory: ${{github.workspace}}/hsds
+        # Only run if the whole workflow failed due to h5pyd tests
+        if: ${{failure() && steps.h5pyd-tests.outcome == 'failure'}}
+        run: |
+          cat hs.log
+
+      - name: Run h5pyd tests with h5py
+        env:
+          HS_USERNAME: test_user1
+          HS_PASSWORD: test
+          TEST2_USERNAME: test_user1
+          TEST2_PASSWORD: test
+          USE_H5PY: 1
         run: |
-          HS_USERNAME=test_user1 HS_PASSWORD=test TEST2_USERNAME=test_user1 TEST2_PASSWORD=test python testall.py
+          python testall.py

h5pyd/_hl/dataset.py

@@ -1007,7 +1007,9 @@ def __getitem__(self, args, new_dtype=None):
         self.log.debug("selection_constructor")
 
         if selection.nselect == 0:
-            return numpy.ndarray(selection.mshape, dtype=new_dtype)
+            # force compliance with h5py selection behavior
+            shape = numpy.empty(self.shape)[args].shape
+            return numpy.ndarray(shape, dtype=new_dtype)
         # Up-converting to (1,) so that numpy.ndarray correctly creates
         # np.void rows in case of multi-field dtype. (issue 135)
         single_element = selection.mshape == ()

test/hl/common.py

@@ -20,6 +20,7 @@
 
 import numpy as np
 import unittest as ut
+from platform import system
 
 
 # Check if non-ascii filenames are supported
@@ -45,22 +46,22 @@ def getTestFileName(basename, subfolder=None):
 
     if config.get("use_h5py"):
         filename = "out"
-        if not op.isdir(filename):
+        if not os.path.isdir(filename):
             os.mkdir(filename)
         if subfolder:
-            filename = op.join(filename, subfolder)
-            if not op.isdir(filename):
+            filename = os.path.join(filename, subfolder)
+            if not os.path.isdir(filename):
                 os.mkdir(filename)
-        filename = op.join(filename, f"{basename}.h5")
+        filename = os.path.join(filename, f"{basename}.h5")
     else:
         if "H5PYD_TEST_FOLDER" in os.environ:
             filename = os.environ["H5PYD_TEST_FOLDER"]
         else:
             # default to the root folder
             filename = "/"
         if subfolder:
-            filename = op.join(filename, subfolder)
-        filename = op.join(filename, f"{basename}.h5")
+            filename = os.path.join(filename, subfolder)
+        filename = os.path.join(filename, f"{basename}.h5")
     return filename
 
 
@@ -242,18 +243,25 @@ def getPathFromDomain(self, domain):
         E.g. "mytest.h5pyd_test.hdfgroup.org" to
              "/org/hdfgroup/h5pyd_test/mytest
         """
-        if domain.find('/') > -1:
+        if (domain.find('/') > -1) or (domain.find(':') > -1):
             # looks like the domain already is specified as a path
             return domain
 
         names = domain.split('.')
         names.reverse()
-        path = '/'
+
+        # strip empty names
+        names = [name for name in names if name]
+
+        path = os.path.abspath(os.sep)
+
         for name in names:
-            if name:
-                path += name
-                path += '/'
-        path = path[:-1]  # strip trailing slash
+            path = os.path.join(path, name)
+
+        # strip trailing slash
+        if path[-1] == os.sep:
+            path = path[:-1]
+
         return path
 
     def is_hsds(self, id=None):

test/hl/test_dataset_getitem.py

@@ -287,15 +287,23 @@ def setUp(self):
     @ut.expectedFailure
     def test_ellipsis(self):
         """ Ellipsis -> ndarray promoted to underlying shape """
-        out = self.dset[...]
-        self.assertArrayEqual(out, self.data)
+        if self.is_hsds():
+            out = self.dset[...]
+            self.assertArrayEqual(out, self.data)
+        else:
+            # Manually raise error to allow running tests with h5py
+            raise IOError("HSDS failure")
 
     # FIXME: HSDS failure
     @ut.expectedFailure
     def test_tuple(self):
         """ () -> same as ellipsis """
-        out = self.dset[...]
-        self.assertArrayEqual(out, self.data)
+        if self.is_hsds():
+            out = self.dset[...]
+            self.assertArrayEqual(out, self.data)
+        else:
+            # Manually raise error to allow running tests with h5py
+            raise IOError("HSDS failure")
 
     def test_slice(self):
         """ slice -> ValueError """
@@ -553,11 +561,8 @@ def test_shape(self):
     def test_indexlist(self):
         self.assertNumpyBehavior(self.dset, self.data, np.s_[:, [0, 1, 2]])
 
-    @ut.expectedFailure
     def test_index_emptylist(self):
         self.assertNumpyBehavior(self.dset, self.data, np.s_[:, []])
-        # test with single empty list failing - but results seems compat
-        # with h5py 3.2.1 at least
         self.assertNumpyBehavior(self.dset, self.data, np.s_[[]])
 
 

test/hl/test_dimscale.py

@@ -134,7 +134,8 @@ def test_everything(self):
         else:
             with self.assertRaises(UnicodeError):
                 dset.dims.create_scale(f['scale_name'], '√')
-        with self.assertRaises(AttributeError):
+
+        with self.assertRaises((AttributeError, TypeError)):
             dset.dims.create_scale(f['scale_name'], 67)
 
         f.close()

test/hl/test_file.py

@@ -67,7 +67,9 @@ def test_create(self):
         self.assertTrue(f.id.id is not None)
         self.assertTrue('/' in f)
         # should not see id as a file
-        self.assertFalse(h5py.is_hdf5(f.id.id))
+        # skip for h5py, since its is_hdf5 implementation expects a path
+        if h5py.__name__ == "h5pyd":
+            self.assertFalse(h5py.is_hdf5(f.id.id))
         # Check domain's timestamps
         if h5py.__name__ == "h5pyd":
             # print("modified:", datetime.fromtimestamp(f.modified), f.modified)

test/hl/test_folder.py

@@ -25,6 +25,10 @@
 class TestFolders(TestCase):
 
     def test_list(self):
+        if config.get("use_h5py"):
+            # Folders not supported for h5py
+            return
+
         # loglevel = logging.DEBUG
         # logging.basicConfig( format='%(asctime)s %(message)s', level=loglevel)
         test_domain = self.getFileName("folder_test")
@@ -34,11 +38,6 @@ def test_list(self):
         # create test file if not present.
         # on first run, this may take a minute before it is visible as a folder item
         f = h5py.File(filepath, mode='a')
-        if config.get("use_h5py"):
-            # Folders not supported for h5py
-            f.close()
-            return
-
         self.assertTrue(f.id.id is not None)
 
         f.close()
@@ -170,17 +169,15 @@ def test_list(self):
         f.close()
 
     def test_create_folder(self):
+        if config.get("use_h5py"):
+            # Folders not supported for h5py
+            return
+
         empty = self.getFileName("empty")
         empty_path = self.getPathFromDomain(empty)
 
-        print("empty_path", empty_path)
-
         f = h5py.File(empty_path, mode='a')
         self.assertTrue(f.id.id is not None)
-        if config.get("use_h5py"):
-            # Folders not supported for h5py
-            f.close()
-            return
 
         f.close()
 
@@ -195,15 +192,15 @@ def test_create_folder(self):
         d.close()
 
     def test_root_folder(self):
+        if config.get("use_h5py"):
+            # Folders not supported for h5py
+            return
+
         test_domain = self.getFileName("folder_test")
 
         filepath = self.getPathFromDomain(test_domain)
         f = h5py.File(filepath, mode='a')
         self.assertTrue(f.id.id is not None)
-        if config.get("use_h5py"):
-            # Folders not supported for h5py
-            f.close()
-            return
 
         f.close()
 

test/hl/test_group.py

@@ -101,6 +101,8 @@ def test_create(self):
             self.assertTrue(False)  # shouldn't get here'
         except RuntimeError:
             pass  # expected
+        except OSError:
+            pass  # also acceptable
 
         del r['tmp']
         self.assertEqual(len(r), 4)