misc fixes

README.rst

@@ -79,4 +79,4 @@ This repository uses `pre-commit`_ to ensure that all committed code follows min
 Acknowledgments
 ===============
 
-`cloud-volume <https://github.com/seung-lab/cloud-volume>_` for compressed morton code and shard/minishard mask implementation.
+`cloud-volume <https://github.com/seung-lab/cloud-volume>_` (BSD 3-Clause licensed) for compressed morton code and shard/minishard mask implementation.

docs/examples.rst

@@ -154,38 +154,30 @@ BigBrain is a very large image (6572 × 7404 × 5711 voxels) reconstructed from
    link-mesh-fragments --no-colon-suffix mesh_labels.csv classif/
 
 
-.. _Waxholm Rat:
-
-In the ``examples/Waxholm`` directory of the source distribution, you will find
-``WHS_SD_rat_T2star_v1.nii.gz``, sourced from
-`nitrc <https://www.nitrc.org/frs/?group_id=1081>`
-
 Conversion of the grey-level template image (sharded precomputed)
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 .. code-block:: sh
 
    volume-to-precomputed \
        --generate-info \
-       --sharded 2,2,0 \
-       ./build/WHS_SD_rat_T2star_v1.nii.gz \
-       ./build/output/
+       --sharding 1,1,0 \
+       colin27T1_seg.nii.gz \
+       colin27T1_seg_sharded
 
-At this point, you need to edit ``/build/output/info_fullres.json`` to set
-``"data_type": "uint8"``. This is done to reduce the size of the final volume
-and better normalization. Seprately, by way of nibabel, we also determined the
-min and max of the volume to be 10.203 and 32766.0 respectively.
+At this point, you need to edit ``colin27T1_seg_sharded/info_fullres.json`` to set
+``"data_type": "uint8"``. This is needed because ``colin27T1_seg.nii.gz`` uses
+a peculiar encoding, with slope and intercept set in the NIfTI header, even
+though only integers between 0 and 255 are encoded.
 
 .. code-block:: sh
 
-  generate-scales-info ./build/output/info_fullres.json ./build/output/
+  generate-scales-info colin27T1_seg_sharded/info_fullres.json colin27T1_seg_sharded/
   volume-to-precomputed \
-      --sharding 2,2,0 \
-      --input-max 32766.0 \
-      --input-min 10.203 \
-      ./build/WHS_SD_rat_T2star_v1.nii.gz \
-      ./build/output/
-  compute-scales ./build/output/
+      --sharding 1,1,0 \
+      colin27T1_seg.nii.gz \
+      colin27T1_seg_sharded/
+  compute-scales colin27T1_seg_sharded/
 
 
 .. _Conversion of Big Brain to sharded precomputed format:

script_tests/test_scripts.py

@@ -98,7 +98,7 @@ def test_all_in_one_conversion(examples_dir, tmpdir):
 
 
 def test_sharded_conversion(examples_dir, tmpdir):
-    input_nifti = examples_dir / "Waxholm" / "WHS_SD_rat_T2star_v1.nii.gz"
+    input_nifti = examples_dir / "JuBrain" / "colin27T1_seg.nii.gz"
     # The file may be present but be a git-lfs pointer file, so we need to open
     # it to make sure that it is the actual correct file.
     try:
@@ -107,14 +107,14 @@ def test_sharded_conversion(examples_dir, tmpdir):
         pytest.skip("Cannot find a valid example file {0} for testing: {1}"
                     .format(input_nifti, exc))
 
-    output_dir = tmpdir / "waxholm"
+    output_dir = tmpdir / "colin27T1_seg_sharded"
     assert subprocess.call([
         "volume-to-precomputed",
         "--generate-info",
-        "--sharding", "2,2,0",
+        "--sharding", "1,1,0",
         str(input_nifti),
         str(output_dir)
-    ], env=env) == 0
+    ], env=env) == 4  # datatype not supported by neuroglancer
 
     with open(output_dir / "info_fullres.json", "r") as fp:
         fullres_info = json.load(fp=fp)
@@ -129,7 +129,7 @@ def test_sharded_conversion(examples_dir, tmpdir):
     ], env=env) == 0
     assert subprocess.call([
         "volume-to-precomputed",
-        "--sharding", "2,2,0",
+        "--sharding", "1,1,0",
         str(input_nifti),
         str(output_dir)
     ], env=env) == 0
@@ -143,8 +143,8 @@ def test_sharded_conversion(examples_dir, tmpdir):
                  in os.walk(output_dir)
                  for filename in filenames]
 
-    assert len(all_files) == 16, ("Expecting 19 files, but got "
-                                  f"{len(all_files)}.\n{all_files}")
+    assert len(all_files) == 7, ("Expecting 7 files, but got "
+                                 f"{len(all_files)}.\n{all_files}")
 
 
 def test_slice_conversion(tmpdir):

src/neuroglancer_scripts/accessor.py

@@ -52,7 +52,10 @@ def get_accessor_for_url(url, accessor_options={}):
                 info = json.loads(accessor.fetch_file("info"))
                 if sharded_base.ShardedAccessorBase.info_is_sharded(info):
                     is_sharding = True
-            except Exception:
+            except (DataAccessError, json.JSONDecodeError):
+                # In the event that info does not exist
+                # Or info is malformed
+                # Fallback to default behavior
                 ...
 
         if is_sharding:
@@ -74,7 +77,10 @@ def get_accessor_for_url(url, accessor_options={}):
                 info = json.loads(accessor.fetch_file("info"))
                 if sharded_base.ShardedAccessorBase.info_is_sharded(info):
                     is_sharding = True
-            except Exception:
+            except (DataAccessError, json.JSONDecodeError):
+                # In the event that info does not exist
+                # Or info is malformed
+                # Fallback to default behavior
                 ...
 
         if is_sharding:

src/neuroglancer_scripts/sharded_base.py

@@ -1,3 +1,38 @@
+# BSD 3-Clause License
+#
+# Copyright (c) 2017, Ignacio Tartavull, William Silversmith, and later authors. # noqa
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # noqa
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Minor later modifications (c) 2024 by Xiao Gui <[email protected]>
+# Minor modifications made from https://github.com/seung-lab/cloud-volume/blob/aebf4e2/cloudvolume/datasource/precomputed/image/common.py # noqa
+#
+# This software is made available under the BSD 3-Clause license.
+
 from typing import List, Dict, Any
 from abc import ABC, abstractmethod
 import math

src/neuroglancer_scripts/sharded_file_accessor.py

@@ -381,9 +381,19 @@ class ShardedFileAccessor(neuroglancer_scripts.accessor.Accessor,
     :param str base_dir: path to the directory containing the pyramid
     :param dict key_to_mip_sizes:
     """
-    can_read = False
+    can_read = True
     can_write = True
 
+    @property
+    def info(self):
+        if hasattr(self, '_info'):
+            return self._info
+        return json.loads(self.fetch_file("info"))
+
+    @info.setter
+    def info(self, val):
+        self._info = val
+
     def __init__(self, base_dir, **kwargs):
         ShardedAccessorBase.__init__(self)
         self.base_dir = pathlib.Path(base_dir)
@@ -392,12 +402,6 @@ def __init__(self, base_dir, **kwargs):
         self.shard_dict: Dict[str, ShardedScale] = {}
         self.ro_shard_dict: Dict[str, ShardedScale] = {}
 
-        try:
-            self.info = json.loads(self.fetch_file("info"))
-            self.can_read = True
-        except IOError:
-            ...
-
         import atexit
         atexit.register(self.close)
 

unit_tests/test_accessor.py

@@ -16,6 +16,7 @@
     convert_file_url_to_pathname,
     Accessor,
     URLError,
+    DataAccessError,
 )
 from neuroglancer_scripts.file_accessor import FileAccessor
 from neuroglancer_scripts.http_accessor import HttpAccessor
@@ -60,9 +61,8 @@ def test_get_accessor_for_url(accessor_options):
 @patch.object(ShardedAccessorBase, "info_is_sharded")
 @pytest.mark.parametrize("scheme", ["https://", "http://", ""])
 @pytest.mark.parametrize("fetch_file_returns, info_is_sharded_returns, exp", [
-    (Exception("foobar"), None, False),
+    (DataAccessError("foobar"), None, False),
     ('mal formed json', None, False),
-    (valid_info_str, Exception("foobar"), False),
     (valid_info_str, False, False),
     (valid_info_str, True, True),
 ])