diff --git a/README.rst b/README.rst index 6dee98f..3e6aaf5 100644 --- a/README.rst +++ b/README.rst @@ -79,4 +79,4 @@ This repository uses `pre-commit`_ to ensure that all committed code follows min Acknowledgments =============== -`cloud-volume _` for compressed morton code and shard/minishard mask implementation. +`cloud-volume _` (BSD 3-Clause licensed) for compressed morton code and shard/minishard mask implementation. diff --git a/docs/examples.rst b/docs/examples.rst index 37ed41b..f4d07b5 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -154,12 +154,6 @@ BigBrain is a very large image (6572 × 7404 × 5711 voxels) reconstructed from link-mesh-fragments --no-colon-suffix mesh_labels.csv classif/ -.. _Waxholm Rat: - -In the ``examples/Waxholm`` directory of the source distribution, you will find -``WHS_SD_rat_T2star_v1.nii.gz``, sourced from -`nitrc ` - Conversion of the grey-level template image (sharded precomputed) +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -167,25 +161,23 @@ Conversion of the grey-level template image (sharded precomputed) volume-to-precomputed \ --generate-info \ - --sharded 2,2,0 \ - ./build/WHS_SD_rat_T2star_v1.nii.gz \ - ./build/output/ + --sharding 1,1,0 \ + colin27T1_seg.nii.gz \ + colin27T1_seg_sharded -At this point, you need to edit ``/build/output/info_fullres.json`` to set -``"data_type": "uint8"``. This is done to reduce the size of the final volume -and better normalization. Seprately, by way of nibabel, we also determined the -min and max of the volume to be 10.203 and 32766.0 respectively. +At this point, you need to edit ``colin27T1_seg_sharded/info_fullres.json`` to set +``"data_type": "uint8"``. This is needed because ``colin27T1_seg.nii.gz`` uses +a peculiar encoding, with slope and intercept set in the NIfTI header, even +though only integers between 0 and 255 are encoded. .. code-block:: sh - generate-scales-info ./build/output/info_fullres.json ./build/output/ + generate-scales-info colin27T1_seg_sharded/info_fullres.json colin27T1_seg_sharded/ volume-to-precomputed \ - --sharding 2,2,0 \ - --input-max 32766.0 \ - --input-min 10.203 \ - ./build/WHS_SD_rat_T2star_v1.nii.gz \ - ./build/output/ - compute-scales ./build/output/ + --sharding 1,1,0 \ + colin27T1_seg.nii.gz \ + colin27T1_seg_sharded/ + compute-scales colin27T1_seg_sharded/ .. _Conversion of Big Brain to sharded precomputed format: diff --git a/examples/Waxholm/WHS_SD_rat_T2star_v1.nii.gz b/examples/Waxholm/WHS_SD_rat_T2star_v1.nii.gz deleted file mode 100755 index d4073ef..0000000 --- a/examples/Waxholm/WHS_SD_rat_T2star_v1.nii.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:17b877d2f67b66492b1a343b7c77927da1abbf638417782f3ef0f30ea24c426d -size 944947664 diff --git a/script_tests/test_scripts.py b/script_tests/test_scripts.py index d6dfe41..9ca64dc 100644 --- a/script_tests/test_scripts.py +++ b/script_tests/test_scripts.py @@ -98,7 +98,7 @@ def test_all_in_one_conversion(examples_dir, tmpdir): def test_sharded_conversion(examples_dir, tmpdir): - input_nifti = examples_dir / "Waxholm" / "WHS_SD_rat_T2star_v1.nii.gz" + input_nifti = examples_dir / "JuBrain" / "colin27T1_seg.nii.gz" # The file may be present but be a git-lfs pointer file, so we need to open # it to make sure that it is the actual correct file. try: @@ -107,14 +107,14 @@ def test_sharded_conversion(examples_dir, tmpdir): pytest.skip("Cannot find a valid example file {0} for testing: {1}" .format(input_nifti, exc)) - output_dir = tmpdir / "waxholm" + output_dir = tmpdir / "colin27T1_seg_sharded" assert subprocess.call([ "volume-to-precomputed", "--generate-info", - "--sharding", "2,2,0", + "--sharding", "1,1,0", str(input_nifti), str(output_dir) - ], env=env) == 0 + ], env=env) == 4 # datatype not supported by neuroglancer with open(output_dir / "info_fullres.json", "r") as fp: fullres_info = json.load(fp=fp) @@ -129,7 +129,7 @@ def test_sharded_conversion(examples_dir, tmpdir): ], env=env) == 0 assert subprocess.call([ "volume-to-precomputed", - "--sharding", "2,2,0", + "--sharding", "1,1,0", str(input_nifti), str(output_dir) ], env=env) == 0 @@ -143,8 +143,8 @@ def test_sharded_conversion(examples_dir, tmpdir): in os.walk(output_dir) for filename in filenames] - assert len(all_files) == 16, ("Expecting 19 files, but got " - f"{len(all_files)}.\n{all_files}") + assert len(all_files) == 7, ("Expecting 7 files, but got " + f"{len(all_files)}.\n{all_files}") def test_slice_conversion(tmpdir): diff --git a/src/neuroglancer_scripts/accessor.py b/src/neuroglancer_scripts/accessor.py index e8df168..8bb82d7 100644 --- a/src/neuroglancer_scripts/accessor.py +++ b/src/neuroglancer_scripts/accessor.py @@ -52,7 +52,10 @@ def get_accessor_for_url(url, accessor_options={}): info = json.loads(accessor.fetch_file("info")) if sharded_base.ShardedAccessorBase.info_is_sharded(info): is_sharding = True - except Exception: + except (DataAccessError, json.JSONDecodeError): + # In the event that info does not exist + # Or info is malformed + # Fallback to default behavior ... if is_sharding: @@ -74,7 +77,10 @@ def get_accessor_for_url(url, accessor_options={}): info = json.loads(accessor.fetch_file("info")) if sharded_base.ShardedAccessorBase.info_is_sharded(info): is_sharding = True - except Exception: + except (DataAccessError, json.JSONDecodeError): + # In the event that info does not exist + # Or info is malformed + # Fallback to default behavior ... if is_sharding: diff --git a/src/neuroglancer_scripts/sharded_base.py b/src/neuroglancer_scripts/sharded_base.py index bef918e..4b612a2 100644 --- a/src/neuroglancer_scripts/sharded_base.py +++ b/src/neuroglancer_scripts/sharded_base.py @@ -1,3 +1,38 @@ +# BSD 3-Clause License +# +# Copyright (c) 2017, Ignacio Tartavull, William Silversmith, and later authors. # noqa +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # noqa +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Minor later modifications (c) 2024 by Xiao Gui +# Minor modifications made from https://github.com/seung-lab/cloud-volume/blob/aebf4e2/cloudvolume/datasource/precomputed/image/common.py # noqa +# +# This software is made available under the BSD 3-Clause license. + from typing import List, Dict, Any from abc import ABC, abstractmethod import math diff --git a/src/neuroglancer_scripts/sharded_file_accessor.py b/src/neuroglancer_scripts/sharded_file_accessor.py index 1d66bd5..a299261 100644 --- a/src/neuroglancer_scripts/sharded_file_accessor.py +++ b/src/neuroglancer_scripts/sharded_file_accessor.py @@ -381,9 +381,19 @@ class ShardedFileAccessor(neuroglancer_scripts.accessor.Accessor, :param str base_dir: path to the directory containing the pyramid :param dict key_to_mip_sizes: """ - can_read = False + can_read = True can_write = True + @property + def info(self): + if hasattr(self, '_info'): + return self._info + return json.loads(self.fetch_file("info")) + + @info.setter + def info(self, val): + self._info = val + def __init__(self, base_dir, **kwargs): ShardedAccessorBase.__init__(self) self.base_dir = pathlib.Path(base_dir) @@ -392,12 +402,6 @@ def __init__(self, base_dir, **kwargs): self.shard_dict: Dict[str, ShardedScale] = {} self.ro_shard_dict: Dict[str, ShardedScale] = {} - try: - self.info = json.loads(self.fetch_file("info")) - self.can_read = True - except IOError: - ... - import atexit atexit.register(self.close) diff --git a/unit_tests/test_accessor.py b/unit_tests/test_accessor.py index 64ba826..3b67583 100644 --- a/unit_tests/test_accessor.py +++ b/unit_tests/test_accessor.py @@ -16,6 +16,7 @@ convert_file_url_to_pathname, Accessor, URLError, + DataAccessError, ) from neuroglancer_scripts.file_accessor import FileAccessor from neuroglancer_scripts.http_accessor import HttpAccessor @@ -60,9 +61,8 @@ def test_get_accessor_for_url(accessor_options): @patch.object(ShardedAccessorBase, "info_is_sharded") @pytest.mark.parametrize("scheme", ["https://", "http://", ""]) @pytest.mark.parametrize("fetch_file_returns, info_is_sharded_returns, exp", [ - (Exception("foobar"), None, False), + (DataAccessError("foobar"), None, False), ('mal formed json', None, False), - (valid_info_str, Exception("foobar"), False), (valid_info_str, False, False), (valid_info_str, True, True), ])