From 8d00dff807e1f28155a2078cc8e3f7924a4f640c Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Fri, 31 Dec 2021 19:44:36 -0600
Subject: [PATCH 1/8] Add support for Virtual Datasets

This creates a new class, NXvirtualfield, which stores the paths and
filenames of the virtual sources.
---
 src/nexusformat/nexus/tree.py | 114 ++++++++++++++++++++++++++++++++--
 1 file changed, 109 insertions(+), 5 deletions(-)

diff --git a/src/nexusformat/nexus/tree.py b/src/nexusformat/nexus/tree.py
index ab68f9d..4998ab7 100644
--- a/src/nexusformat/nexus/tree.py
+++ b/src/nexusformat/nexus/tree.py
@@ -186,7 +186,8 @@
 may vary from call to call, so this is mainly useful in iterative searches.
 """
 __all__ = ['NXFile', 'NXobject', 'NXfield', 'NXgroup', 'NXattr',
-           'NXlink', 'NXlinkfield', 'NXlinkgroup', 'NeXusError',
+           'NXvirtualfield', 'NXlink', 'NXlinkfield', 'NXlinkgroup',
+           'NeXusError',
            'nxgetcompression', 'nxsetcompression',
            'nxgetencoding', 'nxsetencoding', 'nxgetlock', 'nxsetlock',
            'nxgetmaxsize', 'nxsetmaxsize', 'nxgetmemory', 'nxsetmemory',
@@ -815,7 +816,7 @@ def _readdata(self, name):
 
         Returns
         -------
-        NXfield or NXlinkfield
+        NXfield, NXvirtualfield, or NXlinkfield
             Field or link defined by the current path.
         """
         _target, _filename, _abspath, _soft = self._getlink()
@@ -823,9 +824,27 @@ def _readdata(self, name):
             return NXlinkfield(name=name, target=_target, file=_filename,
                                abspath=_abspath, soft=_soft)
         else:
-            value, shape, dtype, attrs = self.readvalues()
-            return NXfield(value=value, name=name, dtype=dtype, shape=shape,
-                           attrs=attrs)
+            field = self.get(self.nxpath)
+            # Read in the data if it's not too large
+            if np.prod(field.shape) < 1000:  # i.e., less than 1k dims
+                try:
+                    value = self.readvalue(self.nxpath)
+                except Exception:
+                    value = None
+            else:
+                value = None
+            attrs = self.attrs
+            if 'NX_class' in attrs and text(attrs['NX_class']) == 'SDS':
+                attrs.pop('NX_class')
+            if field.is_virtual:
+                sources = field.virtual_sources()
+                target = sources[0].dset_name
+                files = [s.file_name for s in sources]
+                return NXvirtualfield(target, files, name=name, attrs=attrs,
+                                      shape=field.shape[1:], dtype=field.dtype)
+            else:
+                return NXfield(value=value, name=name, dtype=field.dtype,
+                               shape=field.shape, attrs=attrs)
 
     def _readlink(self, name):
         """Read an object that is an undefined link at the current path.
@@ -4051,6 +4070,91 @@ def implot(self, fmt='', xmin=None, xmax=None, ymin=None, ymax=None,
 SDS = NXfield  # For backward compatibility
 
 
+class NXvirtualfield(NXfield):
+
+    """NeXus Virtual Field
+
+    This creates a field that is stored as an HDF5 virtual dataset
+    defined by the file path and file names of the source files.
+    """
+
+    def __init__(self, target, files, name='unknown', shape=None, dtype=None,
+                 group=None, attrs=None, abspath=False, **kwargs):
+        """Initialize the field containing the virtual dataset.
+
+        Parameters
+        ----------
+        target : str or NXfield
+            The field to be added from each source dataset. If it is a
+            string, it defines the path to the field within each source
+            file. If it is a NXfield, the path to the field is used, and
+            its shape and dtype override their respective arguments.
+        files : list of str
+            Paths to the source files. These must either be absolute
+            paths or, if abspath is False, a valid relative path.
+        shape : tuple, optional
+            Shape of each source field, by default None. If None, the
+            shape is derived from the target, which must be a NXfield.
+        dtype : dtype, optional
+            Data type of the virtual dataset, by default None. If None,
+            the data type is derived from the target, which must be a
+            NXfield.
+        group : [type], optional
+            Parent group of NeXus field, by default None
+        attrs : [type], optional
+            Dictionary containing NXfield attributes, by default None
+        """
+        if isinstance(target, NXfield):
+            shape = target.shape
+            dtype = target.dtype
+            target = target.nxfilepath
+        self._vpath = target
+        if abspath:
+            self._vfiles = [os.path.abspath(f) for f in files]
+        else:
+            self._vfiles = files
+        if shape:
+            self._vshape = (len(self._vfiles),) + shape
+        else:
+            self._vshape = None
+        super().__init__(name=name, shape=self._vshape, dtype=dtype,
+                         group=group, attrs=attrs, **kwargs)
+        if shape and dtype:
+            self._create_virtual_data()
+
+    def _create_virtual_data(self):
+        source_shape = self.shape[1:]
+        maxshape = (None,) + source_shape
+        layout = h5.VirtualLayout(shape=self._vshape, dtype=self.dtype,
+                                  maxshape=maxshape)
+        for i, f in enumerate(self._vfiles):
+            layout[i] = h5.VirtualSource(f, self._vpath, shape=source_shape)
+        self._create_memfile()
+        self._memfile.create_virtual_dataset('data', layout)
+
+    def __deepcopy__(self, memo={}):
+        """Return a deep copy of the virtual field and its attributes."""
+        obj = self
+        dpcpy = obj.__class__(self._vpath, self._vfiles)
+        memo[id(self)] = dpcpy
+        dpcpy._name = copy(self.nxname)
+        dpcpy._dtype = copy(obj.dtype)
+        dpcpy._shape = copy(obj.shape)
+        dpcpy._vshape = copy(obj._vshape)
+        dpcpy._vpath = copy(obj._vpath)
+        dpcpy._vfiles = copy(obj._vfiles)
+        dpcpy._create_virtual_data()
+        dpcpy._h5opts = copy(obj._h5opts)
+        dpcpy._changed = True
+        dpcpy._uncopied_data = None
+        for k, v in obj.attrs.items():
+            dpcpy.attrs[k] = copy(v)
+        if 'target' in dpcpy.attrs:
+            del dpcpy.attrs['target']
+        dpcpy._group = None
+        return dpcpy
+
+
 class NXgroup(NXobject):
 
     """NeXus group.

From a55af55d7e4781d83849f9e1de4573862f4c1d08 Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Fri, 31 Dec 2021 19:59:21 -0600
Subject: [PATCH 2/8] Prevent creation of core memory file when reading

The NXvirtualfield only needs to create a core memory file if the
virtual dataset is being created for the first time.
---
 src/nexusformat/nexus/tree.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/nexusformat/nexus/tree.py b/src/nexusformat/nexus/tree.py
index 4998ab7..a981b68 100644
--- a/src/nexusformat/nexus/tree.py
+++ b/src/nexusformat/nexus/tree.py
@@ -841,7 +841,8 @@ def _readdata(self, name):
                 target = sources[0].dset_name
                 files = [s.file_name for s in sources]
                 return NXvirtualfield(target, files, name=name, attrs=attrs,
-                                      shape=field.shape[1:], dtype=field.dtype)
+                                      shape=field.shape[1:], dtype=field.dtype,
+                                      create_vds=False)
             else:
                 return NXfield(value=value, name=name, dtype=field.dtype,
                                shape=field.shape, attrs=attrs)
@@ -4079,7 +4080,8 @@ class NXvirtualfield(NXfield):
     """
 
     def __init__(self, target, files, name='unknown', shape=None, dtype=None,
-                 group=None, attrs=None, abspath=False, **kwargs):
+                 group=None, attrs=None, abspath=False, create_vds=True,
+                 **kwargs):
         """Initialize the field containing the virtual dataset.
 
         Parameters
@@ -4119,7 +4121,7 @@ def __init__(self, target, files, name='unknown', shape=None, dtype=None,
             self._vshape = None
         super().__init__(name=name, shape=self._vshape, dtype=dtype,
                          group=group, attrs=attrs, **kwargs)
-        if shape and dtype:
+        if create_vds and shape and dtype:
             self._create_virtual_data()
 
     def _create_virtual_data(self):

From 164cd503ba198aca67cc80b67468daf6397a568b Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Fri, 31 Dec 2021 20:00:06 -0600
Subject: [PATCH 3/8] Add a nxdir script

This allows the contents of the NeXus file to be displayed from the
command line.
---
 setup.cfg                        |  1 +
 src/nexusformat/nexus/tree.py    |  9 +++++++--
 src/nexusformat/scripts/nxdir.py | 31 +++++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100755 src/nexusformat/scripts/nxdir.py

diff --git a/setup.cfg b/setup.cfg
index bbd8078..461c454 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -43,6 +43,7 @@ nexusformat = notebooks/*.ipynb
 [options.entry_points]
 console_scripts =
     nxstack = nexusformat.scripts.nxstack:main
+    nxdir = nexusformat.scripts.nxdir:main
     nxduplicate = nexusformat.scripts.nxduplicate:main
     nexusformat = nexusformat.scripts.nexusformat:main
 
diff --git a/src/nexusformat/nexus/tree.py b/src/nexusformat/nexus/tree.py
index a981b68..b354c78 100644
--- a/src/nexusformat/nexus/tree.py
+++ b/src/nexusformat/nexus/tree.py
@@ -7296,16 +7296,21 @@ def duplicate(input_file, output_file, mode='w-', **kwargs):
 nxduplicate = duplicate
 
 
-def directory(filename):
+def directory(filename, short=False):
     """Print the contents of the named NeXus file.
 
     Parameters
     ----------
     filename : str
         Name of the file to be read.
+    short : bool, optional
+        True if only a short tree is to be printed, by default False
     """
     root = load(filename)
-    print(root.tree)
+    if short:
+        print(root.short_tree)
+    else:
+        print(root.tree)
 
 
 nxdir = directory
diff --git a/src/nexusformat/scripts/nxdir.py b/src/nexusformat/scripts/nxdir.py
new file mode 100755
index 0000000..be21397
--- /dev/null
+++ b/src/nexusformat/scripts/nxdir.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2021, NeXpy Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING, distributed with this software.
+# -----------------------------------------------------------------------------
+import argparse
+
+from nexusformat import __version__
+from nexusformat.nexus import nxdir
+
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description="Print a NeXus file tree")
+    parser.add_argument('file', action='store', help="name of NeXus file")
+    parser.add_argument('-s', '--short', action='store_true',
+                        help="print only the first level")
+    parser.add_argument('-v', '--version', action='version',
+                        version=f'nxdir v{__version__}')
+
+    args = parser.parse_args()
+
+    nxdir(args.file, short=args.short)
+
+
+if __name__ == "__main__":
+    main()

From 0e7c17b70ca0993de678c0644dd69f562f85805b Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Sat, 1 Jan 2022 16:21:41 -0600
Subject: [PATCH 4/8] Add command to consolidate files with virtual data

---
 setup.cfg                                |  5 ++-
 src/nexusformat/nexus/tree.py            | 57 ++++++++++++++++++++++--
 src/nexusformat/scripts/nxconsolidate.py | 41 +++++++++++++++++
 3 files changed, 98 insertions(+), 5 deletions(-)
 create mode 100755 src/nexusformat/scripts/nxconsolidate.py

diff --git a/setup.cfg b/setup.cfg
index 461c454..ebd23fd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -42,10 +42,11 @@ nexusformat = notebooks/*.ipynb
 
 [options.entry_points]
 console_scripts =
-    nxstack = nexusformat.scripts.nxstack:main
+    nexusformat = nexusformat.scripts.nexusformat:main
+    nxconsolidate = nexusformat.scripts.nxconsolidate:main
     nxdir = nexusformat.scripts.nxdir:main
     nxduplicate = nexusformat.scripts.nxduplicate:main
-    nexusformat = nexusformat.scripts.nexusformat:main
+    nxstack = nexusformat.scripts.nxstack:main
 
 [options.extras_require]
 testing = pytest
diff --git a/src/nexusformat/nexus/tree.py b/src/nexusformat/nexus/tree.py
index b354c78..98c109f 100644
--- a/src/nexusformat/nexus/tree.py
+++ b/src/nexusformat/nexus/tree.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -----------------------------------------------------------------------------
-# Copyright (c) 2013-2021, NeXpy Development Team.
+# Copyright (c) 2013-2022, NeXpy Development Team.
 #
 # Author: Paul Kienzle, Ray Osborn
 #
@@ -192,8 +192,8 @@
            'nxgetencoding', 'nxsetencoding', 'nxgetlock', 'nxsetlock',
            'nxgetmaxsize', 'nxsetmaxsize', 'nxgetmemory', 'nxsetmemory',
            'nxgetrecursive', 'nxsetrecursive',
-           'nxclasses', 'nxload', 'nxsave', 'nxduplicate', 'nxdir', 'nxdemo',
-           'nxversion']
+           'nxclasses', 'nxload', 'nxsave', 'nxduplicate', 'nxdir',
+           'nxconsolidate', 'nxdemo', 'nxversion']
 
 import numbers
 import os
@@ -7316,6 +7316,57 @@ def directory(filename, short=False):
 nxdir = directory
 
 
+def consolidate(files, data, scan=None):
+    """Create NXdata using a virtual field to combine multiple files.
+
+    Parameters
+    ----------
+    files : list of str or NXroot
+        List of files to be consolidated. The list should be sorted to
+        ensure that the scan variable changes monotonically.
+    data : str or NXdata
+        Path to the NXdata group to be consolidated. If the argument is
+        a NXdata group, its path within the NeXus file is used.
+    scan : str or NXfield, optional
+        Path to the scan variable in each file, by default None. If the
+        argument is a NXfield, its path within the NeXus file is used.
+        If not specified, the scan is constructed from file indices.
+    """
+
+    if isinstance(files[0], str):
+        files = [nxload(f) for f in files]
+    if isinstance(data, NXdata):
+        data = data.nxpath
+    if scan:
+        if isinstance(scan, NXfield):
+            scan = scan.nxpath
+        scan_files = [f for f in files if data in f and scan in f
+                      and f[data].nxsignal.exists()]
+    else:
+        scan_files = [f for f in files if data in f
+                      and f[data].nxsignal.exists()]
+    scan_file = scan_files[0]
+    if scan:
+        scan_axis = NXfield([f[scan] for f in scan_files if scan in f],
+                            name=scan_file[scan].nxname)
+        if 'long_name' in scan_file[scan].attrs:
+            scan_axis.attrs['long_name'] = scan_file[scan].attrs['long_name']
+    else:
+        scan_axis = NXfield(range(len(scan_files)), name='file_index',
+                            long_name='File Index')
+    signal = scan_file[data].nxsignal
+    axes = scan_file[data].nxaxes
+    sources = [f[signal.nxpath].nxfilename for f in scan_files]
+    scan_field = NXvirtualfield(signal, sources, name=signal.nxname)
+    scan_data = NXdata(scan_field, [scan_axis] + axes,
+                       name=scan_file[data].nxname)
+    scan_data.title = scan_file[data].nxtitle
+    return scan_data
+
+
+nxconsolidate = consolidate
+
+
 def demo(argv):
     """Process a list of command line commands.
 
diff --git a/src/nexusformat/scripts/nxconsolidate.py b/src/nexusformat/scripts/nxconsolidate.py
new file mode 100755
index 0000000..2c66c46
--- /dev/null
+++ b/src/nexusformat/scripts/nxconsolidate.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2022, NeXpy Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING, distributed with this software.
+# -----------------------------------------------------------------------------
+import argparse
+
+from nexusformat import __version__
+from nexusformat.nexus import NXentry, NXroot, nxconsolidate
+from nexusformat.nexus.tree import natural_sort
+
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description="Copy a NeXus file to another file")
+    parser.add_argument('files', action='store', nargs='*',
+                        help="name of NeXus input files (wild cards allowed)")
+    parser.add_argument('-d', '--data', required=True,
+                        help="path to the NXdata group")
+    parser.add_argument('-s', '--scan', required=True,
+                        help="path to the scan variable")
+    parser.add_argument('-e', '--entry', default='entry',
+                        help="name of NXentry group")
+    parser.add_argument('-o', '--output', required=True,
+                        help="name of NeXus output file")
+    parser.add_argument('-v', '--version', action='version',
+                        version=f'nxconsolidate v{__version__}')
+
+    args = parser.parse_args()
+
+    scan_data = nxconsolidate(sorted(args.files, key=natural_sort), args.data,
+                              scan=args.scan)
+    NXroot(NXentry(scan_data, name=args.entry)).save(args.output, 'w-')
+
+
+if __name__ == "__main__":
+    main()

From 199af56561e2aa26d8e1a165f9032da39234d387 Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Sat, 1 Jan 2022 16:29:20 -0600
Subject: [PATCH 5/8] Use pytest.approx for floating point comparisons

---
 tests/test_data.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_data.py b/tests/test_data.py
index 7c7bb86..43ac6bb 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -321,16 +321,16 @@ def test_data_smoothing(x):
 
     assert smooth_data.nxsignal.shape == (101,)
     assert smooth_data.nxaxes[0].shape == (101,)
-    assert smooth_data.nxsignal[0] == np.sin(x)[0]
-    assert smooth_data.nxsignal[-1] == np.sin(x)[-1]
+    assert smooth_data.nxsignal[0] == pytest.approx(np.sin(x)[0])
+    assert smooth_data.nxsignal[-1] == pytest.approx(np.sin(x)[-1])
 
     smooth_data = data.smooth(factor=4)
 
     assert smooth_data.nxsignal.shape == (41,)
     assert smooth_data.nxaxes[0].shape == (41,)
-    assert smooth_data.nxsignal[0] == np.sin(x)[0]
-    assert smooth_data.nxsignal[4] == np.sin(x)[1]
-    assert smooth_data.nxsignal[-1] == np.sin(x)[-1]
+    assert smooth_data.nxsignal[0] == pytest.approx(np.sin(x)[0])
+    assert smooth_data.nxsignal[4] == pytest.approx(np.sin(x)[1])
+    assert smooth_data.nxsignal[-1] == pytest.approx(np.sin(x)[-1])
 
 
 def test_data_selection():

From b5589052ee1d48db96a3ddec607000ec8cf0211e Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Sun, 2 Jan 2022 10:56:36 -0600
Subject: [PATCH 6/8] Sort by scan variable in nxconsolidate

---
 src/nexusformat/nexus/tree.py            | 47 +++++++++++++-----------
 src/nexusformat/scripts/nxconsolidate.py |  7 +---
 2 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/src/nexusformat/nexus/tree.py b/src/nexusformat/nexus/tree.py
index 98c109f..7a7590c 100644
--- a/src/nexusformat/nexus/tree.py
+++ b/src/nexusformat/nexus/tree.py
@@ -7316,14 +7316,14 @@ def directory(filename, short=False):
 nxdir = directory
 
 
-def consolidate(files, data, scan=None):
+def consolidate(files, data_path, scan_path=None):
     """Create NXdata using a virtual field to combine multiple files.
 
     Parameters
     ----------
     files : list of str or NXroot
-        List of files to be consolidated. The list should be sorted to
-        ensure that the scan variable changes monotonically.
+        List of files to be consolidated. If a scan variable is defined,
+        the files are sorted by its values.
     data : str or NXdata
         Path to the NXdata group to be consolidated. If the argument is
         a NXdata group, its path within the NeXus file is used.
@@ -7335,32 +7335,37 @@ def consolidate(files, data, scan=None):
 
     if isinstance(files[0], str):
         files = [nxload(f) for f in files]
-    if isinstance(data, NXdata):
-        data = data.nxpath
-    if scan:
-        if isinstance(scan, NXfield):
-            scan = scan.nxpath
-        scan_files = [f for f in files if data in f and scan in f
-                      and f[data].nxsignal.exists()]
+    if isinstance(data_path, NXdata):
+        data_path = data_path.nxpath
+    if scan_path:
+        if isinstance(scan_path, NXfield):
+            scan_path = scan_path.nxpath
+        scan_files = [f for f in files if data_path in f and scan_path in f
+                      and f[data_path].nxsignal.exists()]
     else:
-        scan_files = [f for f in files if data in f
-                      and f[data].nxsignal.exists()]
+        scan_files = [f for f in files if data_path in f
+                      and f[data_path].nxsignal.exists()]
     scan_file = scan_files[0]
-    if scan:
-        scan_axis = NXfield([f[scan] for f in scan_files if scan in f],
-                            name=scan_file[scan].nxname)
-        if 'long_name' in scan_file[scan].attrs:
-            scan_axis.attrs['long_name'] = scan_file[scan].attrs['long_name']
+    if scan_path:
+        scan_values = [f[scan_path] for f in scan_files]
+        scan_values, scan_files = list(
+            zip(*(sorted(zip(scan_values, scan_files)))))
+        scan_axis = NXfield(scan_values, name=scan_file[scan_path].nxname)
+        if 'long_name' in scan_file[scan_path].attrs:
+            scan_axis.attrs['long_name'] = (
+                scan_file[scan_path].attrs['long_name'])
+        if 'units' in scan_file[scan_path].attrs:
+            scan_axis.attrs['units'] = scan_file[scan_path].attrs['units']
     else:
         scan_axis = NXfield(range(len(scan_files)), name='file_index',
                             long_name='File Index')
-    signal = scan_file[data].nxsignal
-    axes = scan_file[data].nxaxes
+    signal = scan_file[data_path].nxsignal
+    axes = scan_file[data_path].nxaxes
     sources = [f[signal.nxpath].nxfilename for f in scan_files]
     scan_field = NXvirtualfield(signal, sources, name=signal.nxname)
     scan_data = NXdata(scan_field, [scan_axis] + axes,
-                       name=scan_file[data].nxname)
-    scan_data.title = scan_file[data].nxtitle
+                       name=scan_file[data_path].nxname)
+    scan_data.title = data_path
     return scan_data
 
 
diff --git a/src/nexusformat/scripts/nxconsolidate.py b/src/nexusformat/scripts/nxconsolidate.py
index 2c66c46..1330d71 100755
--- a/src/nexusformat/scripts/nxconsolidate.py
+++ b/src/nexusformat/scripts/nxconsolidate.py
@@ -10,7 +10,6 @@
 
 from nexusformat import __version__
 from nexusformat.nexus import NXentry, NXroot, nxconsolidate
-from nexusformat.nexus.tree import natural_sort
 
 
 def main():
@@ -21,8 +20,7 @@ def main():
                         help="name of NeXus input files (wild cards allowed)")
     parser.add_argument('-d', '--data', required=True,
                         help="path to the NXdata group")
-    parser.add_argument('-s', '--scan', required=True,
-                        help="path to the scan variable")
+    parser.add_argument('-s', '--scan', help="path to the scan variable")
     parser.add_argument('-e', '--entry', default='entry',
                         help="name of NXentry group")
     parser.add_argument('-o', '--output', required=True,
@@ -32,8 +30,7 @@ def main():
 
     args = parser.parse_args()
 
-    scan_data = nxconsolidate(sorted(args.files, key=natural_sort), args.data,
-                              scan=args.scan)
+    scan_data = nxconsolidate(args.files, args.data, scan_path=args.scan)
     NXroot(NXentry(scan_data, name=args.entry)).save(args.output, 'w-')
 
 

From fdbd99647aa76d37b0b58c523a8e4f9219961cd0 Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Sun, 2 Jan 2022 11:57:38 -0600
Subject: [PATCH 7/8] Add a test function for virtual fields

---
 tests/test_data.py | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/tests/test_data.py b/tests/test_data.py
index 43ac6bb..75e3b7e 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -1,8 +1,11 @@
+import os
 import warnings
 
 import numpy as np
 import pytest
-from nexusformat.nexus.tree import NXdata, NXentry, NXfield, NXroot, NXsubentry
+from nexusformat.nexus.tree import (NXdata, NXentry, NXfield, NXroot,
+                                    NXsubentry, NXvirtualfield, nxconsolidate,
+                                    nxload)
 
 
 @pytest.fixture
@@ -394,3 +397,41 @@ def test_smart_indices(x, v):
     assert all(v[0][row, col].nxvalue == v[0].nxvalue[row, col])
     assert np.all(v[0][row[:, np.newaxis], col].nxvalue ==
                   v[0].nxvalue[row[:, np.newaxis], col])
+
+
+@pytest.mark.parametrize("path", [True, False])
+def test_virtual_fields(tmpdir, path, v):
+
+    s1 = NXroot(NXentry(NXdata(v)))
+    s2 = NXroot(NXentry(NXdata(2*v)))
+    s3 = NXroot(NXentry(NXdata(3*v)))
+
+    s1.save(os.path.join(tmpdir, "s1.nxs"), "w")
+    s2.save(os.path.join(tmpdir, "s2.nxs"), "w")
+    s3.save(os.path.join(tmpdir, "s3.nxs"), "w")
+
+    sources = [f.nxfilename for f in [s1, s2, s3]]
+
+    if path:
+        vds1 = NXvirtualfield("entry/data/v", sources, shape=v.shape,
+                              dtype=v.dtype)
+    else:
+        vds1 = NXvirtualfield(s1["entry/data/v"], sources)
+
+    assert vds1.shape == (3,) + v.shape
+    assert vds1.dtype == v.dtype
+    assert vds1.sum() == 6 * v.sum()
+
+    vds2 = nxconsolidate(sources, "entry/data")
+
+    assert vds2.nxsignal.shape == vds1.shape
+    assert vds2.nxsignal.dtype == v.dtype
+    assert vds2.sum() == 6 * v.sum()
+
+    NXroot(NXentry(vds2)).save(os.path.join(tmpdir, "vds.nxs"), "w")
+    vds3 = nxload(os.path.join(tmpdir, "vds.nxs"))
+
+    assert "entry/data/v" in vds3
+    assert vds3["entry/data/v"].shape == vds1.shape
+    assert vds3["entry/data/v"].dtype == v.dtype
+    assert vds3["entry/data/v"].sum() == 6 * v.sum()

From 87dcb6a912c29bf9f2e9801a10414615e6a8f29a Mon Sep 17 00:00:00 2001
From: Ray Osborn <rayosborn@mac.com>
Date: Sun, 2 Jan 2022 12:18:14 -0600
Subject: [PATCH 8/8] Add check of h5py `is_virtual` property

---
 tests/test_data.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_data.py b/tests/test_data.py
index 75e3b7e..5be3c5a 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -435,3 +435,4 @@ def test_virtual_fields(tmpdir, path, v):
     assert vds3["entry/data/v"].shape == vds1.shape
     assert vds3["entry/data/v"].dtype == v.dtype
     assert vds3["entry/data/v"].sum() == 6 * v.sum()
+    assert vds3.nxfile["entry/data/v"].is_virtual