[MRG] Update legacy json and param conversion function for new json format

dev_scripts/convert_params.py

@@ -10,7 +10,9 @@
 from pathlib import Path
 from requests.exceptions import HTTPError
 
-from hnn_core import convert_to_hdf5
+from hnn_core import convert_to_json
+
+root_path = Path(__file__).parents[1]
 
 
 def download_folder_contents(owner, repo, path):
@@ -49,7 +51,7 @@ def download_folder_contents(owner, repo, path):
     return temp_dir
 
 
-def convert_param_files_from_repo(owner, repo, path):
+def convert_param_files_from_repo(owner, repo, repo_path, local_path):
     """Converts param and json parameter files to a hdf5 file.
 
     Parameters
@@ -66,17 +68,19 @@ def convert_param_files_from_repo(owner, repo, path):
     None
     """
     # Download param files
-    temp_dir = download_folder_contents(owner, repo, path)
+    temp_dir = download_folder_contents(owner, repo, repo_path)
     # Get list of json and param files
     file_list = [Path(temp_dir, f)
                  for f in os.listdir(temp_dir)
                  if f.endswith('.param') or f.endswith('.json')]
     # Assign output location and names
-    output_dir = Path(__file__).parents[1] / 'hnn_core' / 'param'
+    output_dir = Path(local_path)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
     output_filenames = [Path(output_dir, f.name.split('.')[0])
                         for f in file_list]
 
-    [convert_to_hdf5(file, outfile)
+    [convert_to_json(file, outfile)
      for (file, outfile) in zip(file_list, output_filenames)]
 
     # Delete downloads
@@ -88,8 +92,14 @@ def convert_param_files_from_repo(owner, repo, path):
     # hnn param files
     convert_param_files_from_repo(owner='jonescompneurolab',
                                   repo='hnn',
-                                  path='param')
+                                  repo_path='param',
+                                  local_path=(root_path /
+                                              'network_configuration'),
+                                  )
     # hnn-core json files
     convert_param_files_from_repo(owner='jonescompneurolab',
                                   repo='hnn-core',
-                                  path='hnn_core/param')
+                                  repo_path='hnn_core/param',
+                                  local_path=(root_path /
+                                              'network_configuration'),
+                                  )

doc/whats_new.rst

@@ -29,9 +29,6 @@ Changelog
 
 - Add ability to specify number of cells in :class:`~hnn_core.Network`,
   by `Nick Tolley`_ in :gh:`705`
-
-- Add feature to convert param and json files to HDF5 format, by `George Dang`_
-  in :gh:`723`
 
 - Fixed figure annotation overlap in multiple sub-plots, 
   by `Camilo Diaz`_ in :gh:`741`
@@ -61,13 +58,16 @@ Changelog
 - Added :class:`~hnn_core/viz/NetworkPlotter` to visualize and animate network simulations,
   by `Nick Tolley`_ in :gh:`649`.
 
-- Added GUI feature to include Tonic input drives in simulations, 
+- Added GUI feature to include Tonic input drives in simulations,
   by `Camilo Diaz` :gh:`773`
 
-- :func:`~plot_laminar_lfp`, :func:`~plot_dipole`, :func:`~plot_spikes_hist`, 
+- :func:`~plot_laminar_lfp`, :func:`~plot_dipole`, :func:`~plot_spikes_hist`,
   and :func:`~plot_spikes_raster` now plotted from 0 to tstop. Inputs tmin and tmax are deprecated,
   by `Katharina Duecker`_ in :gh:`769`
 
+- Add function :func:`~hnn_core/params/convert_to_json` to convert legacy param
+  and json files to new json format, by `George Dang`_ in :gh:`772`
+
 Bug
 ~~~
 - Fix inconsistent connection mapping from drive gids to cell gids, by

hnn_core/__init__.py

@@ -1,5 +1,5 @@
-from .dipole import simulate_dipole, read_dipole, average_dipoles, Dipole
-from .params import Params, read_params, convert_to_hdf5
+from .dipole import simulate_dipole, read_dipole, average_dipoles, Dipole,_read_dipole_txt
+from .params import Params, read_params, convert_to_json
 from .network import Network, pick_connection
 from .network_models import jones_2009_model, law_2021_model, calcium_model
 from .cell import Cell

hnn_core/params.py

@@ -662,27 +662,50 @@ def compare_dictionaries(d1, d2):
     return d1
 
 
-def convert_to_hdf5(params_fname, out_fname, include_drives=True,
-                    overwrite=True, write_output=False):
-    """Converts json or param format to hdf5
+def convert_to_json(params_fname,
+                    out_fname,
+                    network_connectivity='jones_2009_model',
+                    include_drives=True,
+                    overwrite=True):
+    """Converts legacy json or param format to hierarchical json format
 
     Parameters
     ----------
     params_fname : str or Path
         Path to file
     out_fname: str
         Path to output
+    network_connectivity: str or None, default:' jones_2009_model'
+         Options: ['jones_2009_model', 'law_2021_model', 'calcium_model', None]
+         Neocortical network model to use. Models are defined in
+         network_models. If None, the base Network object with no defined
+         network connectivity will be used.
     include_drives: bool, default=True
         Include drives from params file
     overwrite: bool, default=True
         Overwrite file
-    write_output: bool, default=False
-        Write out simulations
     Returns
     -------
     None
     """
     from .network import Network
+    from .network_models import jones_2009_model, law_2021_model, calcium_model
+
+    # Assign network model callables
+    model_functions = {'jones_2009_model': jones_2009_model,
+                       'law_2021_model': law_2021_model,
+                       'calcium_model': calcium_model
+                       }
+    if network_connectivity:
+        try:
+            network_model = model_functions[network_connectivity]
+        except KeyError:
+            raise KeyError(f'Invalid network connectivity: '
+                           f'"{network_connectivity}"; '
+                           f'Valid options: {list(model_functions.keys())}')
+    else:
+        network_model = Network
+
     # Validate inputs
     _validate_type(params_fname, (str, Path), 'params_fname')
     _validate_type(out_fname, (str, Path), 'out_fname')
@@ -692,17 +715,18 @@ def convert_to_hdf5(params_fname, out_fname, include_drives=True,
     params_suffix = params_fname.suffix.lower().split('.')[-1]
 
     # Add suffix if not supplied
-    if out_fname.suffix != '.hdf5':
-        out_fname = out_fname.with_suffix('.hdf5')
-
-    net = Network(params=read_params(params_fname),
-                  add_drives_from_params=include_drives,
-                  legacy_mode=True if params_suffix == 'param' else False,
-                  )
-    net.write(fname=out_fname,
-              overwrite=overwrite,
-              write_output=write_output,
-              )
+    if out_fname.suffix != '.json':
+        out_fname = out_fname.with_suffix('.json')
+
+    net = network_model(params=read_params(params_fname),
+                        add_drives_from_params=include_drives,
+                        legacy_mode=(True if params_suffix == 'param'
+                                     else False),
+                        )
+
+    net.write_configuration(fname=out_fname,
+                            overwrite=overwrite,
+                            )
     return
 
 

hnn_core/tests/test_io.py

@@ -215,7 +215,7 @@ def test_cell_response_to_dict(jones_2009_network):
     result2 = _cell_response_to_dict(net, write_output=True)
     assert bool(result2) and isinstance(result2, dict)
 
-    # Check for None if kw supplied
+    # Check for empty dict if kw supplied
     result3 = _cell_response_to_dict(net, write_output=False)
     assert result3 == dict()
 

hnn_core/tests/test_params.py

@@ -8,7 +8,12 @@
 
 import pytest
 
-from hnn_core import read_params, Params, jones_2009_model, convert_to_hdf5
+from hnn_core import (read_params, Params, convert_to_json,
+                      Network)
+from hnn_core.hnn_io import read_network_configuration
+from hnn_core.network_models import (jones_2009_model, law_2021_model,
+                                     calcium_model)
+
 
 hnn_core_root = Path(__file__).parents[1]
 
@@ -74,29 +79,140 @@ def test_base_params():
     assert params == params_base
 
 
-def test_convert_to_hdf5_bad_type():
-    """Tests type validation in convert_to_hdf5 function"""
-    good_path = hnn_core_root
-    path_str = str(good_path)
-    bad_path = 5
-
-    # Valid path and string, but not actual files
-    with pytest.raises(
-            ValueError,
-            match="Unrecognized extension, expected one of"
-    ):
-        convert_to_hdf5(good_path, path_str)
-
-    # Bad params_fname
-    with pytest.raises(
-            TypeError,
-            match="params_fname must be an instance of str or Path"
-    ):
-        convert_to_hdf5(bad_path, good_path)
-
-    # Bad out_fname
-    with pytest.raises(
-            TypeError,
-            match="out_fname must be an instance of str or Path"
-    ):
-        convert_to_hdf5(good_path, bad_path)
+class TestConvertToJson:
+    """Tests convert_to_json function"""
+
+    path_default = Path(hnn_core_root, 'param', 'default.json')
+
+    def test_default_network_connectivity(self, tmp_path):
+        """Tests conversion with default parameters"""
+
+        net_params = jones_2009_model(params=read_params(self.path_default),
+                                      add_drives_from_params=True)
+
+        # Write json and check if constructed network is equal
+        outpath = Path(tmp_path, 'default.json')
+        convert_to_json(self.path_default,
+                        outpath
+                        )
+        net_json = read_network_configuration(outpath)
+        assert net_json == net_params
+
+        # Write json without drives
+        outpath_no_drives = Path(tmp_path, 'default_no_drives.json')
+        convert_to_json(self.path_default,
+                        outpath_no_drives,
+                        include_drives=False
+                        )
+        net_json_no_drives = read_network_configuration(outpath_no_drives)
+        assert net_json_no_drives != net_json
+        assert bool(net_json_no_drives.external_drives) is False
+
+        # Check that writing with no extension will add one
+        outpath_no_ext = Path(tmp_path, 'default_no_ext')
+        convert_to_json(self.path_default,
+                        outpath_no_ext
+                        )
+        assert outpath_no_ext.with_suffix('.json').exists()
+
+    def test_law_network_connectivity(self, tmp_path):
+        """Tests conversion with Law 2021 network connectivity model"""
+
+        net_params = law_2021_model(read_params(self.path_default),
+                                    add_drives_from_params=True,
+                                    )
+
+        # Write json and check if constructed network is equal
+        outpath = Path(tmp_path, 'default.json')
+        convert_to_json(self.path_default,
+                        outpath,
+                        network_connectivity='law_2021_model')
+        net_json = read_network_configuration(outpath)
+        assert net_json == net_params
+
+    def test_calcium_network_connectivity(self, tmp_path):
+        """Tests conversion with calcium network connectivity model"""
+
+        net_params = calcium_model(read_params(self.path_default),
+                                   add_drives_from_params=True,
+                                   )
+
+        # Write json and check if constructed network is equal
+        outpath = Path(tmp_path, 'default.json')
+        convert_to_json(self.path_default,
+                        outpath,
+                        network_connectivity='calcium_model')
+        net_json = read_network_configuration(outpath)
+        assert net_json == net_params
+
+    def test_no_network_connectivity(self, tmp_path):
+        """Tests conversion with no network connectivity model"""
+
+        net_params = Network(read_params(self.path_default),
+                             add_drives_from_params=True,
+                             )
+
+        # Write json and check if constructed network is equal
+        outpath = Path(tmp_path, 'default.json')
+        convert_to_json(self.path_default,
+                        outpath,
+                        network_connectivity=None)
+        net_json = read_network_configuration(outpath)
+        assert net_json == net_params
+
+    def test_convert_to_json_legacy(self, tmp_path):
+        """Tests conversion of a param legacy file to json"""
+
+        # Download params
+        param_url = ('https://raw.githubusercontent.com/hnnsolver/'
+                     'hnn-core/test_data/default.param')
+        params_base_fname = Path(hnn_core_root, 'param', 'default.param')
+        if not op.exists(params_base_fname):
+            urlretrieve(param_url, params_base_fname)
+        net_params = jones_2009_model(read_params(params_base_fname),
+                                      add_drives_from_params=True,
+                                      legacy_mode=True
+                                      )
+
+        # Write json and check if constructed network is equal
+        outpath = Path(tmp_path, 'default.json')
+        convert_to_json(params_base_fname, outpath)
+        net_json = read_network_configuration(outpath)
+        assert net_json == net_params
+
+    def test_convert_to_json_bad_type(self):
+        """Tests type validation in convert_to_json function"""
+
+        good_path = hnn_core_root
+        path_str = str(good_path)
+        bad_path = 5
+        bad_network_conn = 'bad_model'
+
+        # Valid path and string, but not actual files
+        with pytest.raises(
+                ValueError,
+                match="Unrecognized extension, expected one of"
+        ):
+            convert_to_json(good_path, path_str)
+
+        # Bad params_fname
+        with pytest.raises(
+                TypeError,
+                match="params_fname must be an instance of str or Path"
+        ):
+            convert_to_json(bad_path, good_path)
+
+        # Bad out_fname
+        with pytest.raises(
+                TypeError,
+                match="out_fname must be an instance of str or Path"
+        ):
+            convert_to_json(good_path, bad_path)
+
+        # Bad network_connectivity
+        with pytest.raises(
+                KeyError,
+                match="Invalid network connectivity:"
+        ):
+            convert_to_json(good_path, good_path,
+                            network_connectivity=bad_network_conn)