diff --git a/dev_scripts/convert_params.py b/dev_scripts/convert_params.py index 979146dce..9f52d0331 100644 --- a/dev_scripts/convert_params.py +++ b/dev_scripts/convert_params.py @@ -10,7 +10,9 @@ from pathlib import Path from requests.exceptions import HTTPError -from hnn_core import convert_to_hdf5 +from hnn_core import convert_to_json + +root_path = Path(__file__).parents[1] def download_folder_contents(owner, repo, path): @@ -49,7 +51,7 @@ def download_folder_contents(owner, repo, path): return temp_dir -def convert_param_files_from_repo(owner, repo, path): +def convert_param_files_from_repo(owner, repo, repo_path, local_path): """Converts param and json parameter files to a hdf5 file. Parameters @@ -66,17 +68,19 @@ def convert_param_files_from_repo(owner, repo, path): None """ # Download param files - temp_dir = download_folder_contents(owner, repo, path) + temp_dir = download_folder_contents(owner, repo, repo_path) # Get list of json and param files file_list = [Path(temp_dir, f) for f in os.listdir(temp_dir) if f.endswith('.param') or f.endswith('.json')] # Assign output location and names - output_dir = Path(__file__).parents[1] / 'hnn_core' / 'param' + output_dir = Path(local_path) + if not os.path.exists(output_dir): + os.makedirs(output_dir) output_filenames = [Path(output_dir, f.name.split('.')[0]) for f in file_list] - [convert_to_hdf5(file, outfile) + [convert_to_json(file, outfile) for (file, outfile) in zip(file_list, output_filenames)] # Delete downloads @@ -88,8 +92,14 @@ def convert_param_files_from_repo(owner, repo, path): # hnn param files convert_param_files_from_repo(owner='jonescompneurolab', repo='hnn', - path='param') + repo_path='param', + local_path=(root_path / + 'network_configuration'), + ) # hnn-core json files convert_param_files_from_repo(owner='jonescompneurolab', repo='hnn-core', - path='hnn_core/param') + repo_path='hnn_core/param', + local_path=(root_path / + 'network_configuration'), + ) diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 2b86badcb..efe2486c5 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -29,9 +29,6 @@ Changelog - Add ability to specify number of cells in :class:`~hnn_core.Network`, by `Nick Tolley`_ in :gh:`705` - -- Add feature to convert param and json files to HDF5 format, by `George Dang`_ - in :gh:`723` - Fixed figure annotation overlap in multiple sub-plots, by `Camilo Diaz`_ in :gh:`741` @@ -61,13 +58,16 @@ Changelog - Added :class:`~hnn_core/viz/NetworkPlotter` to visualize and animate network simulations, by `Nick Tolley`_ in :gh:`649`. -- Added GUI feature to include Tonic input drives in simulations, +- Added GUI feature to include Tonic input drives in simulations, by `Camilo Diaz` :gh:`773` -- :func:`~plot_laminar_lfp`, :func:`~plot_dipole`, :func:`~plot_spikes_hist`, +- :func:`~plot_laminar_lfp`, :func:`~plot_dipole`, :func:`~plot_spikes_hist`, and :func:`~plot_spikes_raster` now plotted from 0 to tstop. Inputs tmin and tmax are deprecated, by `Katharina Duecker`_ in :gh:`769` +- Add function :func:`~hnn_core/params/convert_to_json` to convert legacy param + and json files to new json format, by `George Dang`_ in :gh:`772` + Bug ~~~ - Fix inconsistent connection mapping from drive gids to cell gids, by diff --git a/hnn_core/__init__.py b/hnn_core/__init__.py index f560eaa4e..876d9d2df 100644 --- a/hnn_core/__init__.py +++ b/hnn_core/__init__.py @@ -1,5 +1,5 @@ -from .dipole import simulate_dipole, read_dipole, average_dipoles, Dipole -from .params import Params, read_params, convert_to_hdf5 +from .dipole import simulate_dipole, read_dipole, average_dipoles, Dipole,_read_dipole_txt +from .params import Params, read_params, convert_to_json from .network import Network, pick_connection from .network_models import jones_2009_model, law_2021_model, calcium_model from .cell import Cell diff --git a/hnn_core/params.py b/hnn_core/params.py index 6f26fe2cb..b8bb5a600 100644 --- a/hnn_core/params.py +++ b/hnn_core/params.py @@ -662,9 +662,12 @@ def compare_dictionaries(d1, d2): return d1 -def convert_to_hdf5(params_fname, out_fname, include_drives=True, - overwrite=True, write_output=False): - """Converts json or param format to hdf5 +def convert_to_json(params_fname, + out_fname, + model_template='jones_2009_model', + include_drives=True, + overwrite=True): + """Converts legacy json or param format to hierarchical json format Parameters ---------- @@ -672,17 +675,37 @@ def convert_to_hdf5(params_fname, out_fname, include_drives=True, Path to file out_fname: str Path to output + model_template: str or None, default:' jones_2009_model' + Options: ['jones_2009_model', 'law_2021_model', 'calcium_model', None] + Neocortical network model to use. Models are defined in + network_models. If None, the base Network object with no defined + network connectivity will be used. include_drives: bool, default=True Include drives from params file overwrite: bool, default=True Overwrite file - write_output: bool, default=False - Write out simulations Returns ------- None """ from .network import Network + from .network_models import jones_2009_model, law_2021_model, calcium_model + + # Assign network model callables + model_functions = {'jones_2009_model': jones_2009_model, + 'law_2021_model': law_2021_model, + 'calcium_model': calcium_model + } + if model_template: + try: + network_model = model_functions[model_template] + except KeyError: + raise KeyError(f'Invalid network connectivity: ' + f'"{model_template}"; ' + f'Valid options: {list(model_functions.keys())}') + else: + network_model = Network + # Validate inputs _validate_type(params_fname, (str, Path), 'params_fname') _validate_type(out_fname, (str, Path), 'out_fname') @@ -692,17 +715,18 @@ def convert_to_hdf5(params_fname, out_fname, include_drives=True, params_suffix = params_fname.suffix.lower().split('.')[-1] # Add suffix if not supplied - if out_fname.suffix != '.hdf5': - out_fname = out_fname.with_suffix('.hdf5') - - net = Network(params=read_params(params_fname), - add_drives_from_params=include_drives, - legacy_mode=True if params_suffix == 'param' else False, - ) - net.write(fname=out_fname, - overwrite=overwrite, - write_output=write_output, - ) + if out_fname.suffix != '.json': + out_fname = out_fname.with_suffix('.json') + + net = network_model(params=read_params(params_fname), + add_drives_from_params=include_drives, + legacy_mode=(True if params_suffix == 'param' + else False), + ) + + net.write_configuration(fname=out_fname, + overwrite=overwrite, + ) return diff --git a/hnn_core/tests/test_io.py b/hnn_core/tests/test_io.py index 7a894c67e..2bf208025 100644 --- a/hnn_core/tests/test_io.py +++ b/hnn_core/tests/test_io.py @@ -215,7 +215,7 @@ def test_cell_response_to_dict(jones_2009_network): result2 = _cell_response_to_dict(net, write_output=True) assert bool(result2) and isinstance(result2, dict) - # Check for None if kw supplied + # Check for empty dict if kw supplied result3 = _cell_response_to_dict(net, write_output=False) assert result3 == dict() diff --git a/hnn_core/tests/test_params.py b/hnn_core/tests/test_params.py index 8b767ebff..e532bd009 100644 --- a/hnn_core/tests/test_params.py +++ b/hnn_core/tests/test_params.py @@ -8,7 +8,12 @@ import pytest -from hnn_core import read_params, Params, jones_2009_model, convert_to_hdf5 +from hnn_core import (read_params, Params, convert_to_json, + Network) +from hnn_core.hnn_io import read_network_configuration +from hnn_core.network_models import (jones_2009_model, law_2021_model, + calcium_model) + hnn_core_root = Path(__file__).parents[1] @@ -74,29 +79,142 @@ def test_base_params(): assert params == params_base -def test_convert_to_hdf5_bad_type(): - """Tests type validation in convert_to_hdf5 function""" - good_path = hnn_core_root - path_str = str(good_path) - bad_path = 5 - - # Valid path and string, but not actual files - with pytest.raises( - ValueError, - match="Unrecognized extension, expected one of" - ): - convert_to_hdf5(good_path, path_str) - - # Bad params_fname - with pytest.raises( - TypeError, - match="params_fname must be an instance of str or Path" - ): - convert_to_hdf5(bad_path, good_path) - - # Bad out_fname - with pytest.raises( - TypeError, - match="out_fname must be an instance of str or Path" - ): - convert_to_hdf5(good_path, bad_path) +class TestConvertToJson: + """Tests convert_to_json function""" + + path_default = Path(hnn_core_root, 'param', 'default.json') + + def test_default_network_connectivity(self, tmp_path): + """Tests conversion with default parameters""" + + net_params = jones_2009_model(params=read_params(self.path_default), + add_drives_from_params=True) + + # Write json and check if constructed network is equal + outpath = Path(tmp_path, 'default.json') + convert_to_json(self.path_default, + outpath + ) + net_json = read_network_configuration(outpath) + assert net_json == net_params + + # Write json without drives + outpath_no_drives = Path(tmp_path, 'default_no_drives.json') + convert_to_json(self.path_default, + outpath_no_drives, + include_drives=False + ) + net_json_no_drives = read_network_configuration(outpath_no_drives) + assert net_json_no_drives != net_json + assert bool(net_json_no_drives.external_drives) is False + + # Check that writing with no extension will add one + outpath_no_ext = Path(tmp_path, 'default_no_ext') + convert_to_json(self.path_default, + outpath_no_ext + ) + assert outpath_no_ext.with_suffix('.json').exists() + + def test_law_network_connectivity(self, tmp_path): + """Tests conversion with Law 2021 network connectivity model""" + + net_params = law_2021_model(read_params(self.path_default), + add_drives_from_params=True, + ) + + # Write json and check if constructed network is equal + outpath = Path(tmp_path, 'default.json') + convert_to_json(self.path_default, + outpath, + model_template='law_2021_model') + net_json = read_network_configuration(outpath) + assert net_json == net_params + + def test_calcium_network_connectivity(self, tmp_path): + """Tests conversion with calcium network connectivity model""" + + net_params = calcium_model(read_params(self.path_default), + add_drives_from_params=True, + ) + + # Write json and check if constructed network is equal + outpath = Path(tmp_path, 'default.json') + convert_to_json(self.path_default, + outpath, + model_template='calcium_model') + net_json = read_network_configuration(outpath) + assert net_json == net_params + + def test_no_network_connectivity(self, tmp_path): + """Tests conversion with no network connectivity model""" + + net_params = Network(read_params(self.path_default), + add_drives_from_params=True, + ) + + # Write json and check if constructed network is equal + outpath = Path(tmp_path, 'default.json') + convert_to_json(self.path_default, + outpath, + model_template=None) + net_json = read_network_configuration(outpath) + assert net_json == net_params + # Should only have external drive connections defined, n=22 + assert len(net_json.connectivity) == len(net_params.connectivity) == 22 + + def test_convert_to_json_legacy(self, tmp_path): + """Tests conversion of a param legacy file to json""" + + # Download params + param_url = ('https://raw.githubusercontent.com/hnnsolver/' + 'hnn-core/test_data/default.param') + params_base_fname = Path(hnn_core_root, 'param', 'default.param') + if not op.exists(params_base_fname): + urlretrieve(param_url, params_base_fname) + net_params = jones_2009_model(read_params(params_base_fname), + add_drives_from_params=True, + legacy_mode=True + ) + + # Write json and check if constructed network is equal + outpath = Path(tmp_path, 'default.json') + convert_to_json(params_base_fname, outpath) + net_json = read_network_configuration(outpath) + assert net_json == net_params + + def test_convert_to_json_bad_type(self): + """Tests type validation in convert_to_json function""" + + good_path = hnn_core_root + path_str = str(good_path) + bad_path = 5 + bad_model = 'bad_model' + + # Valid path and string, but not actual files + with pytest.raises( + ValueError, + match="Unrecognized extension, expected one of" + ): + convert_to_json(good_path, path_str) + + # Bad params_fname + with pytest.raises( + TypeError, + match="params_fname must be an instance of str or Path" + ): + convert_to_json(bad_path, good_path) + + # Bad out_fname + with pytest.raises( + TypeError, + match="out_fname must be an instance of str or Path" + ): + convert_to_json(good_path, bad_path) + + # Bad model_template + with pytest.raises( + KeyError, + match="Invalid network connectivity:" + ): + convert_to_json(good_path, good_path, + model_template=bad_model)