Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ensure sorting analyzer in zarr are consolidated #3378

Merged
merged 7 commits into from
Sep 12, 2024
Merged

Ensure sorting analyzer in zarr are consolidated #3378

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a5ed0b3
Ensure sorting analyzer in zarr are consolidated
alejoe91 Sep 6, 2024
c12ceb6
Use open_consolidated when possible
alejoe91 Sep 9, 2024
4d7f4b5
Merge branch 'main' of github.com:SpikeInterface/spikeinterface into …
alejoe91 Sep 9, 2024
8b5de9d
propagate storage option
samuelgarcia Sep 11, 2024
c56d625
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 11, 2024
d2b82b9
Merge branch 'main' of github.com:SpikeInterface/spikeinterface into …
alejoe91 Sep 11, 2024
ff07ac6
Fix run_info and consolidate metadata
alejoe91 Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions src/spikeinterface/core/sortinganalyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ def create_sorting_analyzer(
return_scaled : bool, default: True
All extensions that play with traces will use this global return_scaled : "waveforms", "noise_levels", "templates".
This prevent return_scaled being differents from different extensions and having wrong snr for instance.
overwrite: bool, default: False
If True, overwrite the folder if it already exists.


Returns
-------
Expand Down Expand Up @@ -563,11 +566,13 @@ def create_zarr(cls, folder, sorting, recording, sparsity, return_scaled, rec_at

recording_info = zarr_root.create_group("extensions")

zarr.consolidate_metadata(zarr_root.store)

@classmethod
def load_from_zarr(cls, folder, recording=None, storage_options=None):
import zarr

zarr_root = zarr.open(str(folder), mode="r", storage_options=storage_options)
zarr_root = zarr.open_consolidated(str(folder), mode="r", storage_options=storage_options)
alejoe91 marked this conversation as resolved.
Show resolved Hide resolved

# load internal sorting in memory
sorting = NumpySorting.from_sorting(
Expand Down Expand Up @@ -2002,7 +2007,7 @@ def _save_data(self, **kwargs):
except:
raise Exception(f"Could not save {ext_data_name} as extension data")
elif self.format == "zarr":

import zarr
import numcodecs

extension_group = self._get_zarr_extension_group(mode="r+")
Expand Down Expand Up @@ -2036,6 +2041,8 @@ def _save_data(self, **kwargs):
except:
raise Exception(f"Could not save {ext_data_name} as extension data")
extension_group[ext_data_name].attrs["object"] = True
# we need to re-consolidate
zarr.consolidate_metadata(self.sorting_analyzer._get_zarr_root().store)

def _reset_extension_folder(self):
"""
Expand All @@ -2051,7 +2058,7 @@ def _reset_extension_folder(self):
import zarr

zarr_root = zarr.open(self.folder, mode="r+")
extension_group = zarr_root["extensions"].create_group(self.extension_name, overwrite=True)
_ = zarr_root["extensions"].create_group(self.extension_name, overwrite=True)

def reset(self):
"""
Expand Down
Loading