From 29aad8ea8a36d0b8fee6afa802cd0c622da8e31f Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Thu, 26 Oct 2017 13:49:37 -0700 Subject: [PATCH 01/10] Binarizer -> Binner b/c Binarizer is a sklearn transformer --- neuroglia/spike.py | 4 ++-- tests/test_spike.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/neuroglia/spike.py b/neuroglia/spike.py index 21e6a11..164e6b1 100644 --- a/neuroglia/spike.py +++ b/neuroglia/spike.py @@ -10,8 +10,8 @@ def get_neuron(neuron_spikes): assert len(unique_neurons)==1 return unique_neurons[0] -class Binarizer(BaseEstimator,TransformerMixin): - """Binarize a population of spike events into an array of spike counts. +class Binner(BaseEstimator,TransformerMixin): + """Bin a population of spike events into an array of spike counts. """ def __init__(self,sample_times): diff --git a/tests/test_spike.py b/tests/test_spike.py index dbe19cd..620c808 100644 --- a/tests/test_spike.py +++ b/tests/test_spike.py @@ -5,7 +5,7 @@ import numpy.testing as npt import xarray.testing as xrt -from neuroglia.spike import Smoother, Binarizer +from neuroglia.spike import Smoother, Binner from sklearn.base import clone @@ -21,8 +21,8 @@ def test_Smoother(): npt.assert_array_equal(smoothed.index,TS) clone(smoother) -def test_Binarizer(): - binarizer = Binarizer(sample_times=TS) +def test_Binner(): + binarizer = Binner(sample_times=TS) binarized = binarizer.fit_transform(SPIKES) npt.assert_array_equal(binarized.index,TS[:-1]) From 5f1c58f17ae94706b539be584cb9d79770b45b2e Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Thu, 26 Oct 2017 15:25:13 -0700 Subject: [PATCH 02/10] added edge detection --- neuroglia/trace.py | 44 +++++++++++++++++++++++++++++++++++ tests/test_traces.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 tests/test_traces.py diff --git a/neuroglia/trace.py b/neuroglia/trace.py index e69de29..9200ce4 100644 --- a/neuroglia/trace.py +++ b/neuroglia/trace.py @@ -0,0 +1,44 @@ +import numpy as np +from sklearn.base import TransformerMixin, BaseEstimator + +def edge_detector(X,falling=False): + + X = np.apply_along_axis( + func1d=np.diff, + axis=0, + arr=X.copy(), + ) + empty_row = np.zeros(shape=(1,X.shape[1]),dtype=X.dtype) + X = np.vstack((empty_row,X)) + + if falling: + return X < 0 + else: + return X > 0 + +class EdgeDetector(BaseEstimator,TransformerMixin): + """docstring for EdgeDetector.""" + def __init__(self, falling=False): + self.falling = falling + + def fit(self,X,y=None): + return self + + def transform(self,X): + + return edge_detector(X,self.falling) + +class WhenTrueFinder(BaseEstimator,TransformerMixin): + """docstring for WhenTrueFinder.""" + def __init__(self): + pass + + def fit(self,X,y=None): + return self + + def transform(self,X): + return (X[X > 0] + .stack() + .reset_index()[['level_0','level_1']] + .rename(columns={'level_0':'time','level_1':'neuron'}) + ) diff --git a/tests/test_traces.py b/tests/test_traces.py new file mode 100644 index 0000000..5c1c262 --- /dev/null +++ b/tests/test_traces.py @@ -0,0 +1,55 @@ +import numpy as np +import numpy.testing as npt + +import pandas as pd +import pandas.testing as pdt + +from neuroglia.trace import EdgeDetector, WhenTrueFinder + +X = np.array( + [[0, 0, 1], + [1, 1, 0], + [0, 1, 1] + ]) + +RISING = np.array( + [[0, 0, 0], + [1, 1, 0], + [0, 0, 1], + ] +) + +FALLING = np.array( + [[0, 0, 0], + [0, 0, 1], + [1, 0, 0], + ] +) + +def test_EdgeDetector(): + detector = EdgeDetector() + output = detector.fit_transform(X) + npt.assert_array_equal(output,RISING) + + detector = EdgeDetector(falling=True) + output = detector.fit_transform(X) + npt.assert_array_equal(output,FALLING) + +DF = pd.DataFrame( + data=RISING, + index=[0.1,0.2,0.3], + columns=['n1','n2','n3'], +) + +WHENTRUE = pd.DataFrame(dict( + neuron=['n1','n2','n3'], + time=[0.2,0.2,0.3], +)).set_index('time') + +def test_WhenTrueFinder(): + finder = WhenTrueFinder() + output = finder.fit_transform(DF) + output = output.sort_values(['time','neuron']).set_index('time') + print(output) + print(DF) + pdt.assert_frame_equal(output,WHENTRUE) From 04c3db8c6db2a336c10e6c49fade6a95a46f51da Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Thu, 26 Oct 2017 15:45:10 -0700 Subject: [PATCH 03/10] output pandas dataframe if input --- neuroglia/trace.py | 19 +++++++++++++++++-- tests/test_traces.py | 24 +++++++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/neuroglia/trace.py b/neuroglia/trace.py index 9200ce4..5c2af9e 100644 --- a/neuroglia/trace.py +++ b/neuroglia/trace.py @@ -1,8 +1,16 @@ import numpy as np +import pandas as pd from sklearn.base import TransformerMixin, BaseEstimator def edge_detector(X,falling=False): + df = True + try: + index = X.index + columns = X.columns + except AttributeError: + df = False + X = np.apply_along_axis( func1d=np.diff, axis=0, @@ -12,9 +20,16 @@ def edge_detector(X,falling=False): X = np.vstack((empty_row,X)) if falling: - return X < 0 + X = X < 0 + else: + X = X > 0 + + X = X.astype(int) + + if df: + return pd.DataFrame(data=X,index=index,columns=columns) else: - return X > 0 + return X class EdgeDetector(BaseEstimator,TransformerMixin): """docstring for EdgeDetector.""" diff --git a/tests/test_traces.py b/tests/test_traces.py index 5c1c262..007cbdc 100644 --- a/tests/test_traces.py +++ b/tests/test_traces.py @@ -12,6 +12,12 @@ [0, 1, 1] ]) +XDF = pd.DataFrame( + data=X, + index=[0.1,0.2,0.3], + columns=['n1','n2','n3'], +) + RISING = np.array( [[0, 0, 0], [1, 1, 0], @@ -19,6 +25,12 @@ ] ) +DF = pd.DataFrame( + data=RISING, + index=[0.1,0.2,0.3], + columns=['n1','n2','n3'], +) + FALLING = np.array( [[0, 0, 0], [0, 0, 1], @@ -35,11 +47,13 @@ def test_EdgeDetector(): output = detector.fit_transform(X) npt.assert_array_equal(output,FALLING) -DF = pd.DataFrame( - data=RISING, - index=[0.1,0.2,0.3], - columns=['n1','n2','n3'], -) + detector = EdgeDetector() + output = detector.fit_transform(XDF) + npt.assert_array_equal(output.values,RISING) + print(output) + print(DF) + pdt.assert_frame_equal(output,DF) + WHENTRUE = pd.DataFrame(dict( neuron=['n1','n2','n3'], From 5d06f328c8cfb1802b7d28b8b70b55d4ad1f7c31 Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Thu, 26 Oct 2017 16:37:11 -0700 Subject: [PATCH 04/10] implement a pandas-friendly binarizer --- neuroglia/trace.py | 29 +++++++++++++++++++++++++++++ tests/test_traces.py | 23 ++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/neuroglia/trace.py b/neuroglia/trace.py index 5c2af9e..c59705e 100644 --- a/neuroglia/trace.py +++ b/neuroglia/trace.py @@ -1,6 +1,35 @@ import numpy as np import pandas as pd from sklearn.base import TransformerMixin, BaseEstimator +from sklearn.preprocessing import binarize + +class Binarizer(BaseEstimator, TransformerMixin): + """docstring for scikit learn Binarizer + """ + + def __init__(self, threshold=0.0, copy=True): + self.threshold = threshold + self.copy = copy + + def fit(self, X, y=None): + return self + + def transform(self, X): + df = True + try: + index = X.index + columns = X.columns + except AttributeError: + df = False + + X_ = binarize(X, threshold=self.threshold, copy=self.copy) + + if df: + return pd.DataFrame(data=X_,index=index,columns=columns) + else: + return X_ + + def edge_detector(X,falling=False): diff --git a/tests/test_traces.py b/tests/test_traces.py index 007cbdc..39a4378 100644 --- a/tests/test_traces.py +++ b/tests/test_traces.py @@ -4,7 +4,7 @@ import pandas as pd import pandas.testing as pdt -from neuroglia.trace import EdgeDetector, WhenTrueFinder +from neuroglia.trace import EdgeDetector, WhenTrueFinder, Binarizer X = np.array( [[0, 0, 1], @@ -67,3 +67,24 @@ def test_WhenTrueFinder(): print(output) print(DF) pdt.assert_frame_equal(output,WHENTRUE) + +X2 = np.array( + [[0, 0, 1], + [1, 2, 0], + [0, 1, 10] + ]) + +X2DF = pd.DataFrame( + data=X, + index=[0.1,0.2,0.3], + columns=['n1','n2','n3'], +) + +def test_Binarizer(): + binarizer = Binarizer() + output = binarizer.fit_transform(X2) + npt.assert_array_equal(output,X) + + binarizer = Binarizer() + output = binarizer.fit_transform(X2DF) + pdt.assert_frame_equal(output,XDF) From 9711fd68949c40ff79e6142e93c72cbe036894cd Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Fri, 27 Oct 2017 14:18:05 -0700 Subject: [PATCH 05/10] fix tensor size --- neuroglia/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuroglia/event.py b/neuroglia/event.py index d4bb192..5ab7830 100644 --- a/neuroglia/event.py +++ b/neuroglia/event.py @@ -71,7 +71,7 @@ def extractor(ev): tracizer = self.Tracizer(t,**self.tracizer_kwargs) traces = tracizer.fit_transform(local_spikes) - traces.index = self.sample_times + traces.index = self.sample_times[:-1] return xr.DataArray(traces,dims=['sample_times','neuron']) From cf0099d8e388269276b0894936cec11824fa8153 Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Sat, 28 Oct 2017 17:02:59 -0700 Subject: [PATCH 06/10] tensorizer -> sampler --- neuroglia/event.py | 6 +++--- tests/test_event.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/neuroglia/event.py b/neuroglia/event.py index 5ab7830..c23f1eb 100644 --- a/neuroglia/event.py +++ b/neuroglia/event.py @@ -7,7 +7,7 @@ from .utils import create_interpolator, events_to_xr_dim from .spike import Smoother, DEFAULT_TAU -class EventTraceTensorizer(BaseEstimator,TransformerMixin): +class PeriEventTraceSampler(BaseEstimator,TransformerMixin): """docstring for EventTensorizer.""" def __init__(self, traces, sample_times): self.sample_times = sample_times @@ -38,8 +38,8 @@ def extractor(ev): return xr.concat(tensor,dim=concat_dim) -class EventSpikeTensorizer(BaseEstimator,TransformerMixin): - """docstring for EventSpikeTensorizer.""" +class PeriEventSpikeSampler(BaseEstimator,TransformerMixin): + """docstring for PeriEventSpikeSampler.""" def __init__(self, spikes, sample_times, tracizer=None,tracizer_kwargs=None): self.spikes = spikes self.sample_times = sample_times diff --git a/tests/test_event.py b/tests/test_event.py index a76501c..6c34c4d 100644 --- a/tests/test_event.py +++ b/tests/test_event.py @@ -5,7 +5,7 @@ import numpy.testing as npt import xarray.testing as xrt -from neuroglia.event import EventTraceTensorizer, EventSpikeTensorizer +from neuroglia.event import PeriEventTraceSampler, PeriEventSpikeSampler from sklearn.base import clone @@ -37,8 +37,8 @@ TS = np.arange(0,1,0.01) -def test_EventTraceTensorizer_dims(): - tensorizer = EventTraceTensorizer(DFF,sample_times=TS) +def test_PeriEventTraceSampler_dims(): + tensorizer = PeriEventTraceSampler(DFF,sample_times=TS) tensor = tensorizer.fit_transform(EVENTS) npt.assert_equal(tensor['neuron'].data,NEURON) @@ -47,8 +47,8 @@ def test_EventTraceTensorizer_dims(): clone(tensorizer) -def test_EventSpikeTensorizer(): - tensorizer = EventSpikeTensorizer(SPIKES,sample_times=TS) +def test_PeriEventSpikeSampler(): + tensorizer = PeriEventSpikeSampler(SPIKES,sample_times=TS) tensor = tensorizer.fit_transform(EVENTS) npt.assert_equal(tensor['neuron'].data,SPIKES['neuron'].unique()) @@ -58,9 +58,9 @@ def test_EventSpikeTensorizer(): clone(tensorizer) -def test_EventSpikeTensorizer_no_response(): +def test_PeriEventSpikeSampler_no_response(): spikes = pd.DataFrame({'neuron':[0,0,1],'time':[0.01,0.2,1.6]}) - tensorizer = EventSpikeTensorizer(spikes,sample_times=TS) + tensorizer = PeriEventSpikeSampler(spikes,sample_times=TS) tensor = tensorizer.fit_transform(EVENTS) From 1f36514a84e23b2d7bbf85dba88cd1ba1079b75a Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Mon, 30 Oct 2017 13:23:06 -0700 Subject: [PATCH 07/10] ResponseExtractor -> ResponseReducer --- neuroglia/tensor.py | 4 ++-- tests/test_tensor.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/neuroglia/tensor.py b/neuroglia/tensor.py index 1b28881..ec3fe51 100644 --- a/neuroglia/tensor.py +++ b/neuroglia/tensor.py @@ -2,10 +2,10 @@ from sklearn.base import BaseEstimator, TransformerMixin import numpy as np -class ResponseExtractor(BaseEstimator,TransformerMixin): +class ResponseReducer(BaseEstimator,TransformerMixin): """docstring for Annotator.""" def __init__(self, method='mean', dim='sample_times'): - super(ResponseExtractor, self).__init__() + super(ResponseReducer, self).__init__() if method == 'mean': self.method = np.mean diff --git a/tests/test_tensor.py b/tests/test_tensor.py index 2efb2c0..dcd81f8 100644 --- a/tests/test_tensor.py +++ b/tests/test_tensor.py @@ -5,7 +5,7 @@ import numpy.testing as npt import xarray.testing as xrt -from neuroglia.tensor import ResponseExtractor +from neuroglia.tensor import ResponseReducer LBL = ['a','b','a','b'] NRN = ['roi_1','roi_2'] @@ -26,8 +26,8 @@ }, ) -def test_ResponseExtractor_smoke(): - extractor = ResponseExtractor() +def test_ResponseReducer_smoke(): + extractor = ResponseReducer() responses = extractor.fit_transform(TENSOR) npt.assert_array_equal(responses['event'],LBL) From 56ea7bd50bbd068e577ff0a7bdb29b5b9c799daa Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Mon, 30 Oct 2017 13:24:09 -0700 Subject: [PATCH 08/10] fix reduce method --- neuroglia/tensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuroglia/tensor.py b/neuroglia/tensor.py index ec3fe51..936be5f 100644 --- a/neuroglia/tensor.py +++ b/neuroglia/tensor.py @@ -19,4 +19,4 @@ def fit(self, X, y=None): return self def transform(self, X): - return X.mean(dim=self.dim) + return X.reduce(self.method,dim=self.dim) From 37fd36d592cd8505841b37e2b4a0107f34d5bcb5 Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Mon, 30 Oct 2017 14:35:43 -0700 Subject: [PATCH 09/10] fillna --- neuroglia/event.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/neuroglia/event.py b/neuroglia/event.py index c23f1eb..95cb832 100644 --- a/neuroglia/event.py +++ b/neuroglia/event.py @@ -40,9 +40,10 @@ def extractor(ev): class PeriEventSpikeSampler(BaseEstimator,TransformerMixin): """docstring for PeriEventSpikeSampler.""" - def __init__(self, spikes, sample_times, tracizer=None,tracizer_kwargs=None): + def __init__(self, spikes, sample_times, fillna=True, tracizer=None,tracizer_kwargs=None): self.spikes = spikes self.sample_times = sample_times + self.fillna = fillna self.Tracizer = tracizer self.tracizer_kwargs = tracizer_kwargs @@ -71,7 +72,7 @@ def extractor(ev): tracizer = self.Tracizer(t,**self.tracizer_kwargs) traces = tracizer.fit_transform(local_spikes) - traces.index = self.sample_times[:-1] + traces.index = self.sample_times[:len(traces)] return xr.DataArray(traces,dims=['sample_times','neuron']) @@ -80,4 +81,9 @@ def extractor(ev): concat_dim = events_to_xr_dim(X) # concatenate the DataArrays into a single DataArray - return xr.concat(tensor,dim=concat_dim) + tensor = xr.concat(tensor,dim=concat_dim) + + if self.fillna: + tensor.fillna(0.0) + + return tensor From 99e3d7d8c1f3fd9b41f0e6491b6d3781ac8abf59 Mon Sep 17 00:00:00 2001 From: Justin Kiggins Date: Tue, 31 Oct 2017 11:03:20 -0700 Subject: [PATCH 10/10] fix fillna --- neuroglia/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuroglia/event.py b/neuroglia/event.py index 95cb832..68dd113 100644 --- a/neuroglia/event.py +++ b/neuroglia/event.py @@ -84,6 +84,6 @@ def extractor(ev): tensor = xr.concat(tensor,dim=concat_dim) if self.fillna: - tensor.fillna(0.0) + tensor = tensor.fillna(0) return tensor