init commit

BaseLayer.py

@@ -0,0 +1,53 @@
+import sys
+import os
+sys.path.append('..')
+
+import numpy as np
+import queue
+
+class BaseLayer(object):
+    def __init__(self, input_channel, input_shape, output_channel):
+        self.input_channel = input_channel     ## Neuron number
+        self.input_shape = input_shape
+        self.output_channel = output_channel
+
+        self.t = 0
+        self.sample_i = 0
+        self.output_history = list() 
+
+        self.len_t = 0 
+
+    def next_t(self):
+        self.t += 1
+
+    def previous_t(self):
+        self.t -= 1
+        if self.t <= 0:
+            self.t = 0
+
+    def clear_t(self):
+        self.t = 0
+
+    def next_sample(self):
+        self.sample_i += 1
+        self.clear_t()
+        self.clear_historoy()
+
+    def previous_sample(self):
+        self.sample_i -= 1
+        if self.sample_i <= 0:
+            self.sample_i = 0
+        self.clear_t()
+        self.clear_historoy()
+
+    def set_sample(self, i):
+        self.sample_i = i
+        self.clear_t()
+        self.clear_historoy()
+
+    def clear_historoy(self):
+        self.output_history.clear()
+
+
+    def set_len_t(self, t):
+        self.len_t = t

Neuron.py

@@ -0,0 +1,139 @@
+import numpy as np
+from numpy.random import seed
+from numpy.random import randn
+import cfg
+class Neuron(object):
+    def __init__(self, if_st_neuron_clear=False):
+        self.if_st_neuron_clear = if_st_neuron_clear       
+        pass
+
+
+    def neuron_space_expert(self, syn, thres): 
+        V_internal =  syn
+        if V_internal > thres:
+            V_spike = 1
+            V_internal = 0
+        else:
+            V_spike = 0
+        return V_spike
+
+    def neuron_temporal_expert(self,  syn, stim, thres):
+        if syn > 0.5:
+             syn = 0.5 
+
+        V_internal =  syn - 0.5 + stim   # 0.5 is a user defined value
+
+        if V_internal >= thres:
+            V_spike = 1
+            V_internal = 0
+        else:
+            V_spike = 0 
+
+        if V_internal >= 2:           ## max voltage range
+            V_internal = 0
+        elif V_internal < 0:          ## min voltage range
+            V_internal = 0 
+        else:
+            pass  
+
+        return V_spike, V_internal           
+
+    def neuron_ST(self, init, syn, thres, st_ions, st_leakage): 
+        V_internal =  init + syn + st_ions + st_leakage
+        if V_internal >= thres:
+            V_spike = 1
+            V_internal = 0
+        else:
+            V_spike = 0 
+            if self.if_st_neuron_clear:
+                V_internal = 0
+
+        if V_internal > 2:       ## max voltage range
+              V_internal = 2
+        elif V_internal < -0.5:  ## min voltage range
+              V_internal = -0.5 
+        else:
+            pass     
+        return V_spike, V_internal      
+
+    def neuron_SIMO(self, init, input_value, threshold, output_num,ions, leakage):
+                self.input_value = input_value
+                self.output_num = output_num
+                syn = input_value
+                V_spike_index = 0
+                V_spike_internal = np.empty(self.output_num,int)
+                V_decoder = np.full((1,self.output_num),0) 
+                V_internal = syn + init + ions - leakage
+                for i in range(0, self.output_num):
+                    if V_internal >= threshold[i]:
+                            V_spike_internal[i] = 1
+                    else:
+                            V_spike_internal[i] = 0
+
+                if np.all(V_spike_internal == 0): 
+                        V_spike = 0
+                        V_spike_index = 0
+                else:
+                        V_decoder = np.transpose(V_spike_internal) * threshold
+                        V_spike_index =np.argsort(V_decoder)[-1]
+                        V_spike = 1
+
+                return V_spike, V_spike_index 
+
+    def neuron_computation(self, st_1d_spike,threshold_value,input_num,output_num):
+            n1 = Neuron()
+            self.neuron_output = np.full((output_num,1),0) 
+
+            for i in range(0, output_num):    # event
+                decoder = np.full((output_num,1),0)
+                for j in range(0, input_num): #ST core number
+                        init = 0
+                        input_value = st_1d_spike[j,i]
+                        threshold = threshold_value[j,:]
+                        V_spike, V_spike_index = n1.neuron_SIMO(init, input_value, threshold, output_num,0, 0)
+                        if V_spike == 1:
+                            decoder[V_spike_index] +=1
+                        else:
+                            pass
+                self.neuron_output[i] = np.argsort(decoder[:,0])[-1]
+            print(self.neuron_output)                
+
+    def neuron_CANN(self, syn, init, stim, resting, thres):
+        V_internal =  init + syn + stim + resting
+        if V_internal >= thres:
+            V_spike = 1
+            V_internal = 0
+        else:
+            V_spike = 0 
+
+        return V_spike, V_internal  
+
+    def neuron_weight(self,neuro_num,type):
+        weight = np.full((neuro_num,neuro_num),0, dtype=float)
+        if type == 0:  #random
+            mean = 0
+            stdv = 0.1
+            for i in range(0, neuro_num):
+                seed(1)
+                weight[i,:] =  np.random.normal(mean, stdv, neuro_num)
+            return weight 
+        elif type == 1:  #sequence
+             np.fill_diagonal(weight , 1) 
+             weight = np.roll(weight,1,axis = 0) 
+        else:
+            pass
+
+        return weight   
+
+    def neuron_location(self, init, syn1, thres):  
+        V_internal =  init + syn1 
+        if V_internal >= thres:
+            V_spike = 1
+            V_internal = 0
+        else:
+            V_spike = 0 
+
+        return V_spike, V_internal  
+
+
+

README.md

@@ -0,0 +1,47 @@
+# SGF
+
+This version aims to demonstrate the key features of the SGF.
+
+## intall
+```
+conda create -n sgf python=3.7
+conda activate sgf
+```
+
+```
+conda install numpy=1.20 -y
+conda install opencv=3.4.2 -y
+conda install h5py=2.8.0 -y
+conda install matplotlib=3.3.4 -y
+conda install imageio=2.9.0 -y
+conda install scikit-learn=0.24 -y
+conda install xlwt=1.3.0 -y
+conda install xlrd=2.0.1 -y
+conda install xlutils=2.0.0 -y
+```
+
+```
+pip install opencv-python   
+pip install opencv-contrib-python 
+```
+
+## Dataset
+1. Download DVS Gesture dataset from https://research.ibm.com/interactive/dvsgesture/
+
+2. change the data_path and code_path in cfg.py
+
+3. Preprocess the event data into npy files. (This may take a few hours and need ~70GB free space on disk)
+
+```python
+cd ${your_path}/SGF_submmit
+python process_dvs_gesture.py
+python dvsgesture_t.py
+```
+
+## Run
+
+```python
+cd ${your_path}/SGF_submmit
+python main.py --train_test --iter 36 --selected_events 2+8+9+10+1_3_4+5_6+7
+```
+36 equals to training/test sample ratio 1.5:1.

STLayer.py

@@ -0,0 +1,97 @@
+import numpy as np
+from Neuron import Neuron
+import cfg
+
+class Spatiotemporal_Core(Neuron):
+    def __init__(self,  dataset, 
+                        c_space_num, s_thre, t_window, t_thre, if_st_neuron_clear=False):
+        super().__init__()           
+        self.c_space_num = c_space_num
+        self.s_thre = s_thre
+        self.t_window = t_window
+        self.t_thre = t_thre
+        self.dataset = dataset
+        self.if_st_neuron_clear = if_st_neuron_clear
+
+    def Spaceprocessing(self): 
+        neuron1 = Neuron(self.if_st_neuron_clear)      
+        #self.space= np.full((np.shape(self.dataset)[0], int(np.shape(self.dataset)[1]/self.c_space_num), int(np.shape(self.dataset)[2]/self.c_space_num), np.shape(self.dataset)[3]),0)
+        init= np.full((int(np.shape(self.dataset)[1]/self.c_space_num), int(np.shape(self.dataset)[2]/self.c_space_num), np.shape(self.dataset)[3]),0)
+        self.V_space= np.full((np.shape(self.dataset)[0], int(np.shape(self.dataset)[1]/self.c_space_num), int(np.shape(self.dataset)[2]/self.c_space_num), np.shape(self.dataset)[3]),0)
+        self.spaceneuron= np.full((np.shape(self.dataset)[0], int(np.shape(self.dataset)[1]/self.c_space_num), int(np.shape(self.dataset)[2]/self.c_space_num), np.shape(self.dataset)[3]),0)
+        for i in range(0, np.shape(self.dataset)[3]):  ##event
+           for j in range(0, np.shape(self.dataset)[0]):  ##frame
+              for k in range(0, np.shape(self.spaceneuron)[1]):
+                  for l in range(0, np.shape(self.spaceneuron)[2]):
+                     # self.space[j,k,l,i] = np.where(sum(sum(self.dataset[j, k:k+self.c_space_num, l:l+self.c_space_num, i]))>self.s_thre,1,0)
+                       syn = sum(sum(self.dataset[j, k*self.c_space_num:(k+1)*self.c_space_num, l*self.c_space_num:(l+1)*self.c_space_num, i]))
+                       V_spike, V = neuron1.neuron_ST(init[k,l,i] ,syn, self.s_thre,0,0)
+                       self.spaceneuron[j,k,l,i] = V_spike 
+                       init[k,l,i] = V
+                       self.V_space[j,k,l,i] = V                                                                  
+        return self.spaceneuron
+
+    def Temporalprocessing(self):
+        neuron1 = Neuron(self.if_st_neuron_clear)  
+        init= np.full((np.shape(self.dataset)[1], int(np.shape(self.dataset)[2]/self.c_space_num), np.shape(self.dataset)[3]),0)
+        self.V_time= np.full((np.shape(self.dataset)[0], int(np.shape(self.dataset)[1]/self.c_space_num), int(np.shape(self.dataset)[2]/self.c_space_num), np.shape(self.dataset)[3]),0)
+        #self.timeneuron = np.full((int(np.shape(self.spaceneuron)[0]/self.t_window), np.shape(self.spaceneuron)[1], np.shape(self.spaceneuron)[2], np.shape(self.spaceneuron)[3]),0)
+        self.timeneuron = np.full((int(np.shape(self.spaceneuron)[0]), np.shape(self.spaceneuron)[1], np.shape(self.spaceneuron)[2], np.shape(self.spaceneuron)[3]),0)
+        for i in range(0, np.shape(self.timeneuron)[3]): ## event
+            for j in range(0, np.shape(self.timeneuron)[0]):
+                for k in range(0, np.shape(self.spaceneuron)[1]):
+                    for l in range(0, np.shape(self.spaceneuron)[2]):
+                         #syn = sum(self.spaceneuron[j*self.t_window:(j+1)*self.t_window, k, l, i])
+                         syn = sum(self.spaceneuron[j:j+self.t_window, k, l, i])
+                         V_spike, V = neuron1.neuron_ST(init[k,l,i] ,syn, self.t_thre,0,0)
+                         self.timeneuron[j,k,l,i] = V_spike 
+                         init[k,l,i] = V
+                         self.V_time[j,k,l,i] = V  
+        return self.timeneuron  
+
+    def Stprocessing(self):  
+        #self.stcore = np.full((np.shape(self.spaceneuron)[0], np.shape(self.spaceneuron)[1], np.shape(self.spaceneuron)[2], np.shape(self.spaceneuron)[3]),0)
+        #timeneuron = np.full((np.shape(self.spaceneuron)[0], np.shape(self.spaceneuron)[1], np.shape(self.spaceneuron)[2], np.shape(self.spaceneuron)[3]),0)
+        #for i in range(0, np.shape(self.stcore)[3]): ## event        
+        #    for j in range(0, np.shape(self.timeneuron)[0]):
+        #        timeneuron[j*self.t_window :(j+1)*self.t_window,:,:,i]  = self.timeneuron[j,:,:,i]
+
+        #for i in range(0, np.shape(self.stcore)[3]): ## event 
+           # for j in range(0, np.shape(self.stcore)[0]):
+                #self.stcore[j,:,:,i] = self.spaceneuron[j,:,:,i]*self.timeneuron[j,:,:,i]
+        #self.stcore= self.spaceneuron * timeneuron 
+        self.stcore = self.spaceneuron * self.timeneuron
+
+
+    def stspike(self):
+        event_num = np.shape(self.dataset)[3]
+        self.ST_spike= np.full((np.shape(self.stcore)[1],np.shape(self.stcore)[2],np.shape(self.stcore)[3]),0) 
+        for i in range(0,event_num): ## event number
+                for j in range(0, np.shape(self.stcore)[1]):
+                    for k in range (0,np.shape(self.stcore)[2]):
+                        self.ST_spike[j,k,i] = sum(self.stcore[:,j,k,i])
+        self.ST_spike_1d = np.reshape(self.ST_spike,(np.shape(self.stcore)[1]*np.shape(self.stcore)[2],event_num))
+        # stspike = cfg.code_path + '/data/stspike'
+        # np.save(stspike, self.ST_spike_1d) 
+
+    def stthresholdpatt(self, num):
+        threshold_index = np.full((num, np.shape(self.ST_spike_1d)[1]),0)
+        threshold_value = np.full((num, np.shape(self.ST_spike_1d)[1]),0)
+        for i in range(0, np.shape(self.ST_spike_1d)[1]):  ## event
+                for j in range(0, num):    
+                    threshold_index[j,i] = np.argsort(self.ST_spike_1d[:,i])[-j-1]
+                    threshold_value[j,i] = self.ST_spike_1d[threshold_index[j,i],i]
+        SIMO_thres = cfg.code_path + '/data/reference_threshold'
+        np.save(SIMO_thres, threshold_value)            
+        return threshold_index, threshold_value 
+
+    def stfeature(self):
+        c_range = 60
+        st_num = np.shape(self.ST_spike_1d)[0]
+        threshold_index = np.full((int(st_num/c_range), np.shape(self.ST_spike_1d)[1]),0)
+        threshold_value = np.full((int(st_num/c_range), np.shape(self.ST_spike_1d)[1]),0)
+        for i in range(0, np.shape(self.ST_spike_1d)[1]):  ## event
+                            for j in range(0,int(st_num/c_range)): ##
+                                threshold_index[j,i] = np.argsort(self.ST_spike_1d[c_range*j:c_range*(j+1),i])[-1] + c_range*j
+                                threshold_value[j,i] = self.ST_spike_1d[threshold_index[j,i],i]
+        return threshold_index, threshold_value                        

SensingLayer.py

@@ -0,0 +1,34 @@
+import sys
+import os
+
+import numpy as np
+from BaseLayer import BaseLayer
+from logger import Logger
+
+
+class SensingLayer(BaseLayer):
+    def __init__(self, dataset, t_interval, thre, polarity):
+        #super(SensingLayer, self).__init__(input_channel, input_shape, output_channel)
+        self.t_interval = t_interval
+        self.thre = thre
+        self.dataset = dataset
+        self.polarity = polarity
+
+    def DiffSensing(self): 
+        dataset = self.dataset
+        self.sensing_output = np.full((dataset.videos[0].shape[0], dataset.videos[0].shape[1], dataset.videos[0].shape[2], dataset.event_num),0)
+        self.sensing_diff = np.full(( dataset.videos[0].shape[1], dataset.videos[0].shape[2]),0)
+        for i in range(0, dataset.event_num):   ## event number
+            Event_register = dataset.videos[i]
+            for j in range(0,dataset.videos[0].shape[0]-self.t_interval-1):  ## frame number 
+
+             self.sensing_diff= Event_register[j+self.t_interval] - Event_register[j]
+
+             if self.polarity == 0:
+                self.sensing_output[j,:,:,i] = np.where(np.logical_and(abs(self.sensing_diff) > self.thre
+                            , abs(self.sensing_diff) < 250), 100, 0)
+             elif self.polarity == 1: 
+                self.sensing_output[j,:,:,i] = np.where(abs(self.sensing_diff) < self.thre, 0,(np.where(self.sensing_diff>0, 1, -1)))                 
+             else:
+                break
+