push current work from cluster computing

.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+data/pickles/*
+reports/*

FlowCollection.py

@@ -5,6 +5,7 @@
 import pandas as pd
 
 import pdb
+from decorators import timeit
 
 class FlowCollection:
 
@@ -59,7 +60,8 @@ def plot_3D(self, xaxis, yaxis, zaxis):
         for metaexp in self.metaexps:
             metaexp.plot_3D(xaxis, yaxis, zaxis)
 
-    def gateXD(self, settings):
+    @timeit
+    def gateXD(self, settings, **kwargs):
         self.max_est_components = 10
 
         for metaexp in self.metaexps:

PipelineHelper.py

@@ -6,8 +6,11 @@
 from CompensationReport import CompensationReport
 from FlowCollection import FlowCollection
 import utility as util
+from decorators import timeit
 import pdb
 import os
+import datetime
+import json
 
 def compensate(expt_tubes, settings, perform=True, report=True, title=""):
 
@@ -97,7 +100,8 @@ def QC(expt_tubes, settings, perform=True, report=True):
     exp_uncomp = compensation['uncompensated']
     return(exp_comp, exp_uncomp)
 
-def executeXDGating(settings, save=True, load=False):
+@timeit
+def executeXDGating(settings, save=True, load=False, **kwargs):
     if not load:
         surface_tubes, expt_tubes = util.read_surface_tube_table(settings['TUBE_FILE'])
         exp_comp, exp_uncomp = QC(expt_tubes, settings, perform = True, report = True)
@@ -109,10 +113,19 @@ def executeXDGating(settings, save=True, load=False):
         fc_list = []
         for gr in groups:
             fc_list.append(FlowCollection(exp_comp, iter_param, gr, settings))   
-
+
+
+        fc_log = {}
         for fc in fc_list:
-            fc.gateXD(settings) 
+            fc.gateXD(settings, log_time = fc_log) 
             #fc.plot_3D('PE-A', 'FITC-A', 'APC-A')
+
+        # write each individual run to an output file
+        to_date = datetime.datetime.today().strftime('%Y%m%d')
+        fc_output_log_file = "FC_log_{}.txt".format(to_date)
+        with open(fc_output_log_file, 'a') as out_file:
+            out_file.write(json.dumps(fc_log))
+
     else:
         with open('pickles/controls_only/surface_tubes.pkl', 'rb') as handle:
             surface_tubes = pickle.load(handle)

config/exp_controls_only_test_config.yml

@@ -0,0 +1,8 @@
+EXP_NAME: controls_only_test
+TUBE_FILE: data/exp_controls_only_test_tube_table.txt
+SIGMA34: 2
+SIGMA41: 3
+THRESHOLD34: 500
+THRESHOLD41: 1000
+BASE41: fitcapc
+COMPENSATION: True

decorators.py

@@ -0,0 +1,26 @@
+import time
+
+def timeit(method):
+    def timed(*args, **kw):
+        """
+        Stores the number of milliseconds it took to run a function.
+        Output is the result dict -> keys are function names, 
+        values are dicts -> in which keys are iteration number and values are ms.
+        """
+        ts = time.time()
+        result = method(*args, **kw)
+        te = time.time()
+        if 'log_time' in kw:
+            name = kw.get('log_name', method.__name__.upper())
+
+            if name in kw['log_time']:
+                iter_number = len(kw['log_time'][name].keys())
+                kw['log_time'][name][iter_number] = int((te - ts) * 1000)
+            else:
+                kw['log_time'][name] = {}
+                kw['log_time'][name][0] = int((te - ts) * 1000)
+        else:
+            print '%r  %2.2f ms' % \
+                  (method.__name__, (te - ts) * 1000)
+        return result
+    return timed

run_pipeline.py

@@ -3,9 +3,6 @@
 #matplotlib.use("Agg")
 import matplotlib.pyplot as plt
 
-import cytoflow as flow
-import timeit
-
 import MetadataHelper
 import utility as util
 
@@ -17,18 +14,18 @@
 from LiveDeadReport import LiveDeadReport
 from SurfaceReport import SurfaceReport
 from PipelineHelper import executeXDGating
+import datetime
+import json
 
 if __name__ == '__main__':
-    flow.set_default_scale("logicle")
     plt.style.use('astroml')
-    DATE = '05092018'
-    config_name = 'controls_only'
+    DATE = '11142018'
+    config_name = 'controls_only_test'
     settings_path = "config/exp_{0!s}_config.yml".format(config_name)
     fs = False
 
 
     print("Running exp {0!s}...".format(config_name))
-    start = timeit.default_timer()
     print("Loading settings...")
     settings = MetadataHelper.loadSettings(settings_path)
     print("Settings loaded!")
@@ -37,8 +34,16 @@
     all_stats = MetadataHelper.loadExistingFlowGates(pickle_path)
     all_stats = False
     if all_stats is False:
-        print("No exisiting gating found, generating new gates...")
-        fc_list = executeXDGating(settings, save=True, load=False)
+        print("No existing gating found, generating new gates...")
+
+        pipeline_log = {}
+        fc_list = executeXDGating(settings, save=True, load=False, log_time = pipeline_log)
+
+        to_date = datetime.datetime.today().strftime('%Y%m%d')
+        pipeline_output_log_file = "pipeline_log_{}.txt".format(to_date)
+        with open(pipeline_output_log_file, 'a') as out_file:
+            out_file.write(json.dumps(pipeline_log))
+
         print("Gating completed!")
 
         all_stats = util.get_allstats_table(fc_list, settings)
@@ -154,6 +159,3 @@
     new_agg_df[header]= new_agg_df['index'].apply(pd.Series)
     new_agg_df.dropna(axis=1, how='all', inplace=True)
     new_agg_df.to_csv('{}/{}_stats.tsv'.format(tables_directory,settings['EXP_NAME']), sep='\t', index=False)
-
-    stop = timeit.default_timer()
-    print(stop-start, " to process exp ", str(config_name))

utility.py

@@ -14,7 +14,6 @@
 
 import cytoflow as flow
 import cytoflow.utility as cutil
-
 def get_allstats_table(fc_list, settings):
     conditions_n = len(fc_list)
     all_stats = pd.DataFrame()