fractionation now use fraction_csv to find the csv file

all csv entries in the json file are now lowercase
updated everything that looks for CSV to look for csv instead
fractionationSlide now use spline for smooth sliding and smoother time offets, based on tests this improves convergence

CADETMatch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: CADETMatch
-Version: 0.4
+Version: 0.4.1
 Summary: CADETMatch is a parameter estimation and error modeling library for CADET
 Home-page: https://github.com/modsim/CADET-Match
 Author: William Heymann

CADETMatch.egg-info/SOURCES.txt

@@ -39,32 +39,9 @@ CADETMatch.egg-info/top_level.txt
 CADETMatch/scores/__init__.py
 CADETMatch/scores/absoluteHeight.py
 CADETMatch/scores/absoluteTime.py
-CADETMatch/scores/breakthrough.py
-CADETMatch/scores/breakthroughCross.py
-CADETMatch/scores/breakthroughHybrid.py
-CADETMatch/scores/breakthroughHybrid2.py
 CADETMatch/scores/curve.py
-CADETMatch/scores/derivative_similarity.py
-CADETMatch/scores/derivative_similarity_cross.py
-CADETMatch/scores/derivative_similarity_hybrid.py
-CADETMatch/scores/derivative_similarity_hybrid2.py
-CADETMatch/scores/derivative_similarity_hybrid2_spline.py
-CADETMatch/scores/dextran.py
-CADETMatch/scores/dextran3.py
-CADETMatch/scores/dextranAngle.py
-CADETMatch/scores/dextranHybrid.py
-CADETMatch/scores/dextranHybrid2.py
-CADETMatch/scores/dextranHybrid2_spline.py
-CADETMatch/scores/dextranQuad.py
 CADETMatch/scores/dextranSSE.py
 CADETMatch/scores/dextranShape.py
-CADETMatch/scores/dextranTest.py
-CADETMatch/scores/dextran_original.py
-CADETMatch/scores/dextran_spline.py
-CADETMatch/scores/fractionation.py
-CADETMatch/scores/fractionationCombine.py
-CADETMatch/scores/fractionationMeanVariance.py
-CADETMatch/scores/fractionationMoment.py
 CADETMatch/scores/fractionationSSE.py
 CADETMatch/scores/fractionationSlide.py
 CADETMatch/scores/logsse.py
@@ -74,15 +51,7 @@ CADETMatch/scores/shapeDecaySimple.py
 CADETMatch/scores/shapeOnly.py
 CADETMatch/scores/shapeSimple.py
 CADETMatch/scores/similarity.py
-CADETMatch/scores/similarityCross.py
-CADETMatch/scores/similarityCrossDecay.py
 CADETMatch/scores/similarityDecay.py
-CADETMatch/scores/similarityHybrid.py
-CADETMatch/scores/similarityHybrid2.py
-CADETMatch/scores/similarityHybrid2_spline.py
-CADETMatch/scores/similarityHybridDecay.py
-CADETMatch/scores/similarityHybridDecay2.py
-CADETMatch/scores/similarityHybridDecay2Spline.py
 CADETMatch/scores/sse.py
 CADETMatch/scores/width.py
 CADETMatch/search/__init__.py
@@ -92,8 +61,8 @@ CADETMatch/search/multistart.py
 CADETMatch/search/nsga2.py
 CADETMatch/search/nsga2_improved.py
 CADETMatch/search/nsga3.py
-CADETMatch/search/nsga3_deap.py
 CADETMatch/search/nsga3_improved.py
+CADETMatch/search/nsga3_old.py
 CADETMatch/search/nsga3_strategy.py
 CADETMatch/search/scoretest.py
 CADETMatch/search/spea2.py

CADETMatch/CADETMatch.pyproj

@@ -14,10 +14,9 @@
     <IsWindowsApplication>False</IsWindowsApplication>
     <InterpreterId>CondaEnv|CondaEnv|CADETMatch</InterpreterId>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
-    <CommandLineArguments>"F:\DS9\fractionation\NSGA3_binding_fractions.json"  2</CommandLineArguments>
+    <CommandLineArguments>"F:\temp\cadet_tutorial\lesson_5\multiple_components\example_nsga3_fractionation.json"  2</CommandLineArguments>
     <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
-    <InterpreterArguments>
-    </InterpreterArguments>
+    <InterpreterArguments>-m scoop -n 6</InterpreterArguments>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <DebugSymbols>true</DebugSymbols>

CADETMatch/cache.py

@@ -169,6 +169,9 @@ def setupSettings(self):
         with settings_file.open() as json_data:
             self.settings = json.load(json_data)
 
+            if 'CSV' in self.settings:
+                self.settings['csv'] = self.settings['CSV']
+
             self.settings['population'] = int(self.settings['population'])
             self.settings['maxPopulation'] = int(self.settings.get('maxPopulation', self.settings['population'] * 10))
             self.settings['minPopulation'] = int(self.settings.get('minPopulation', self.settings['population']))
@@ -305,11 +308,15 @@ def setupExperiment(self, experiment, sim=None, dataFromSim=0):
             else:
                 CV_time = (area * length) / flow
 
+        #force CSV to lowercase
+        if 'CSV' in experiment:
+            experiment['csv'] = experiment['CSV']
+
         if dataFromSim:
             temp['time'], temp['value'] = get_times_values(sim, {'isotherm':experiment['isotherm']})
 
-        elif 'CSV' in experiment:
-            data = numpy.loadtxt(experiment['CSV'], delimiter=',')
+        elif 'csv' in experiment:
+            data = numpy.loadtxt(experiment['csv'], delimiter=',')
 
             temp['time'] = data[:, 0]
             temp['value'] = data[:, 1]
@@ -330,11 +337,15 @@ def setupExperiment(self, experiment, sim=None, dataFromSim=0):
 
             temp[featureName] = {}
 
+            #switch to lower case
             if 'CSV' in feature:
+                feature['csv'] = feature['CSV']
+
+            if 'csv' in feature:
                 if dataFromSim:
                     temp[featureName]['time'], temp[featureName]['value'] = get_times_values(sim, {'isotherm':feature['isotherm']})
                 else:
-                    dataLocal = numpy.loadtxt(feature['CSV'], delimiter=',')
+                    dataLocal = numpy.loadtxt(feature['csv'], delimiter=',')
                     temp[featureName]['time'] = dataLocal[:, 0]
                     temp[featureName]['value'] = dataLocal[:, 1]
             else:

CADETMatch/checkpoint_algorithms.py

@@ -22,7 +22,7 @@ def eaMuCommaLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, setting
 
     sim_start = generation_start = time.time()
 
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     result_data = {'input':[], 'output':[], 'output_meta':[], 'results':{}, 'times':{}, 'input_transform':[], 'input_transform_extended':[], 'strategy':[], 
                    'mean':[], 'confidence':[]}
     with path.open('a', newline='') as csvfile:
@@ -133,7 +133,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, settings
 
     sim_start = generation_start = time.time()
 
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     result_data = {'input':[], 'output':[], 'output_meta':[], 'results':{}, 'times':{}, 'input_transform':[], 'input_transform_extended':[], 'strategy':[], 
                    'mean':[], 'confidence':[]}
     with path.open('a', newline='') as csvfile:
@@ -255,7 +255,7 @@ def nsga2(populationSize, ngen, cache, tools):
     cxpb = cache.settings['crossoverRate']
     checkpointFile = Path(cache.settings['resultsDirMisc'], cache.settings['checkpointFile'])
 
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     result_data = {'input':[], 'output':[], 'output_meta':[], 'results':{}, 'times':{}, 'input_transform':[], 'input_transform_extended':[], 'strategy':[], 
                    'mean':[], 'confidence':[]}
     with path.open('a', newline='') as csvfile:

CADETMatch/kde_generator.py

@@ -398,12 +398,12 @@ def setupReferenceResult(cache):
     for experiment in cache.settings['experiments']:
         temp_exp = dict(experiment)
 
-        if 'CSV' in temp_exp:
-            temp_exp['data'] = numpy.loadtxt(temp_exp['CSV'], delimiter=',')
+        if 'csv' in temp_exp:
+            temp_exp['data'] = numpy.loadtxt(temp_exp['csv'], delimiter=',')
 
         for feature in temp_exp['features']:
-            if 'CSV' in feature:
-                feature['data'] = numpy.loadtxt(feature['CSV'], delimiter=',')
+            if 'csv' in feature:
+                feature['data'] = numpy.loadtxt(feature['csv'], delimiter=',')
         temp.append(temp_exp)
 
     return temp
@@ -472,7 +472,7 @@ def pump_delay(cache, data, pump_delay_time=None):
     #delay = quantize_delay(delay[0], interval)
 
     #data[:,1] = score.roll(data[:,1], delay)
-    data[:,1] = score.roll_spline(data[:,0], data[:,1], -delay)
+    data[:,1] = score.roll_spline(data[:,0], data[:,1], delay)
 
     return delay
 

CADETMatch/match.py

@@ -144,7 +144,7 @@ def createDirectories(cache, json_path):
         pass
 
 def createCSV(cache):
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     if not path.exists():
         with path.open('w', newline='') as csvfile:
             writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)

CADETMatch/score.py

@@ -23,7 +23,7 @@ def roll_spline(times, values, shift):
 
     spline = scipy.interpolate.InterpolatedUnivariateSpline(times, values, ext=1)
 
-    times_new = times + shift
+    times_new = times - shift
 
     values_new = spline(times_new)
 

CADETMatch/scores/fractionationSSE.py

@@ -68,7 +68,7 @@ def run(sim_data, feature):
 
 def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     temp = {}
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     start = numpy.array(data.iloc[:, 0])

CADETMatch/scores/fractionationSlide.py

@@ -14,7 +14,7 @@
 
 
 def goal(offset, frac_exp, sim_data_time, sim_data_value, start, stop):
-    sim_data_value = score.roll(sim_data_value, int(offset))
+    sim_data_value = score.roll_spline(sim_data_time, sim_data_value, offset)
     frac_sim = util.fractionate(start, stop, sim_data_time, sim_data_value)
     return numpy.sum((frac_exp-frac_sim)**2)
 
@@ -32,21 +32,6 @@ def searchRange(times, start_frac, stop_frac, CV_time):
     searchIndexStop = numpy.argmax( times[times <= searchStop])
     return searchIndexStart, searchIndexStop
 
-def rollRange(times, sim_value, searchIndexStart, searchIndexStop):
-    peakMaxIndex = numpy.argmax(sim_value)
-    peakMaxTime = times[peakMaxIndex]
-    peakMaxValue = sim_value[peakMaxIndex]
-
-    above1 = sim_value < 0.1 * peakMaxValue
-    beforePeak = times < peakMaxTime
-
-    search = sim_value[beforePeak & above1]
-    searchMax = numpy.argmax(search)
-
-    rollLeft = searchIndexStart - searchMax
-    rollRight = searchIndexStop - searchMax
-    return rollLeft, rollRight, searchMax
-
 def run(sim_data, feature):
     simulation = sim_data['simulation']
     timeFunc = feature['timeFunc']
@@ -77,14 +62,13 @@ def run(sim_data, feature):
         exp_values = numpy.array(data[str(component)])
         sim_value = simulation.root.output.solution[feature['unit']]["solution_outlet_comp_%03d" % component]
 
-        rollLeft, rollRight, searchMax = rollRange(times, sim_value, searchIndexStart, searchIndexStop)
-
         bounds = find_bounds(times, sim_value)
         result = scipy.optimize.differential_evolution(goal, bounds = [bounds,], 
                                                        args = (exp_values, times, sim_value, start, stop))
 
-        time_offset = times[int(abs(result.x[0]))]
-        sim_data_value = score.roll(sim_value, int(result.x[0]))
+        time_offset = abs(result.x[0])
+        sim_data_value = score.roll_spline(times, sim_value, result.x[0])
+
         fracOffset = util.fractionate(start, stop, times, sim_data_value)
 
         value_score = value_func(max(fracOffset))
@@ -121,7 +105,7 @@ def find_bounds(times, values):
 
 def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     temp = {}
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     headers = data.columns.values.tolist()
@@ -152,7 +136,7 @@ def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     return temp
 
 def headers(experimentName, feature):
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     data_headers = data.columns.values.tolist()

CADETMatch/scores/obsolete/dextranQuad.py

@@ -42,10 +42,6 @@ def run(sim_data, feature):
 
     pearson, diff_time = score.pearson_spline(exp_time_values, sim_data_zero, exp_data_zero)
 
-    #sse_diff, diff_time = score.sse_spline(exp_time_values, sim_data_zero, exp_data_zero)
-
-    #sim_data_zero = score.roll_spline(exp_time_values, sim_data_zero, diff_time)
-
     p = poly_fit(exp_time_values, sim_data_zero)
     if p is None:
         return settings.failure

CADETMatch/scores/obsolete/fractionation.py

@@ -62,7 +62,7 @@ def run(sim_data, feature):
 
 def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     temp = {}
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     start = numpy.array(data.iloc[:, 0])
@@ -92,7 +92,7 @@ def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     return temp
 
 def headers(experimentName, feature):
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
     #remove first two columns since those are the start and stop times
     cols = cols - 2

CADETMatch/scores/obsolete/fractionationCombine.py

@@ -73,7 +73,7 @@ def run(sim_data, feature):
 
 def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     temp = {}
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     start = numpy.array(data.iloc[:, 0])
@@ -108,7 +108,7 @@ def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     return temp
 
 def headers(experimentName, feature):
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
     #remove first two columns since those are the start and stop times
     cols = cols - 2

CADETMatch/scores/obsolete/fractionationMeanVariance.py

@@ -58,7 +58,7 @@ def run(sim_data, feature):
 
 def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     temp = {}
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     headers = data.columns.values.tolist()
@@ -97,7 +97,7 @@ def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     return temp
 
 def headers(experimentName, feature):
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     data_headers = data.columns.values.tolist()

CADETMatch/scores/obsolete/fractionationMoment.py

@@ -60,7 +60,7 @@ def run(sim_data, feature):
 
 def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     temp = {}
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     headers = data.columns.values.tolist()
@@ -103,7 +103,7 @@ def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     return temp
 
 def headers(experimentName, feature):
-    data = pandas.read_csv(feature['csv'])
+    data = pandas.read_csv(feature['fraction_csv'])
     rows, cols = data.shape
 
     data_headers = data.columns.values.tolist()

CADETMatch/search/graphSpace.py

@@ -34,7 +34,7 @@ def run(cache, tools, creator):
     meta_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit_meta)
     grad_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit)
 
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     with path.open('a', newline='') as csvfile:
         writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
 

CADETMatch/search/mcmc.py

@@ -358,7 +358,7 @@ def run(cache, tools, creator):
         scoop.logger.info("loading kde")
         kde, kde_scaler = kde_generator.getKDE(cache)
 
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     with path.open('a', newline='') as csvfile:
         writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
 

CADETMatch/search/multistart.py

@@ -32,7 +32,7 @@ def run(cache, tools, creator):
     meta_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit_meta)
     grad_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit)
 
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     with path.open('a', newline='') as csvfile:
         writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
 

CADETMatch/search/scoretest.py

@@ -11,7 +11,7 @@
 
 def run(cache, tools, creator):
     "run the parameter estimation"
-    path = Path(cache.settings['resultsDirBase'], cache.settings['CSV'])
+    path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     with path.open('a', newline='') as csvfile:
         writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
         pop = cache.toolbox.population(n=0)