Moved more search options to the retired folder

Added a lot more default options so that fewer settings have to be specified
Fixed some spelling errors
Bumped the version number
Still working on fixing some glitches with MCMC for the tutorial
create_spline now uses an optimizer to find the appropriate cutoff frequency for smoothing, will test on some experimental data to see how well this works out

CADETMatch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: CADETMatch
-Version: 0.4.7
+Version: 0.4.8
 Summary: CADETMatch is a parameter estimation and error modeling library for CADET
 Home-page: https://github.com/modsim/CADET-Match
 Author: William Heymann

CADETMatch.egg-info/SOURCES.txt

@@ -58,15 +58,8 @@ CADETMatch/search/__init__.py
 CADETMatch/search/graphSpace.py
 CADETMatch/search/mcmc.py
 CADETMatch/search/multistart.py
-CADETMatch/search/nsga2.py
-CADETMatch/search/nsga2_improved.py
 CADETMatch/search/nsga3.py
-CADETMatch/search/nsga3_improved.py
-CADETMatch/search/nsga3_old.py
-CADETMatch/search/nsga3_strategy.py
 CADETMatch/search/scoretest.py
-CADETMatch/search/spea2.py
-CADETMatch/search/spea2_improved.py
 CADETMatch/transform/__init__.py
 CADETMatch/transform/diameter.py
 CADETMatch/transform/keq.py

CADETMatch.egg-info/requires.txt

@@ -1,17 +1,18 @@
-joblib
-addict
-emcee
+joblib>=0.14.0
+addict>=2.2.1
+emcee>=2.2.1
 SAlib
-corner
-deap
-scoop
-psutil
-openpyxl
-numpy
-scipy
-matplotlib
-pandas
-h5py
-CADET
-seaborn
+corner>=2.0.1
+deap>=1.3.0
+scoop>=0.7
+psutil>=5.6.3
+openpyxl>=3.0.0
+numpy>=1.16.2
+scipy>=1.3.1
+matplotlib>=3.1.1
+pandas>=0.25.1
+h5py>=2.10.0
+CADET>=0.2
+seaborn>=0.9
 sklearn
+scikit-learn>=0.21

CADETMatch/CADETMatch.pyproj

@@ -14,7 +14,7 @@
     <IsWindowsApplication>False</IsWindowsApplication>
     <InterpreterId>CondaEnv|CondaEnv|CADETMatch</InterpreterId>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
-    <CommandLineArguments>"C:\Users\kosh_000\Documents\Visual Studio 2017\Projects\CADETMatch\Examples\Fractionation\NSGA3_combine_fraction.json"  2</CommandLineArguments>
+    <CommandLineArguments>"C:\Users\kosh_000\CADET\lessons\lesson_0\example.json"  2</CommandLineArguments>
     <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
     <InterpreterArguments>-m scoop -n 6</InterpreterArguments>
   </PropertyGroup>
@@ -257,16 +257,16 @@
     <Compile Include="search\multistart.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\nsga2_improved.py">
+    <Compile Include="search\retired\nsga2_improved.py">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="search\nsga3.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\nsga3_improved.py">
+    <Compile Include="search\retired\nsga3_improved.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\nsga3_strategy.py">
+    <Compile Include="search\retired\nsga3_strategy.py">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="search\retired\nsga2_mut.py">
@@ -281,16 +281,16 @@
     <Compile Include="search\retired\multiswarm.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\nsga2.py">
+    <Compile Include="search\retired\nsga2.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\nsga3_old.py">
+    <Compile Include="search\retired\nsga3_old.py">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="search\scoretest.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\spea2.py">
+    <Compile Include="search\retired\spea2.py">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="search\retired\spea2a.py">
@@ -314,7 +314,7 @@
     <Compile Include="search\retired\theta_nsga3.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="search\spea2_improved.py">
+    <Compile Include="search\retired\spea2_improved.py">
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="search\__init__.py">

CADETMatch/cache.py

@@ -176,9 +176,11 @@ def setupSettings(self):
 
             if 'CSV' in self.settings:
                 self.settings['csv'] = self.settings['CSV']
+            if 'csv' not in self.settings:
+                self.settings['csv'] = 'results.csv'
 
             self.settings['population'] = int(self.settings['population'])
-            self.settings['maxPopulation'] = int(self.settings.get('maxPopulation', self.settings['population'] * 10))
+            self.settings['maxPopulation'] = int(self.settings.get('maxPopulation', self.settings['population']))
             self.settings['minPopulation'] = int(self.settings.get('minPopulation', self.settings['population']))
 
 

CADETMatch/checkpoint_algorithms.py

@@ -18,7 +18,7 @@ def eaMuCommaLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, setting
 
     assert lambda_ >= mu, "lambda must be greater or equal to mu."
 
-    checkpointFile = Path(settings['resultsDirMisc'], settings['checkpointFile'])
+    checkpointFile = Path(settings['resultsDirMisc'], settings.get('checkpointFile', 'check'))
 
     sim_start = generation_start = time.time()
 
@@ -41,16 +41,16 @@ def eaMuCommaLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, setting
             cache.generationsOfProgress = cp['generationsOfProgress']
             cache.lastProgressGeneration = cp['lastProgressGeneration']
 
-            if cp['gradCheck'] > cache.settings['gradCheck']:
+            if cp['gradCheck'] > cache.settings.get('gradCheck', 1.0):
                 gradCheck = cp['gradCheck']
             else:
-                gradCheck = cache.settings['gradCheck']
+                gradCheck = cache.settings.get('gradCheck', 1.0)
 
         else:
             # Start a new evolution
             start_gen = 0    
 
-            gradCheck = settings['gradCheck']
+            gradCheck = settings.get('gradCheck', 1.0)
 
             halloffame = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit)
             meta_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit_meta)
@@ -115,7 +115,7 @@ def eaMuCommaLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, setting
             with checkpointFile.open('wb') as cp_file:
                 pickle.dump(cp, cp_file)
 
-            if avg >= settings['stopAverage'] or bestMin >= settings['stopBest'] or stalled:
+            if avg >= settings.get('stopAverage', 1.0) or bestMin >= settings.get('stopBest', 1.0) or stalled:
                 util.finish(cache)
                 util.graph_corner_process(cache, last=True)
                 return halloffame
@@ -133,7 +133,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, settings
     #from line_profiler import LineProfiler
     #profile = LineProfiler(search.spea2.selSPEA2)
 
-    checkpointFile = Path(settings['resultsDirMisc'], settings['checkpointFile'])
+    checkpointFile = Path(settings['resultsDirMisc'], settings.get('checkpointFile', 'check'))
 
     sim_start = generation_start = time.time()
 
@@ -156,15 +156,15 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, settings
             cache.generationsOfProgress = cp['generationsOfProgress']
             cache.lastProgressGeneration = cp['lastProgressGeneration']
 
-            if cp['gradCheck'] > cache.settings['gradCheck']:
+            if cp['gradCheck'] > cache.settings.get('gradCheck', 1.0):
                 gradCheck = cp['gradCheck']
             else:
-                gradCheck = cache.settings['gradCheck']
+                gradCheck = cache.settings.get('gradCheck', 1.0)
         else:
             # Start a new evolution
             start_gen = 0    
 
-            gradCheck = settings['gradCheck']
+            gradCheck = settings.get('gradCheck', 1.0)
 
             halloffame = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit)
             meta_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit_meta)
@@ -233,7 +233,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, settings
             with checkpointFile.open('wb') as cp_file:
                 pickle.dump(cp, cp_file)
 
-            if avg >= settings['stopAverage'] or bestMin >= settings['stopBest'] or stalled:
+            if avg >= settings.get('stopAverage', 1.0) or bestMin >= settings.get('stopBest', 1.0) or stalled:
                 break
                 #util.finish(cache)
                 #util.graph_corner_process(cache, last=True)
@@ -261,7 +261,7 @@ def nsga2(populationSize, ngen, cache, tools):
     cache.settings['maxPopulation'] = cache.settings['maxPopulation'] + (-cache.settings['maxPopulation'] % 4)
 
     cxpb = cache.settings['crossoverRate']
-    checkpointFile = Path(cache.settings['resultsDirMisc'], cache.settings['checkpointFile'])
+    checkpointFile = Path(cache.settings['resultsDirMisc'], cache.settings.get('checkpointFile', 'check'))
 
     path = Path(cache.settings['resultsDirBase'], cache.settings['csv'])
     result_data = {'input':[], 'output':[], 'output_meta':[], 'results':{}, 'times':{}, 'input_transform':[], 'input_transform_extended':[], 'strategy':[], 
@@ -282,10 +282,10 @@ def nsga2(populationSize, ngen, cache, tools):
             cache.generationsOfProgress = cp['generationsOfProgress']
             cache.lastProgressGeneration = cp['lastProgressGeneration']
 
-            if cp['gradCheck'] > cache.settings['gradCheck']:
+            if cp['gradCheck'] > cache.settings.get('gradCheck', 1.0):
                 gradCheck = cp['gradCheck']
             else:
-                gradCheck = cache.settings['gradCheck']
+                gradCheck = cache.settings.get('gradCheck', 1.0)
 
         else:
             # Start a new evolution
@@ -301,7 +301,7 @@ def nsga2(populationSize, ngen, cache, tools):
             halloffame = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit)
             meta_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit_meta)
             grad_hof = pareto.ParetoFront(similar=util.similar, similar_fit=util.similar_fit)
-            gradCheck = cache.settings['gradCheck']
+            gradCheck = cache.settings.get('gradCheck', 1.0)
 
 
         sim_start = generation_start = time.time()
@@ -402,7 +402,7 @@ def nsga2(populationSize, ngen, cache, tools):
             with checkpointFile.open('wb') as cp_file:
                 pickle.dump(cp, cp_file)
 
-            if avg >= cache.settings['stopAverage'] or bestMin >= cache.settings['stopBest'] or stalled:
+            if avg >= cache.settings.get('stopAverage', 1.0) or bestMin >= cache.settings.get('stopBest', 1.0) or stalled:
                 util.finish(cache)
                 util.graph_corner_process(cache, last=True)
                 return halloffame

CADETMatch/kde_generator.py

@@ -172,7 +172,6 @@ def synthetic_error_simulation(json_path):
         load_settings = experiment['load']
         error_slope_settings = experiment['error_slope']
         error_base_settings = experiment['error_base']
-        base_settings = experiment['base']
         count_settings = experiment['count']
         experimental_csv = experiment['experimental_csv']
         units = experiment['units']

CADETMatch/match.py

@@ -48,7 +48,7 @@ def main(path=None, map_function=None):
             setup(cache, json_path)
             hof = evo.run(cache)
 
-    continue_mcmc(cache)
+    continue_mcmc(cache, map_function)
 
     if "repeat" in cache.settings:
         repeat = int(cache.settings['repeat'])
@@ -266,20 +266,20 @@ def find_percentile(cache):
 
         return lb_trans, ub_trans
 
-def continue_mcmc(cache):
+def continue_mcmc(cache, map_function):
     if cache.continueMCMC:
         lb, ub = find_percentile(cache)
 
         json_path = util.setupMCMC(cache, lb, ub)
         scoop.logger.info(json_path)
 
-        setup(cache, json_path)
+        setup(cache, json_path, map_function)
 
         hof = evo.run(cache)
 
 if __name__ == "__main__":
     start = time.time()
     main()
-    scoop.logger.info('Sysem has finished')
+    scoop.logger.info('System has finished')
     scoop.logger.info("The total runtime was %s seconds" % (time.time() - start))
     sys.exit()

CADETMatch/scores/dextranShape.py

@@ -69,10 +69,6 @@ def setup(sim, feature, selectedTimes, selectedValues, CV_time, abstol):
     exp_data_zero = numpy.zeros(len(selectedValues))
     exp_data_zero[min_index:max_index+1] = selectedValues[min_index:max_index+1]
 
-    exp_spline = util.create_spline(selectedTimes, exp_data_zero).derivative(1)
-
-    [high, low] = util.find_peak(selectedTimes, exp_spline(selectedTimes))
-
     temp['min_time'] = feature['start']
     temp['max_time'] = feature['stop']
     temp['max_value'] = max_value

CADETMatch/search/mcmc.py

@@ -330,7 +330,7 @@ def write_interval(interval, cache, checkpoint, checkpointFile, train_chain, run
 def run(cache, tools, creator):
     "run the parameter estimation"
     random.seed()
-    checkpointFile = Path(cache.settings['resultsDirMisc'], cache.settings['checkpointFile'])
+    checkpointFile = Path(cache.settings['resultsDirMisc'], cache.settings.get('checkpointFile', 'check'))
     checkpoint = getCheckPoint(checkpointFile,cache)
 
     burn_seq = checkpoint.get('burn_seq', [])

CADETMatch/search/nsga3.py

@@ -12,10 +12,11 @@ def run(cache, tools, creator):
     random.seed()
     parameters = len(cache.MIN_VALUE)
 
-    populationSize=parameters * cache.settings['population']
-    CXPB = cache.settings['crossoverRate']
+    populationSize=parameters * cache.settings.get('population', 100)
+    CXPB = cache.settings.get('crossoverRate', 1.0)
+    MUTPB = cache.settings.get('mutationRate', 1.0)
 
-    totalGenerations = parameters * cache.settings['generations']
+    totalGenerations = parameters * cache.settings.get('generations', 1000)
 
     pop = cache.toolbox.population(n=populationSize)
 
@@ -26,7 +27,7 @@ def run(cache, tools, creator):
                               mu=populationSize, 
                               lambda_=populationSize, 
                               cxpb=CXPB, 
-                              mutpb=cache.settings['mutationRate'],
+                              mutpb=MUTPB,
                               ngen=totalGenerations,
                               settings=cache.settings,
                               tools=tools,
@@ -56,14 +57,9 @@ def setupDEAP(cache, fitness, grad_fitness, grad_search, map_function, creator,
 
     cache.toolbox.register("mate", tools.cxSimulatedBinaryBounded, eta=30.0, low=cache.MIN_VALUE, up=cache.MAX_VALUE)
 
-    #if cache.adaptive:
-    #    cache.toolbox.register("mutate", util.mutationBoundedAdaptive, low=cache.MIN_VALUE, up=cache.MAX_VALUE, indpb=1.0/len(cache.MIN_VALUE))
-    #    cache.toolbox.register("force_mutate", util.mutationBoundedAdaptive, low=cache.MIN_VALUE, up=cache.MAX_VALUE, indpb=1.0/len(cache.MIN_VALUE))
-    #else:
     cache.toolbox.register("mutate", tools.mutPolynomialBounded, eta=20.0, low=cache.MIN_VALUE, up=cache.MAX_VALUE, indpb=1.0/len(cache.MIN_VALUE))
     cache.toolbox.register("force_mutate", tools.mutPolynomialBounded, eta=20.0, low=cache.MIN_VALUE, up=cache.MAX_VALUE, indpb=1.0/len(cache.MIN_VALUE))
 
-    #cache.toolbox.register("select", nsga3_selection.sel_nsga_iii)
     cache.toolbox.register("select", tools.selNSGA3, ref_points=ref_points)
     cache.toolbox.register("evaluate", fitness, json_path=cache.json_path)
     cache.toolbox.register("evaluate_grad", grad_fitness, json_path=cache.json_path)

CADETMatch/util.py

@@ -50,7 +50,15 @@ def find_extreme(seq):
         return [0, 0]
 
 def create_spline(times, values):
-    return scipy.interpolate.PchipInterpolator(times, smoothing(times, values, 11))
+
+    def goal(cutoff):
+        smooth_values = smoothing(times, values, cutoff[0])
+        sse = numpy.sum((values-smooth_values)**2)
+        return sse
+
+    result = scipy.optimize.differential_evolution(goal, bounds = [(1e-5,0.499),], polish=False)
+    cutoff = result.x[0]
+    return scipy.interpolate.PchipInterpolator(times, smoothing(times, values, cutoff))
 
 def get_times_values(simulation, target, selected = None):
 
@@ -126,32 +134,18 @@ def averageFitness(offspring, cache):
     bestMin = -sys.float_info.max
     bestProd = -sys.float_info.max
 
-    #scoop.logger.info(offspring)
-
     for i in offspring:
-        #scoop.logger.info('idx: %s  value:%s', i, i.fitness.values)
         total += sum(i.fitness.values)
         number += len(i.fitness.values)
         bestMin = max(bestMin, min(i.fitness.values))
         bestProd = max(bestProd, product_score(i.fitness.values))
-    #scoop.logger.info('number: %s', number)
     result = [total/number, bestMin, bestProd]
 
     return result
 
-def smoothing(times, values, filter_length=None):
-    #temporarily get rid of smoothing for debugging
-    #return values
-    #filter length must be odd, set to 10% of the feature size and then make it odd if necesary
-
-    #if filter_length is None:
-    #    filter_length = int(.1 * len(values))
-    #    if filter_length % 2 == 0:
-    #        filter_length += 1
-    #return scipy.signal.savgol_filter(values, filter_length, 3)
-
+def smoothing(times, values, cutoff_frequency=0.005):
     N  = 3    # Filter order
-    Wn = 0.005 # Cutoff frequency
+    Wn = cutoff_frequency # Cutoff frequency
     dt = times[1] - times[0]
     Fs = 1 / dt 
     B, A = scipy.signal.butter(N, Wn, output='ba', fs=Fs, btype='lowpass')