progress_elapsed_time has been added as a new config option, during t…

…he main simulation eval loop if more than this much time has passed it will print out some progress information inside a generation, this is really useful for space search or large population where a single generation can take hours or even an entire day to see how progress is

the default is to write out the information every 5 minutes, if a generation takes less than this much time to run then no sub-generation progress will be written out

.gitignore

@@ -326,3 +326,4 @@ __pycache__/
 /Examples/MCMC/Dextran/nsga3_mcmc_6_emcee6_new
 /Examples/MCMC/NonBinding/fit_delay
 /Examples/MCMC/Linear/fit
+/Examples/Example1/Dextran/fit_nsga3_powell

CADETMatch/CADETMatch.pyproj

@@ -14,7 +14,7 @@
     <IsWindowsApplication>False</IsWindowsApplication>
     <InterpreterId>CondaEnv|CondaEnv|CADETMatch</InterpreterId>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
-    <CommandLineArguments>"C:\Users\kosh_000\Documents\Visual Studio 2017\Projects\CADETMatch\Examples\MCMC\Linear\fit\mcmc_refine\MCMC_linear.json" 6</CommandLineArguments>
+    <CommandLineArguments>"C:\Users\kosh_000\Documents\Visual Studio 2017\Projects\CADETMatch\Examples\Example1\Dextran\NSGA3_dextran.json" 6</CommandLineArguments>
     <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
     <InterpreterArguments>
     </InterpreterArguments>

CADETMatch/cache.py

@@ -133,6 +133,7 @@ def setup(self, json_path, load_plugins=True):
         self.stallGenerations = int(self.settings.get('stallGenerations', 10))
         self.stallCorrect = int(self.settings.get('stallCorrect', 5))
         self.progressCorrect = int(self.settings.get('progressCorrect', 5))
+        self.progress_elapsed_time = int(self.settings.get('progress_elapsed_time', 300))
 
         self.fullTrainingData = int(self.settings.get('fullTrainingData', 0))
 

CADETMatch/util.py

@@ -1053,7 +1053,16 @@ def process_population(toolbox, cache, population, fitnesses, writer, csvfile, h
 
     lookup = create_lookup(population)
 
-    for result in fitnesses:
+    last_time = time.time()
+    elapsed = cache.progress_elapsed_time
+
+    for idx, result in enumerate(fitnesses):
+
+        if (time.time() - last_time) > elapsed:
+            percent = idx / len(population)
+            multiprocessing.get_logger().info("Generation %s approximately %.1f %% complete with %s/%s done", generation, percent*100, idx, len(population))
+            last_time = time.time()
+
         fit, csv_line, results, individual = result
 
         ind = pop_lookup(lookup, individual)

Examples/Example1/Dextran/NSGA3_dextran.json

@@ -24,6 +24,7 @@
   "graphMetaTime": 600,
   "finalGradRefinement": 1,
   "fullTrainingData": 1,
+  "progress_elapsed_time": 2,
   "parameters": [
     {
       "transform": "norm_log",