updated example creation with changes to the cadet python api

smoothing now uses bessel instead of butter filtering. On square pulses the butter filter was causing a ringing effect which gets worse as the pulse is sharper or the sample rate is lower

bumped version number
depend on CADET-Python

smoothing now resamples to 5000 points (based on experimental and synthetic test suite) if there are more than 5000 points. This solves a problem on the forums where long duration runs at very high sampling rates where causing problems.

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "python.pythonPath": "C:\\Users\\kosh_000\\anaconda3\\envs\\addict_test\\python.exe",
+    "python.pythonPath": "C:\\Users\\kosh_000\\anaconda3\\envs\\cadet_run\\python.exe",
     "python.languageServer": "Default",
     "python.analysis.typeCheckingMode": "strict",
     "python.analysis.diagnosticMode": "workspace",

CADETMatch.pyproj

@@ -11,7 +11,7 @@
     <OutputPath>.</OutputPath>
     <ProjectTypeGuids>{888888a0-9f3d-457c-b088-3a5042f75d52}</ProjectTypeGuids>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
-    <InterpreterId>CondaEnv|CondaEnv|addict_test</InterpreterId>
+    <InterpreterId>CondaEnv|CondaEnv|cadet_devel</InterpreterId>
     <CommandLineArguments>"F:\match_dll\experiments\single\dextran.json" 6</CommandLineArguments>
     <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
     <SuppressConfigureTestFrameworkPrompt>true</SuppressConfigureTestFrameworkPrompt>

CADETMatch/create_example_config.py

@@ -10,7 +10,7 @@
 
 def create_experiments(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -145,7 +145,7 @@ def create_scores(defaults):
 def create_slicing(defaults):
     "create all the scores that have the same config except for the score name"
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -200,7 +200,7 @@ def create_slicing(defaults):
 def create_fractionation(defaults):
     "create the ceiling"
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -280,7 +280,7 @@ def create_fractionation(defaults):
 def create_ceiling(defaults):
     "create the ceiling"
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -320,7 +320,7 @@ def create_ceiling(defaults):
 def create_shared_scores(defaults):
     "create all the scores that have the same config except for the score name"
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -385,7 +385,7 @@ def create_shared_scores(defaults):
 
 def create_search(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
 
     parameter1 = Dict()
@@ -618,7 +618,7 @@ def create_transforms(defaults):
 
 def create_experiments_index(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -666,7 +666,7 @@ def create_experiments_index(defaults):
 
 def create_experiments_linear_exp(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -727,7 +727,7 @@ def create_experiments_linear_exp(defaults):
 
 def create_experiments_cstr(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -785,7 +785,7 @@ def create_experiments_cstr(defaults):
 
 def create_experiments_linear(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -856,7 +856,7 @@ def create_experiments_linear(defaults):
 
 def create_transforms_non(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population
@@ -922,7 +922,7 @@ def create_transforms_non(defaults):
 
 def create_transforms_dextran(defaults):
     config = Dict()
-    config.CADETPath = Cadet.cadet_runner.cadet_path
+    config.CADETPath = Cadet.cadet_path
     config.resultsDir = 'results'
     config.searchMethod = 'NSGA3'
     config.population = defaults.population

CADETMatch/smoothing.py

@@ -39,14 +39,7 @@ def refine_butter(times, values, x, y, fs, start):
     def goal(crit_fs):
         crit_fs = 10.0 ** crit_fs[0]
         try:
-            sos = scipy.signal.butter(
-                butter_order,
-                crit_fs,
-                btype="lowpass",
-                analog=False,
-                fs=fs,
-                output="sos",
-            )
+            sos = scipy.signal.bessel(butter_order, crit_fs, btype="lowpass", analog=False, fs=fs, output="sos", norm="delay")
         except ValueError:
             return 1e6
         low_passed = scipy.signal.sosfiltfilt(sos, values)
@@ -161,9 +154,7 @@ def find_butter(times, values):
 
     for i in numpy.logspace(-6, ub_l, 50):
         try:
-            sos = scipy.signal.butter(
-                butter_order, i, btype="lowpass", analog=False, fs=fs, output="sos"
-            )
+            sos = scipy.signal.bessel(butter_order, i, btype="lowpass", analog=False, fs=fs, output="sos", norm="delay")
             low_passed = scipy.signal.sosfiltfilt(sos, values)
 
             filters.append(i)
@@ -184,9 +175,7 @@ def smoothing_filter_butter(times, values, crit_fs):
         return values
     fs = 1.0 / (times[1] - times[0])
 
-    sos = scipy.signal.butter(
-        butter_order, crit_fs, btype="lowpass", analog=False, fs=fs, output="sos"
-    )
+    sos = scipy.signal.bessel(butter_order, crit_fs, btype="lowpass", analog=False, fs=fs, output="sos", norm="delay")
     low_passed = scipy.signal.sosfiltfilt(sos, values)
     return low_passed
 
@@ -246,7 +235,7 @@ def load_data(name, cache):
 
 def find_smoothing_factors(times, values, name, cache):
     times, values = resample(times, values)
-    min = 1e-2
+    min = 1e-3
 
     s, crit_fs, crit_fs_der = load_data(name, cache)
 
@@ -272,33 +261,38 @@ def find_smoothing_factors(times, values, name, cache):
     knots.append(len(spline.get_knots()))
     all_s.append(min)
 
-    # This limits to 1e-14 max smoothness which is way beyond anything normal
-    for i in range(1, 200):
-        s = min / (1.1 ** i)
-        with warnings.catch_warnings():
-            warnings.filterwarnings("error")
+    max_knots = 600
 
-            try:
-                spline = scipy.interpolate.UnivariateSpline(
-                    times, values_filter, s=s, k=5, ext=3
-                )
-                knots.append(len(spline.get_knots()))
-                all_s.append(s)
+    if len(spline.get_knots()) < max_knots:  #if we already need more knots than the max knots at min accuracy there is no reason to check
+        # This limits to 1e-14 max smoothness which is way beyond anything normal
+        for i in range(1, 200):
+            s = min / (1.1 ** i)
+            with warnings.catch_warnings():
+                warnings.filterwarnings("error")
 
-                if len(spline.get_knots()) > 600:
-                    break
+                try:
+                    spline = scipy.interpolate.UnivariateSpline(
+                        times, values_filter, s=s, k=5, ext=3
+                    )
+                    knots.append(len(spline.get_knots()))
+                    all_s.append(s)
 
-            except Warning:
-                multiprocessing.get_logger().info("caught a warning for %s %s", name, s)
-                break
+                    if len(spline.get_knots()) > max_knots:
+                        break
 
-    knots = numpy.array(knots)
-    all_s = numpy.array(all_s)
+                except Warning:
+                    multiprocessing.get_logger().info("caught a warning for %s %s", name, s)
+                    break
 
-    s, s_knots = find_L(all_s, knots)
+        knots = numpy.array(knots)
+        all_s = numpy.array(all_s)
 
-    if s is not None:
-        s, s_knots = refine_smooth(times, values_filter, all_s, knots, s, name)
+        s, s_knots = find_L(all_s, knots)
+
+        if s is not None:
+            s, s_knots = refine_smooth(times, values_filter, all_s, knots, s, name)
+    else:
+        s = min
 
     spline, factor = create_spline(times, values, crit_fs, s)
 
@@ -429,26 +423,31 @@ def butter(times, values, crit_fs_der):
     return values_filter
 
 
-def resample(times, values):
-    diff_times = times[1:] - times[:-1]
-    max_time = numpy.max(diff_times)
-    min_time = numpy.min(diff_times)
-    per = (max_time - min_time) / min_time
+def resample(times, values, max_samples=5000):
+    if len(times) > max_samples:
+        times_resample = numpy.linspace(times[0], times[-1], max_samples)
+        spline_resample = scipy.interpolate.InterpolatedUnivariateSpline(times, values, k=5, ext=3)
+        values_resample = spline_resample(times_resample)
 
-    if per > 0.01:
-        # time step is not consistent, resample the time steps to a uniform grid based on the smallest time step size seen
-        times_resample = numpy.arange(times[0], times[-1], min_time)
-        times_resample[-1] = times[-1]
-        diff_times = times_resample[1:] - times_resample[:-1]
+        return times_resample, values_resample
+    else: 
+        diff_times = times[1:] - times[:-1]
         max_time = numpy.max(diff_times)
         min_time = numpy.min(diff_times)
         per = (max_time - min_time) / min_time
 
-        spline_resample = scipy.interpolate.InterpolatedUnivariateSpline(
-            times, values, k=5, ext=3
-        )
-        values_resample = spline_resample(times_resample)
+        if per > 0.01:
+            # time step is not consistent, resample the time steps to a uniform grid based on the smallest time step size seen
+            #but not more samples than max_samples
+            times_resample = numpy.arange(times[0], times[-1], min_time)
 
-        return times_resample, values_resample
-    else:
-        return times, values
+            if len(times_resample) > max_samples:
+               times_resample = numpy.linspace(times[0], times[-1], max_samples) 
+
+            times_resample[-1] = times[-1]
+            spline_resample = scipy.interpolate.InterpolatedUnivariateSpline(times, values, k=5, ext=3)
+            values_resample = spline_resample(times_resample)
+
+            return times_resample, values_resample
+        else:
+            return times, values

CADETMatch/version.py

@@ -18,5 +18,5 @@
 __email__ = "[email protected]"
 __license__ = "GNU General Public License v3 (GPLv3)"
 __copyright__ = "2020 %s" % __author__
-__version__ = "0.6.50"
+__version__ = "0.7.0"
 __uri__ = "https://github.com/modsim/CADET-Match"

setup.py

@@ -29,7 +29,7 @@
           'matplotlib>=3.2.1',
           'pandas>=1.0.5',
           'h5py>=2.10.0',
-          'CADET>=0.10',
+          'CADET-Python>=0.11',
           'seaborn>=0.10.1',
           'scikit-learn>=0.23.1',
           'importlib-metadata>=1.7.0',