From e13ed5db29d46022b98f69f6ca6f0be7ab83fc89 Mon Sep 17 00:00:00 2001 From: "richardsomers1998@gmail.com" Date: Mon, 27 Nov 2023 09:42:26 +0000 Subject: [PATCH 01/60] splitting linear regression into linear and polynomial estimators --- causal_testing/testing/estimators.py | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index 4e56562c..e0b0f754 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -272,9 +272,9 @@ def estimate_unit_odds_ratio(self) -> float: """ model = self._run_logistic_regression(self.df) return np.exp(model.params[self.treatment]) + - -class LinearRegressionEstimator(Estimator): +class PolynomialRegressionEstimator(Estimator): """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a linear combination of parameters and functions of the variables (note these functions need not be linear). """ @@ -287,6 +287,7 @@ def __init__( control_value: float, adjustment_set: set, outcome: str, + degree: int, df: pd.DataFrame = None, effect_modifiers: dict[Variable:Any] = None, formula: str = None, @@ -304,7 +305,7 @@ def __init__( self.formula = formula else: terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) - self.formula = f"{outcome} ~ {'+'.join(terms)}" + self.formula = f"{outcome} ~ cr({'+'.join(terms)}, df={degree})" for term in self.effect_modifiers: self.adjustment_set.add(term) @@ -439,6 +440,28 @@ def _get_confidence_intervals(self, model, treatment): return [ci_low, ci_high] +class LinearRegressionEstimator(PolynomialRegressionEstimator): + + def __init__( + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[Variable:Any] = None, + formula: str = None, + alpha: float = 0.05, + ): + super().__init__(treatment, treatment_value, control_value, adjustment_set, outcome, 1, df, effect_modifiers, formula, alpha) + + if formula is None: + terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) + self.formula = f"{outcome} ~ {'+'.join(terms)}" + + class InstrumentalVariableEstimator(Estimator): """ Carry out estimation using instrumental variable adjustment rather than conventional adjustment. This means we do From 81d5b59c7d6a991babc81016207e076df7aa079f Mon Sep 17 00:00:00 2001 From: "richardsomers1998@gmail.com" Date: Mon, 27 Nov 2023 09:52:37 +0000 Subject: [PATCH 02/60] black --- causal_testing/testing/estimators.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index e0b0f754..04b605ee 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -272,11 +272,11 @@ def estimate_unit_odds_ratio(self) -> float: """ model = self._run_logistic_regression(self.df) return np.exp(model.params[self.treatment]) - + class PolynomialRegressionEstimator(Estimator): - """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a linear - combination of parameters and functions of the variables (note these functions need not be linear). + """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a polynomial + combination of parameters and functions of the variables (note these functions need not be polynomial). """ def __init__( @@ -441,6 +441,9 @@ def _get_confidence_intervals(self, model, treatment): class LinearRegressionEstimator(PolynomialRegressionEstimator): + """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a linear + combination of parameters and functions of the variables (note these functions need not be linear). + """ def __init__( # pylint: disable=too-many-arguments @@ -455,7 +458,9 @@ def __init__( formula: str = None, alpha: float = 0.05, ): - super().__init__(treatment, treatment_value, control_value, adjustment_set, outcome, 1, df, effect_modifiers, formula, alpha) + super().__init__( + treatment, treatment_value, control_value, adjustment_set, outcome, 1, df, effect_modifiers, formula, alpha + ) if formula is None: terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) From 5d56060c1e156b5696562b631256fbc0639821b5 Mon Sep 17 00:00:00 2001 From: "richardsomers1998@gmail.com" Date: Mon, 27 Nov 2023 10:20:57 +0000 Subject: [PATCH 03/60] Outline of testing approach --- .../testing/causal_surrogate_assisted.py | 33 +++++++++++++++++++ causal_testing/testing/estimators.py | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 causal_testing/testing/causal_surrogate_assisted.py diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py new file mode 100644 index 00000000..7c1f899f --- /dev/null +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -0,0 +1,33 @@ +from causal_testing.data_collection.data_collector import DataCollector +from causal_testing.testing.causal_test_suite import CausalTestSuite + + +class CausalSurrogateAssistedTestCase: + + def __init__( + self, + test_suite: CausalTestSuite, + # search_alogrithm: SearchAlgorithm, + # simulator: Simulator, + ): + self.test_suite = test_suite + + def execute(self, data_collector: DataCollector, max_executions: int = 200): + df = data_collector.collect_data() + + for _i in range(max_executions): + + # Build surrogate models based on df + + # Define surrogate model fitness function + self.test_suite.execute_test_suite(df) + + # Multiobjective Metaheuristics to find candidate test case + + # Run candidate test case gainst simulator + + # Validate fault + + # If not valid, add to df and loop + + pass \ No newline at end of file diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index 04b605ee..c06597a4 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -275,8 +275,8 @@ def estimate_unit_odds_ratio(self) -> float: class PolynomialRegressionEstimator(Estimator): - """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a polynomial - combination of parameters and functions of the variables (note these functions need not be polynomial). + """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a + polynomial combination of parameters and functions of the variables (note these functions need not be polynomial). """ def __init__( From ea2bc5205c9844510e55b3211d3143cca0d40f39 Mon Sep 17 00:00:00 2001 From: "richardsomers1998@gmail.com" Date: Mon, 27 Nov 2023 11:15:37 +0000 Subject: [PATCH 04/60] cleanup + search algo interface --- .../testing/causal_surrogate_assisted.py | 27 +++++++++++++------ causal_testing/testing/estimators.py | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 7c1f899f..34cbb19b 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -1,15 +1,15 @@ from causal_testing.data_collection.data_collector import DataCollector from causal_testing.testing.causal_test_suite import CausalTestSuite +from causal_testing.testing.estimators import Estimator class CausalSurrogateAssistedTestCase: - def __init__( - self, - test_suite: CausalTestSuite, - # search_alogrithm: SearchAlgorithm, - # simulator: Simulator, - ): + self, + test_suite: CausalTestSuite, + # search_alogrithm: SearchAlgorithm, + # simulator: Simulator, + ): self.test_suite = test_suite def execute(self, data_collector: DataCollector, max_executions: int = 200): @@ -24,10 +24,21 @@ def execute(self, data_collector: DataCollector, max_executions: int = 200): # Multiobjective Metaheuristics to find candidate test case - # Run candidate test case gainst simulator + # Run candidate test case against simulator # Validate fault # If not valid, add to df and loop - pass \ No newline at end of file + pass + + +class SearchAlgorithm: + def __init__(self, test_suite: CausalTestSuite, surrogate_models: list[Estimator]): + self.__generate_fitness_functions(test_suite) + + def __generate_fitness_functions(self, test_suite): + pass + + def search(self): + pass diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index c06597a4..73e88876 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -275,7 +275,7 @@ def estimate_unit_odds_ratio(self) -> float: class PolynomialRegressionEstimator(Estimator): - """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a + """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a polynomial combination of parameters and functions of the variables (note these functions need not be polynomial). """ From 88fb2fe49b2fc164d75506851047c6205244e27d Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Tue, 28 Nov 2023 09:39:56 +0000 Subject: [PATCH 05/60] Testing approach outline with object interfaces --- .../testing/causal_surrogate_assisted.py | 72 ++++++++++++------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 34cbb19b..b3d38d72 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -1,44 +1,62 @@ -from causal_testing.data_collection.data_collector import DataCollector +from causal_testing.data_collection.data_collector import ObservationalDataCollector from causal_testing.testing.causal_test_suite import CausalTestSuite from causal_testing.testing.estimators import Estimator +from dataclasses import dataclass -class CausalSurrogateAssistedTestCase: - def __init__( - self, - test_suite: CausalTestSuite, - # search_alogrithm: SearchAlgorithm, - # simulator: Simulator, - ): - self.test_suite = test_suite - def execute(self, data_collector: DataCollector, max_executions: int = 200): - df = data_collector.collect_data() +class SearchAlgorithm: + def generate_fitness_functions(self, test_suite: CausalTestSuite, surrogate_models: list[Estimator]): + pass + + def search(self) -> list: + pass + + +@dataclass +class SimulationResult: + data: dict + fault: bool - for _i in range(max_executions): - # Build surrogate models based on df +class Simulator: + def startup(self, **kwargs): + pass - # Define surrogate model fitness function - self.test_suite.execute_test_suite(df) + def shutdown(self, **kwargs): + pass - # Multiobjective Metaheuristics to find candidate test case + def run_with_config(self, configuration) -> SimulationResult: + pass - # Run candidate test case against simulator - # Validate fault +class CausalSurrogateAssistedTestCase: + def __init__( + self, + test_suite: CausalTestSuite, + search_alogrithm: SearchAlgorithm, + simulator: Simulator, + ): + self.test_suite = test_suite + self.search_algorithm = search_alogrithm + self.simulator = simulator - # If not valid, add to df and loop + def execute(self, data_collector: ObservationalDataCollector, max_executions: int = 200): + data_collector.collect_data() - pass + for _i in range(max_executions): + surrogate_models = [] + self.search_algorithm.generate_fitness_functions(self.test_suite, surrogate_models) + candidate_test_case = self.search_algorithm.search() -class SearchAlgorithm: - def __init__(self, test_suite: CausalTestSuite, surrogate_models: list[Estimator]): - self.__generate_fitness_functions(test_suite) + self.simulator.startup() + test_result = self.simulator.run_with_config(candidate_test_case) + self.simulator.shutdown() - def __generate_fitness_functions(self, test_suite): - pass + if test_result.fault: + return test_result + else: + data_collector.data = data_collector.data.append(test_result.data, ignore_index=True) - def search(self): - pass + print("No fault found") From 35fe098a0d2098d5b6853b07706f68bfcf9af777 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Tue, 28 Nov 2023 11:15:11 +0000 Subject: [PATCH 06/60] basic surrogate + fitness function generation --- .../testing/causal_surrogate_assisted.py | 70 +++++++++++++++---- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index b3d38d72..889618a1 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -1,15 +1,17 @@ from causal_testing.data_collection.data_collector import ObservationalDataCollector -from causal_testing.testing.causal_test_suite import CausalTestSuite -from causal_testing.testing.estimators import Estimator +from causal_testing.specification.causal_specification import CausalSpecification +from causal_testing.testing.base_test_case import BaseTestCase +from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator from dataclasses import dataclass +from typing import Callable class SearchAlgorithm: - def generate_fitness_functions(self, test_suite: CausalTestSuite, surrogate_models: list[Estimator]): + def generate_fitness_functions(self, specification: CausalSpecification, surrogate_models: list[Estimator]): pass - def search(self) -> list: + def search(self, fitness_functions) -> list: pass @@ -33,22 +35,26 @@ def run_with_config(self, configuration) -> SimulationResult: class CausalSurrogateAssistedTestCase: def __init__( self, - test_suite: CausalTestSuite, + specification: CausalSpecification, search_alogrithm: SearchAlgorithm, simulator: Simulator, ): - self.test_suite = test_suite + self.specification = specification self.search_algorithm = search_alogrithm self.simulator = simulator - def execute(self, data_collector: ObservationalDataCollector, max_executions: int = 200): + def execute( + self, + data_collector: ObservationalDataCollector, + max_executions: int = 200, + custom_data_aggregator: Callable[[dict, dict], dict] = None, + ): data_collector.collect_data() for _i in range(max_executions): - - surrogate_models = [] - self.search_algorithm.generate_fitness_functions(self.test_suite, surrogate_models) - candidate_test_case = self.search_algorithm.search() + surrogate_models = self.generate_surrogates(self.specification, data_collector) + fitness_functions = self.search_algorithm.generate_fitness_functions(surrogate_models) + candidate_test_case = self.search_algorithm.search(fitness_functions) self.simulator.startup() test_result = self.simulator.run_with_config(candidate_test_case) @@ -57,6 +63,46 @@ def execute(self, data_collector: ObservationalDataCollector, max_executions: in if test_result.fault: return test_result else: - data_collector.data = data_collector.data.append(test_result.data, ignore_index=True) + if custom_data_aggregator is not None: + data_collector.data = custom_data_aggregator(data_collector.data, test_result.data) + else: + data_collector.data = data_collector.data.append(test_result.data, ignore_index=True) print("No fault found") + + def generate_surrogates( + self, specification: CausalSpecification, data_collector: ObservationalDataCollector + ) -> list[PolynomialRegressionEstimator]: + surrogate_models = [] + + for u, v in specification.causal_dag.edges: + base_test_case = BaseTestCase(u, v) + minimal_adjustment_set = specification.causal_dag.identification(base_test_case) + + surrogate = PolynomialRegressionEstimator(u, 0, 0, minimal_adjustment_set, v, 4, df=data_collector.data) + surrogate_models.append(surrogate) + + return surrogate_models + + def generate_fitness_functions( # TODO Move this into a GA specific search algorithm (Built for PyGAD fitness functions) + self, surrogate_models: list[PolynomialRegressionEstimator], delta=0.05 + ) -> Callable[[list[float], int], float]: + fitness_functions = [] + + for surrogate in surrogate_models: + + def fitness_function(solution, idx): + surrogate.control_value = solution[0] - delta + surrogate.treatment_value = solution[0] + delta + + adjustment_dict = dict() + for i, adjustment in enumerate(surrogate.adjustment_set): + adjustment_dict[adjustment] = solution[i + 1] + + ate = surrogate.estimate_ate_calculated(adjustment_dict)[0] + + return ate # TODO Need a way here of assessing if high or low ATE is correct + + fitness_functions.append(fitness_function) + + return fitness_functions From d54f6e25328e4585b6214a52ce08679e42e80f29 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Tue, 28 Nov 2023 11:15:32 +0000 Subject: [PATCH 07/60] example code for simulator interfacing --- examples/apsdigitaltwin/README.md | 0 examples/apsdigitaltwin/apsdigitaltwin.py | 52 +++++ examples/apsdigitaltwin/util/model.py | 249 ++++++++++++++++++++++ 3 files changed, 301 insertions(+) create mode 100644 examples/apsdigitaltwin/README.md create mode 100644 examples/apsdigitaltwin/apsdigitaltwin.py create mode 100644 examples/apsdigitaltwin/util/model.py diff --git a/examples/apsdigitaltwin/README.md b/examples/apsdigitaltwin/README.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/apsdigitaltwin/apsdigitaltwin.py b/examples/apsdigitaltwin/apsdigitaltwin.py new file mode 100644 index 00000000..f75d6c2c --- /dev/null +++ b/examples/apsdigitaltwin/apsdigitaltwin.py @@ -0,0 +1,52 @@ +from causal_testing.testing.causal_surrogate_assisted import SimulationResult, Simulator +from examples.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label + + +class APSDigitalTwinSimulator(Simulator): + def __init__(self, constants) -> None: + super().__init__() + + self.constants = constants + + def run_with_config(self, configuration) -> SimulationResult: + min_bg = 200 + max_bg = 0 + end_bg = 0 + end_cob = 0 + end_iob = 0 + open_aps_output = 0 + violation = False + + open_aps = OpenAPS() + model_openaps = Model([configuration[1], 0, 0, configuration[0], configuration[2]], self.constants) + for t in range(1, 121): + if t % 5 == 1: + rate = open_aps.run(model_openaps.history, output_file=f"./openaps_temp", faulty=True) + if rate == -1: + violation = True + open_aps_output += rate + for j in range(5): + model_openaps.add_intervention(t + j, i_label, rate / 5.0) + model_openaps.update(t) + + min_bg = min(min_bg, model_openaps.history[-1][g_label]) + max_bg = max(max_bg, model_openaps.history[-1][g_label]) + + end_bg = model_openaps.history[-1][g_label] + end_cob = model_openaps.history[-1][s_label] + end_iob = model_openaps.history[-1][i_label] + + data = { + "start_bg": configuration[0], + "start_cob": configuration[1], + "start_iob": configuration[2], + "end_bg": end_bg, + "end_cob": end_cob, + "end_iob": end_iob, + "hypo": min_bg, + "hyper": max_bg, + "open_aps_output": open_aps_output, + } + + return SimulationResult(data, violation) + diff --git a/examples/apsdigitaltwin/util/model.py b/examples/apsdigitaltwin/util/model.py new file mode 100644 index 00000000..621ed698 --- /dev/null +++ b/examples/apsdigitaltwin/util/model.py @@ -0,0 +1,249 @@ +import pandas as pd +import matplotlib.pyplot as plt +import os +import platform +import subprocess +import shutil +import json +from datetime import datetime + +s_label = 'Stomach' +j_label = 'Jejunum' +l_label = 'Ileum' +g_label = 'Blood Glucose' +i_label = 'Blood Insulin' + +class OpenAPS: + + def __init__(self, recorded_carbs = None, autosense_ratio = 1.0, test_timestamp = "2023-01-01T18:00:00-00:00", profile_path = None) -> None: + self.shell = "Windows" in platform.system() + oref_help = subprocess.check_output(["oref0", "--help"], shell=self.shell) + + if "oref0 help - this message" not in str(oref_help): + print("ERROR - oref0 not installed") + exit(1) + + if profile_path == None: + self.profile_path = os.environ["profile_path"] + else: + self.profile_path = profile_path + self.basal_profile_path = os.environ["basal_profile_path"] + self.autosense_ratio = autosense_ratio + self.test_timestamp = test_timestamp + self.epoch_time = int(datetime.strptime(test_timestamp, "%Y-%m-%dT%H:%M:%S%z").timestamp() * 1000) + self.pump_history = [] + self.recorded_carbs = recorded_carbs + + def run(self, model_history, output_file = None, faulty = False): + if output_file == None: + output_file = './openaps_temp' + + if not os.path.exists(output_file): + os.mkdir(output_file) + + time_since_start = len(model_history) - 1 + current_epoch = self.epoch_time + 60000 * time_since_start + current_timestamp = datetime.fromtimestamp(current_epoch / 1000).strftime("%Y-%m-%dT%H:%M:%S%z") + + basal_history = [] + temp_basal = '{}' + if model_history[0][i_label] > 0: + basal_history.append(f'{{"timestamp":"{datetime.fromtimestamp(self.epoch_time/1000).strftime("%Y-%m-%dT%H:%M:%S%z")}"' + + f',"_type":"Bolus","amount":{model_history[0][i_label] / 1000},"duration":0}}') + + for idx, (rate, duration, timestamp) in enumerate(self.pump_history): + basal_history.append(f'{{"timestamp":"{timestamp}","_type":"TempBasal","temp":"absolute","rate":{str(rate)}}}') + basal_history.append(f'{{"timestamp":"{timestamp}","_type":"TempBasalDuration","duration (min)":{str(duration)}}}') + if idx == len(self.pump_history) - 1: + if faulty: + temp_basal = f'{{"duration": {duration}, "temp": "absolute", "rate": {str(rate)}}}' + else: + temp_basal_epoch = int(datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ").timestamp() * 1000) + if (current_epoch - temp_basal_epoch) / 60 <= duration: + temp_basal = f'{{"duration": {duration}, "temp": "absolute", "rate": {str(rate)}}}' + basal_history.reverse() + + glucose_history = [] + carb_history = [] + for idx, time_step in enumerate(model_history): + if idx % 5 == 0: + bg_level = int(time_step[g_label]) + new_time_epoch = self.epoch_time + idx * 60000 + new_time_stamp = datetime.fromtimestamp(new_time_epoch/1000).strftime("%Y-%m-%dT%H:%M:%S%z") + glucose_history.append(f'{{"date":{new_time_epoch},"dateString":"{new_time_stamp}","sgv":{bg_level},' + + f'"device":"fakecgm","type":"sgv","glucose":{bg_level}}}') + + if idx == 0: + if time_step[s_label] > 0: + if self.recorded_carbs == None: + carb_history.append(f'{{"enteredBy":"fakecarbs","carbs":{time_step[s_label]},"created_at":"{self.test_timestamp}","insulin": null}}') + else: + carb_history.append(f'{{"enteredBy":"fakecarbs","carbs":{self.recorded_carbs},"created_at":"{self.test_timestamp}","insulin": null}}') + + else: + carb_diff = time_step[s_label] - model_history[idx - 1][s_label] + if carb_diff > 0: + new_time_epoch = self.epoch_time + idx * 60000 + new_time_stamp = datetime.fromtimestamp(new_time_epoch/1000).strftime("%Y-%m-%dT%H:%M:%S%z") + carb_history.append(f'{{"enteredBy":"fakecarbs","carbs":{time_step[s_label]},"created_at":"{new_time_stamp}","insulin":null}}') + glucose_history.reverse() + carb_history.reverse() + + self.__make_file_and_write_to(f"{output_file}/clock.json", f'"{current_timestamp}-00:00"') + self.__make_file_and_write_to(f"{output_file}/autosens.json", '{"ratio":' + str(self.autosense_ratio) + '}') + self.__make_file_and_write_to(f"{output_file}/pumphistory.json", "[" + ','.join(basal_history) + "]") + self.__make_file_and_write_to(f"{output_file}/glucose.json", "[" + ','.join(glucose_history) + "]") + self.__make_file_and_write_to(f"{output_file}/carbhistory.json", "[" + ','.join(carb_history) + "]") + self.__make_file_and_write_to(f"{output_file}/temp_basal.json", temp_basal) + + iob_output = subprocess.check_output([ + "oref0-calculate-iob", + f"{output_file}/pumphistory.json", + self.profile_path, + f"{output_file}/clock.json", + f"{output_file}/autosens.json" + ], shell=self.shell, stderr=subprocess.DEVNULL).decode("utf-8") + self.__make_file_and_write_to(f"{output_file}/iob.json", iob_output) + + meal_output = subprocess.check_output([ + "oref0-meal", + f"{output_file}/pumphistory.json", + self.profile_path, + f"{output_file}/clock.json", + f"{output_file}/glucose.json", + self.basal_profile_path, + f"{output_file}/carbhistory.json" + ], shell=self.shell, stderr=subprocess.DEVNULL).decode("utf-8") + self.__make_file_and_write_to(f"{output_file}/meal.json", meal_output) + + basal_res = subprocess.run([ + "oref0-determine-basal", + f"{output_file}/iob.json", + f"{output_file}/temp_basal.json", + f"{output_file}/glucose.json", + self.profile_path, + "--auto-sens", + f"{output_file}/autosens.json", + "--meal", + f"{output_file}/meal.json", + "--microbolus", + "--currentTime", + str(current_epoch) + ], shell=self.shell, capture_output=True, text=True) + + if "Warning: currenttemp running but lastTemp from pumphistory ended" in basal_res.stdout: + shutil.rmtree(output_file, ignore_errors=True) + return -1 + + self.__make_file_and_write_to(f"{output_file}/suggested.json", basal_res.stdout) + + json_output = open(f"{output_file}/suggested.json") + data = json.load(json_output) + + rate = data["rate"] if "rate" in data else 0 + if rate != 0: + duration = data["duration"] + timestamp = data["deliverAt"] + self.pump_history.append((rate, duration, timestamp)) + + shutil.rmtree(output_file, ignore_errors=True) + + return 1000 * rate / 60.0 + + def __make_file_and_write_to(self, file_path, contents): + file = open(file_path, "w") + file.write(contents) + + def __update_suggested(self, suggested_json, file_path): + if not os.path.isfile(file_path): + new_file = open(file_path, "w") + new_file.write("[]") + new_file.close() + + list_json = None + with open(file_path) as output_file: + list_json = json.load(output_file) + list_json.append(json.loads(suggested_json)) + + with open(file_path, 'w') as writing_json: + json.dump(list_json, writing_json, indent=4, separators=(',',': ')) + +class Model: + def __init__(self, starting_vals, constants): + self.interventions = dict() + + self.history = [] + self.history.append({'step': 0, + s_label: starting_vals[0], + j_label: starting_vals[1], + l_label: starting_vals[2], + g_label: starting_vals[3], + i_label: starting_vals[4]}) + + self.kjs = constants[0] + self.kgj = constants[1] + self.kjl = constants[2] + self.kgl = constants[3] + self.kxg = constants[4] + self.kxgi = constants[5] + self.kxi = constants[6] + + self.tau = constants[7] + self.klambda = constants[8] + self.eta = constants[9] + + self.gprod0 = constants[10] + self.kmu = constants[11] + self.gb = starting_vals[3] + + self.gprod_limit = (self.klambda * self.gb + self.gprod0 * (self.kmu + self.gb)) / (self.klambda + self.gprod0) + + + def update(self, t): + old_s = self.history[t-1][s_label] + old_j = self.history[t-1][j_label] + old_l = self.history[t-1][l_label] + old_g = self.history[t-1][g_label] + old_i = self.history[t-1][i_label] + + new_s = old_s - (old_s * self.kjs) + + new_j = old_j + (old_s * self.kjs) - (old_j * self.kgj) - (old_j * self.kjl) + + phi = 0 if t < self.tau else self.history[t - self.tau][j_label] + new_l = old_l + (phi * self.kjl) - (old_l * self.kgl) + + g_prod = (self.klambda * (self.gb - old_g))/(self.kmu + (self.gb - old_g)) + self.gprod0 if old_g <= self.gprod_limit else 0 + new_g = old_g - (self.kxg + self.kxgi * old_i) * old_g + g_prod + self.eta * (self.kgj * old_j + self.kgl * old_l) + + new_i = old_i - (old_i * self.kxi) + + if t in self.interventions: + for intervention in self.interventions[t]: + if intervention[0] == s_label: + new_s += intervention[1] + elif intervention[0] == i_label: + new_i += intervention[1] + + timestep = {'step': t, s_label: new_s, j_label: new_j, l_label: new_l, g_label: new_g, i_label: new_i} + self.history.append(timestep) + + return [old_s, new_s, old_j, new_j, old_l, new_l, old_g, new_g, old_i, new_i, g_prod, + self.kjs, self.kgj, self.kjl, self.kgl, self.kxg, self.kxgi, self.kxi, self.tau, + self.klambda, self.eta, self.gprod0, self.kmu, self.gb] + + def add_intervention(self, timestep: int, variable: str, intervention: float): + if timestep not in self.interventions: + self.interventions[timestep] = list() + + self.interventions[timestep].append((variable, intervention)) + + def plot(self, timesteps = -1): + if timesteps == -1: + df = pd.DataFrame(self.history) + df.plot('step', [s_label, j_label, l_label, g_label, i_label]) + plt.show() + else: + df = pd.DataFrame(self.history[:timesteps]) + df.plot('step', [s_label, j_label, l_label, g_label, i_label]) + plt.show() \ No newline at end of file From ab169a02b35ae346c5fcbcb468dfa0adaa14a801 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Tue, 28 Nov 2023 11:39:38 +0000 Subject: [PATCH 08/60] GA search base class --- .../testing/causal_surrogate_assisted.py | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 889618a1..954a7da2 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -14,6 +14,38 @@ def generate_fitness_functions(self, specification: CausalSpecification, surroga def search(self, fitness_functions) -> list: pass +class GeneticSearchAlgorithm(SearchAlgorithm): + def __init__(self, delta = 0.05) -> None: + super().__init__() + + self.delta = delta + + def generate_fitness_functions( + self, surrogate_models: list[PolynomialRegressionEstimator] + ) -> Callable[[list[float], int], float]: + fitness_functions = [] + + for surrogate in surrogate_models: + + def fitness_function(solution, idx): + surrogate.control_value = solution[0] - self.delta + surrogate.treatment_value = solution[0] + self.delta + + adjustment_dict = dict() + for i, adjustment in enumerate(surrogate.adjustment_set): + adjustment_dict[adjustment] = solution[i + 1] + + ate = surrogate.estimate_ate_calculated(adjustment_dict)[0] + + return ate # TODO Need a way here of assessing if high or low ATE is correct + + fitness_functions.append(fitness_function) + + return fitness_functions + + def search(self, fitness_functions) -> list: + pass # TODO Implement GA search + @dataclass class SimulationResult: @@ -83,26 +115,3 @@ def generate_surrogates( surrogate_models.append(surrogate) return surrogate_models - - def generate_fitness_functions( # TODO Move this into a GA specific search algorithm (Built for PyGAD fitness functions) - self, surrogate_models: list[PolynomialRegressionEstimator], delta=0.05 - ) -> Callable[[list[float], int], float]: - fitness_functions = [] - - for surrogate in surrogate_models: - - def fitness_function(solution, idx): - surrogate.control_value = solution[0] - delta - surrogate.treatment_value = solution[0] + delta - - adjustment_dict = dict() - for i, adjustment in enumerate(surrogate.adjustment_set): - adjustment_dict[adjustment] = solution[i + 1] - - ate = surrogate.estimate_ate_calculated(adjustment_dict)[0] - - return ate # TODO Need a way here of assessing if high or low ATE is correct - - fitness_functions.append(fitness_function) - - return fitness_functions From e94dc6ecf9c5ec74f7e495e10407177599b6859b Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Tue, 28 Nov 2023 14:22:30 +0000 Subject: [PATCH 09/60] metadata pulled from DAG for inclusion in testing and their expected causal relation --- .../testing/causal_surrogate_assisted.py | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 954a7da2..d201cf59 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -8,24 +8,33 @@ class SearchAlgorithm: - def generate_fitness_functions(self, specification: CausalSpecification, surrogate_models: list[Estimator]): + def generate_fitness_functions(self, surrogate_models: list[Estimator]): pass - def search(self, fitness_functions) -> list: + def search(self, fitness_functions, specification: CausalSpecification) -> list: pass + class GeneticSearchAlgorithm(SearchAlgorithm): - def __init__(self, delta = 0.05) -> None: + def __init__(self, delta=0.05) -> None: super().__init__() self.delta = delta + self.contradiction_functions = { + "positive": lambda x: -1 * x, + "negative": lambda x: x, + "no effect": lambda x: abs(x), + "some effect": lambda x: abs(1/x) + } def generate_fitness_functions( self, surrogate_models: list[PolynomialRegressionEstimator] ) -> Callable[[list[float], int], float]: fitness_functions = [] - for surrogate in surrogate_models: + for surrogate, expected in surrogate_models: + + contradiction_function = self.contradiction_functions[expected] def fitness_function(solution, idx): surrogate.control_value = solution[0] - self.delta @@ -37,14 +46,14 @@ def fitness_function(solution, idx): ate = surrogate.estimate_ate_calculated(adjustment_dict)[0] - return ate # TODO Need a way here of assessing if high or low ATE is correct + return contradiction_function(ate) fitness_functions.append(fitness_function) return fitness_functions - - def search(self, fitness_functions) -> list: - pass # TODO Implement GA search + + def search(self, fitness_functions, specification: CausalSpecification) -> list: + pass # TODO Implement GA search @dataclass @@ -107,11 +116,12 @@ def generate_surrogates( ) -> list[PolynomialRegressionEstimator]: surrogate_models = [] - for u, v in specification.causal_dag.edges: - base_test_case = BaseTestCase(u, v) - minimal_adjustment_set = specification.causal_dag.identification(base_test_case) + for u, v, d in specification.causal_dag.edges.data(): + if "included" in d: + base_test_case = BaseTestCase(u, v) + minimal_adjustment_set = specification.causal_dag.identification(base_test_case) - surrogate = PolynomialRegressionEstimator(u, 0, 0, minimal_adjustment_set, v, 4, df=data_collector.data) - surrogate_models.append(surrogate) + surrogate = PolynomialRegressionEstimator(u, 0, 0, minimal_adjustment_set, v, 4, df=data_collector.data) + surrogate_models.append((surrogate, d["expected"])) return surrogate_models From b5a0f8c6901f526ebce27bcd84b8318dc9ce5b71 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 29 Nov 2023 11:38:07 +0000 Subject: [PATCH 10/60] Minimal working version for CauSAT + minor changes to identification for hidden variables --- causal_testing/specification/causal_dag.py | 12 ++- .../testing/causal_surrogate_assisted.py | 83 +++++++++++++----- causal_testing/testing/estimators.py | 44 +++++++--- examples/apsdigitaltwin/.env | 1 + examples/apsdigitaltwin/.gitignore | 3 + examples/apsdigitaltwin/apsdigitaltwin.py | 62 ++++++++++++-- examples/apsdigitaltwin/dag.dot | 17 ++++ examples/apsdigitaltwin/util/profile.json | 85 +++++++++++++++++++ 8 files changed, 266 insertions(+), 41 deletions(-) create mode 100644 examples/apsdigitaltwin/.env create mode 100644 examples/apsdigitaltwin/.gitignore create mode 100644 examples/apsdigitaltwin/dag.dot create mode 100644 examples/apsdigitaltwin/util/profile.json diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index 849f8e39..aa6e41e7 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -66,7 +66,7 @@ def list_all_min_sep( # 7. Check that there exists at least one neighbour of the treatment nodes that is not in the outcome node set if treatment_node_set_neighbours.difference(outcome_node_set): # 7.1. If so, sample a random node from the set of treatment nodes' neighbours not in the outcome node set - node = set(sample(treatment_node_set_neighbours.difference(outcome_node_set), 1)) + node = set(sample(sorted(treatment_node_set_neighbours.difference(outcome_node_set)), 1)) # 7.2. Add this node to the treatment node set and recurse (left branch) yield from list_all_min_sep( @@ -499,8 +499,13 @@ def depends_on_outputs(self, node: Node, scenario: Scenario) -> bool: if isinstance(scenario.variables[node], Output): return True return any((self.depends_on_outputs(n, scenario) for n in self.graph.predecessors(node))) + + def remove_hidden_adjustment_sets(self, minimal_adjustment_sets: list[str], scenario: Scenario): + return [ + adj for adj in minimal_adjustment_sets if all([not scenario.variables.get(x).hidden for x in adj]) + ] - def identification(self, base_test_case: BaseTestCase): + def identification(self, base_test_case: BaseTestCase, scenario: Scenario = None): """Identify and return the minimum adjustment set :param base_test_case: A base test case instance containing the outcome_variable and the @@ -519,6 +524,9 @@ def identification(self, base_test_case: BaseTestCase): ) else: raise ValueError("Causal effect should be 'total' or 'direct'") + + if scenario is not None: + minimal_adjustment_sets = self.remove_hidden_adjustment_sets(minimal_adjustment_sets, scenario) minimal_adjustment_set = min(minimal_adjustment_sets, key=len) return minimal_adjustment_set diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index d201cf59..38c5bb56 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -3,40 +3,55 @@ from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator +from pygad import GA + from dataclasses import dataclass from typing import Callable +from operator import itemgetter + + +@dataclass +class SimulationResult: + data: dict + fault: bool + + +@dataclass +class SearchFitnessFunction: + fitness_function: Callable + surrogate_model: PolynomialRegressionEstimator class SearchAlgorithm: - def generate_fitness_functions(self, surrogate_models: list[Estimator]): + def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: pass - def search(self, fitness_functions, specification: CausalSpecification) -> list: + def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: pass class GeneticSearchAlgorithm(SearchAlgorithm): - def __init__(self, delta=0.05) -> None: + def __init__(self, delta=0.05, config: dict = None) -> None: super().__init__() self.delta = delta + self.config = config self.contradiction_functions = { "positive": lambda x: -1 * x, "negative": lambda x: x, "no effect": lambda x: abs(x), - "some effect": lambda x: abs(1/x) + "some effect": lambda x: abs(1 / x), } def generate_fitness_functions( self, surrogate_models: list[PolynomialRegressionEstimator] - ) -> Callable[[list[float], int], float]: + ) -> list[SearchFitnessFunction]: fitness_functions = [] for surrogate, expected in surrogate_models: - contradiction_function = self.contradiction_functions[expected] - def fitness_function(solution, idx): + def fitness_function(_ga, solution, idx): surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta @@ -44,22 +59,42 @@ def fitness_function(solution, idx): for i, adjustment in enumerate(surrogate.adjustment_set): adjustment_dict[adjustment] = solution[i + 1] - ate = surrogate.estimate_ate_calculated(adjustment_dict)[0] + ate = surrogate.estimate_ate_calculated(adjustment_dict) return contradiction_function(ate) - fitness_functions.append(fitness_function) + search_fitness_function = SearchFitnessFunction(fitness_function, surrogate) + + fitness_functions.append(search_fitness_function) return fitness_functions - def search(self, fitness_functions, specification: CausalSpecification) -> list: - pass # TODO Implement GA search + def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: + solutions = [] + for fitness_function in fitness_functions: + ga = GA( + num_generations=200, + num_parents_mating=4, + fitness_func=fitness_function.fitness_function, + sol_per_pop=10, + num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), + ) -@dataclass -class SimulationResult: - data: dict - fault: bool + if self.config is not None: + for k, v in self.config.items(): + setattr(ga, k, v) + + ga.run() + solution, fitness, _idx = ga.best_solution() + + solution_dict = dict() + solution_dict[fitness_function.surrogate_model.treatment] = solution[0] + for idx, adj in enumerate(fitness_function.surrogate_model.adjustment_set): + solution_dict[adj] = solution[idx + 1] + solutions.append((solution_dict, fitness)) + + return max(solutions, key=itemgetter(1))[0] class Simulator: @@ -95,7 +130,7 @@ def execute( for _i in range(max_executions): surrogate_models = self.generate_surrogates(self.specification, data_collector) fitness_functions = self.search_algorithm.generate_fitness_functions(surrogate_models) - candidate_test_case = self.search_algorithm.search(fitness_functions) + candidate_test_case = self.search_algorithm.search(fitness_functions, self.specification) self.simulator.startup() test_result = self.simulator.run_with_config(candidate_test_case) @@ -113,15 +148,19 @@ def execute( def generate_surrogates( self, specification: CausalSpecification, data_collector: ObservationalDataCollector - ) -> list[PolynomialRegressionEstimator]: + ) -> list[SearchFitnessFunction]: surrogate_models = [] - for u, v, d in specification.causal_dag.edges.data(): - if "included" in d: - base_test_case = BaseTestCase(u, v) - minimal_adjustment_set = specification.causal_dag.identification(base_test_case) + for u, v in specification.causal_dag.graph.edges: + edge_metadata = specification.causal_dag.graph.adj[u][v] + if "included" in edge_metadata: + from_var = specification.scenario.variables.get(u) + to_var = specification.scenario.variables.get(v) + base_test_case = BaseTestCase(from_var, to_var) + + minimal_adjustment_set = specification.causal_dag.identification(base_test_case, specification.scenario) surrogate = PolynomialRegressionEstimator(u, 0, 0, minimal_adjustment_set, v, 4, df=data_collector.data) - surrogate_models.append((surrogate, d["expected"])) + surrogate_models.append((surrogate, edge_metadata["expected"])) return surrogate_models diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index 73e88876..98b18715 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -274,9 +274,9 @@ def estimate_unit_odds_ratio(self) -> float: return np.exp(model.params[self.treatment]) -class PolynomialRegressionEstimator(Estimator): - """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a - polynomial combination of parameters and functions of the variables (note these functions need not be polynomial). +class LinearRegressionEstimator(Estimator): + """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a + linear combination of parameters and functions of the variables (note these functions need not be linear). """ def __init__( @@ -287,7 +287,6 @@ def __init__( control_value: float, adjustment_set: set, outcome: str, - degree: int, df: pd.DataFrame = None, effect_modifiers: dict[Variable:Any] = None, formula: str = None, @@ -305,7 +304,7 @@ def __init__( self.formula = formula else: terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) - self.formula = f"{outcome} ~ cr({'+'.join(terms)}, df={degree})" + self.formula = f"{outcome} ~ {'+'.join(terms)}" for term in self.effect_modifiers: self.adjustment_set.add(term) @@ -440,9 +439,9 @@ def _get_confidence_intervals(self, model, treatment): return [ci_low, ci_high] -class LinearRegressionEstimator(PolynomialRegressionEstimator): - """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a linear - combination of parameters and functions of the variables (note these functions need not be linear). +class PolynomialRegressionEstimator(LinearRegressionEstimator): + """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a polynomial + combination of parameters and functions of the variables (note these functions need not be polynomial). """ def __init__( @@ -453,18 +452,43 @@ def __init__( control_value: float, adjustment_set: set, outcome: str, + degree: int, df: pd.DataFrame = None, effect_modifiers: dict[Variable:Any] = None, formula: str = None, alpha: float = 0.05, ): super().__init__( - treatment, treatment_value, control_value, adjustment_set, outcome, 1, df, effect_modifiers, formula, alpha + treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers, formula, alpha ) + if effect_modifiers is None: + effect_modifiers = [] + if formula is None: terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) - self.formula = f"{outcome} ~ {'+'.join(terms)}" + self.formula = f"{outcome} ~ cr({'+'.join(terms)}, df={degree})" + + def estimate_ate_calculated(self, adjustment_config: dict = None) -> tuple[float, list[float]]: + model = self._run_linear_regression() + + x = {"Intercept": 1, self.treatment: self.treatment_value} + for k, v in adjustment_config.items(): + x[k] = v + if self.effect_modifiers is not None: + for k, v in self.effect_modifiers.items(): + x[k] = v + + treatment = model.predict(x).iloc[0] + + x[self.treatment] = self.control_value + control = model.predict(x).iloc[0] + + return(treatment - control) + + + + class InstrumentalVariableEstimator(Estimator): diff --git a/examples/apsdigitaltwin/.env b/examples/apsdigitaltwin/.env new file mode 100644 index 00000000..4dd30c43 --- /dev/null +++ b/examples/apsdigitaltwin/.env @@ -0,0 +1 @@ +basal_profile_path = "./example_oref0_data/basal_profile.json" \ No newline at end of file diff --git a/examples/apsdigitaltwin/.gitignore b/examples/apsdigitaltwin/.gitignore new file mode 100644 index 00000000..df652f9c --- /dev/null +++ b/examples/apsdigitaltwin/.gitignore @@ -0,0 +1,3 @@ +data.csv +constants.txt +openaps_temp \ No newline at end of file diff --git a/examples/apsdigitaltwin/apsdigitaltwin.py b/examples/apsdigitaltwin/apsdigitaltwin.py index f75d6c2c..09d2d4d9 100644 --- a/examples/apsdigitaltwin/apsdigitaltwin.py +++ b/examples/apsdigitaltwin/apsdigitaltwin.py @@ -1,12 +1,23 @@ -from causal_testing.testing.causal_surrogate_assisted import SimulationResult, Simulator +from causal_testing.data_collection.data_collector import ObservationalDataCollector +from causal_testing.specification.causal_dag import CausalDAG +from causal_testing.specification.causal_specification import CausalSpecification +from causal_testing.specification.scenario import Scenario +from causal_testing.specification.variable import Input, Output +from causal_testing.testing.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, GeneticSearchAlgorithm, SimulationResult, Simulator from examples.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label +import pandas as pd +import numpy as np + +from dotenv import load_dotenv + class APSDigitalTwinSimulator(Simulator): - def __init__(self, constants) -> None: + def __init__(self, constants, profile_path) -> None: super().__init__() self.constants = constants + self.profile_path = profile_path def run_with_config(self, configuration) -> SimulationResult: min_bg = 200 @@ -17,8 +28,8 @@ def run_with_config(self, configuration) -> SimulationResult: open_aps_output = 0 violation = False - open_aps = OpenAPS() - model_openaps = Model([configuration[1], 0, 0, configuration[0], configuration[2]], self.constants) + open_aps = OpenAPS(profile_path=self.profile_path) + model_openaps = Model([configuration["start_cob"], 0, 0, configuration["start_bg"], configuration["start_iob"]], self.constants) for t in range(1, 121): if t % 5 == 1: rate = open_aps.run(model_openaps.history, output_file=f"./openaps_temp", faulty=True) @@ -37,9 +48,9 @@ def run_with_config(self, configuration) -> SimulationResult: end_iob = model_openaps.history[-1][i_label] data = { - "start_bg": configuration[0], - "start_cob": configuration[1], - "start_iob": configuration[2], + "start_bg": configuration["start_bg"], + "start_cob": configuration["start_cob"], + "start_iob": configuration["start_iob"], "end_bg": end_bg, "end_cob": end_cob, "end_iob": end_iob, @@ -50,3 +61,40 @@ def run_with_config(self, configuration) -> SimulationResult: return SimulationResult(data, violation) +if __name__ == "__main__": + load_dotenv() + + search_bias = Input("search_bias", float, hidden=True) + + start_bg = Input("start_bg", float) + start_cob = Input("start_cob", float) + start_iob = Input("start_iob", float) + open_aps_output = Output("open_aps_output", float) + hyper = Output("hyper", float) + + scenario = Scenario( + variables={ + search_bias, + start_bg, + start_cob, + start_iob, + open_aps_output, + hyper, + } + ) + + dag = CausalDAG("./dag.dot") + specification = CausalSpecification(scenario, dag) + ga_search = GeneticSearchAlgorithm() + + constants = [] + with open("constants.txt", "r") as const_file: + constants = const_file.read().replace("[", "").replace("]", "").split(",") + constants = [np.float64(const) for const in constants] + constants[7] = int(constants[7]) + + simulator = APSDigitalTwinSimulator(constants, "./util/profile.json") + data_collector = ObservationalDataCollector(scenario, pd.read_csv("./data.csv")) + test_case = CausalSurrogateAssistedTestCase(specification, ga_search, simulator) + + print(test_case.execute(data_collector)) \ No newline at end of file diff --git a/examples/apsdigitaltwin/dag.dot b/examples/apsdigitaltwin/dag.dot new file mode 100644 index 00000000..ff50ae86 --- /dev/null +++ b/examples/apsdigitaltwin/dag.dot @@ -0,0 +1,17 @@ +digraph APS_DAG { + rankdir=LR; + + "search_bias" -> "start_bg"; + "search_bias" -> "start_cob"; + "search_bias" -> "start_iob"; + + "start_bg" -> "hyper"; + "start_cob" -> "hyper"; + "start_iob" -> "hyper"; + + "start_bg" -> "open_aps_output" [included, expected=positive]; + "start_cob" -> "open_aps_output" [included, expected=positive]; + "start_iob" -> "open_aps_output" [included, expected=negative]; + + "open_aps_output" -> "hyper"; +} \ No newline at end of file diff --git a/examples/apsdigitaltwin/util/profile.json b/examples/apsdigitaltwin/util/profile.json new file mode 100644 index 00000000..f471a10b --- /dev/null +++ b/examples/apsdigitaltwin/util/profile.json @@ -0,0 +1,85 @@ +{ + "carb_ratios": { + "schedule": [ + { + "x": 0, + "i": 0, + "offset": 0, + "ratio": 10, + "r": 10, + "start": "00:00:00" + } + ], + "units": "grams" + }, + "carb_ratio": 10, + "isfProfile": { + "first": 1, + "sensitivities": [ + { + "endOffset": 1440, + "offset": 0, + "x": 0, + "sensitivity": 50, + "start": "00:00:00", + "i": 0 + } + ], + "user_preferred_units": "mg/dL", + "units": "mg/dL" + }, + "sens": 50, + "bg_targets": { + "first": 1, + "targets": [ + { + "max_bg": 100, + "min_bg": 100, + "x": 0, + "offset": 0, + "low": 100, + "start": "00:00:00", + "high": 100, + "i": 0 + } + ], + "user_preferred_units": "mg/dL", + "units": "mg/dL" + }, + "max_bg": 100, + "min_bg": 100, + "out_units": "mg/dL", + "max_basal": 4, + "min_5m_carbimpact": 8, + "maxCOB": 120, + "max_iob": 6, + "max_daily_safety_multiplier": 4, + "current_basal_safety_multiplier": 5, + "autosens_max": 2, + "autosens_min": 0.5, + "remainingCarbsCap": 90, + "enableUAM": true, + "enableSMB_with_bolus": true, + "enableSMB_with_COB": true, + "enableSMB_with_temptarget": false, + "enableSMB_after_carbs": true, + "prime_indicates_pump_site_change": false, + "rewind_indicates_cartridge_change": false, + "battery_indicates_battery_change": false, + "maxSMBBasalMinutes": 75, + "curve": "rapid-acting", + "useCustomPeakTime": false, + "insulinPeakTime": 75, + "dia": 6, + "current_basal": 1.0, + "basalprofile": [ + { + "minutes": 0, + "rate": 1.0, + "start": "00:00:00", + "i": 0 + } + ], + "max_daily_basal": 1.0 + } + \ No newline at end of file From 4a3c80783443e16ecfe767c985c3a1f8bba898b7 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 29 Nov 2023 13:57:40 +0000 Subject: [PATCH 11/60] Working version, runtime GA constraints + GA configuration --- .../testing/causal_surrogate_assisted.py | 23 ++++++++++++++++++- examples/apsdigitaltwin/.env | 2 +- examples/apsdigitaltwin/apsdigitaltwin.py | 22 ++++++++++++++++-- .../apsdigitaltwin/util/basal_profile.json | 8 +++++++ examples/apsdigitaltwin/util/model.py | 14 ----------- 5 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 examples/apsdigitaltwin/util/basal_profile.json diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 38c5bb56..614bb08d 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -72,17 +72,38 @@ def fitness_function(_ga, solution, idx): def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: solutions = [] - for fitness_function in fitness_functions: + for fitness_function in fitness_functions: + var_space = dict() + var_space[fitness_function.surrogate_model.treatment] = dict() + for adj in fitness_function.surrogate_model.adjustment_set: + var_space[adj] = dict() + + for relationship in list(specification.scenario.constraints): + rel_split = str(relationship).split(" ") + + if rel_split[1] == ">=": + var_space[rel_split[0]]["low"] = int(rel_split[2]) + elif rel_split[1] == "<=": + var_space[rel_split[0]]["high"] = int(rel_split[2]) + + gene_space = [] + gene_space.append(var_space[fitness_function.surrogate_model.treatment]) + for adj in fitness_function.surrogate_model.adjustment_set: + gene_space.append(var_space[adj]) + ga = GA( num_generations=200, num_parents_mating=4, fitness_func=fitness_function.fitness_function, sol_per_pop=10, num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), + gene_space=gene_space ) if self.config is not None: for k, v in self.config.items(): + if k == "gene_space": + raise Exception("Gene space should not be set through config. This is generated from the causal specification") setattr(ga, k, v) ga.run() diff --git a/examples/apsdigitaltwin/.env b/examples/apsdigitaltwin/.env index 4dd30c43..329072bf 100644 --- a/examples/apsdigitaltwin/.env +++ b/examples/apsdigitaltwin/.env @@ -1 +1 @@ -basal_profile_path = "./example_oref0_data/basal_profile.json" \ No newline at end of file +basal_profile_path = "./util/basal_profile.json" \ No newline at end of file diff --git a/examples/apsdigitaltwin/apsdigitaltwin.py b/examples/apsdigitaltwin/apsdigitaltwin.py index 09d2d4d9..4977f87b 100644 --- a/examples/apsdigitaltwin/apsdigitaltwin.py +++ b/examples/apsdigitaltwin/apsdigitaltwin.py @@ -72,6 +72,12 @@ def run_with_config(self, configuration) -> SimulationResult: open_aps_output = Output("open_aps_output", float) hyper = Output("hyper", float) + constraints = { + start_bg >= 70, start_bg <= 180, + start_cob >= 100, start_cob <= 300, + start_iob >= 0, start_iob <= 150 + } + scenario = Scenario( variables={ search_bias, @@ -80,12 +86,24 @@ def run_with_config(self, configuration) -> SimulationResult: start_iob, open_aps_output, hyper, - } + }, + constraints = constraints ) dag = CausalDAG("./dag.dot") specification = CausalSpecification(scenario, dag) - ga_search = GeneticSearchAlgorithm() + + ga_config = { + "parent_selection_type": "tournament", + "K_tournament": 4, + "mutation_type": "random", + "mutation_percent_genes": 50, + "mutation_by_replacement": True, + + "num_generations": 1, + } + + ga_search = GeneticSearchAlgorithm(config=ga_config) constants = [] with open("constants.txt", "r") as const_file: diff --git a/examples/apsdigitaltwin/util/basal_profile.json b/examples/apsdigitaltwin/util/basal_profile.json new file mode 100644 index 00000000..b4333d8a --- /dev/null +++ b/examples/apsdigitaltwin/util/basal_profile.json @@ -0,0 +1,8 @@ +[ + { + "minutes": 0, + "rate": 1, + "start": "00:00:00", + "i": 0 + } +] diff --git a/examples/apsdigitaltwin/util/model.py b/examples/apsdigitaltwin/util/model.py index 621ed698..52b16dc6 100644 --- a/examples/apsdigitaltwin/util/model.py +++ b/examples/apsdigitaltwin/util/model.py @@ -154,20 +154,6 @@ def __make_file_and_write_to(self, file_path, contents): file = open(file_path, "w") file.write(contents) - def __update_suggested(self, suggested_json, file_path): - if not os.path.isfile(file_path): - new_file = open(file_path, "w") - new_file.write("[]") - new_file.close() - - list_json = None - with open(file_path) as output_file: - list_json = json.load(output_file) - list_json.append(json.loads(suggested_json)) - - with open(file_path, 'w') as writing_json: - json.dump(list_json, writing_json, indent=4, separators=(',',': ')) - class Model: def __init__(self, starting_vals, constants): self.interventions = dict() From e93d59c739f4ccbef704072b84873a9d03a8c349 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 29 Nov 2023 14:19:28 +0000 Subject: [PATCH 12/60] formatting --- causal_testing/testing/causal_surrogate_assisted.py | 8 +++++--- causal_testing/testing/estimators.py | 4 ++-- examples/apsdigitaltwin/apsdigitaltwin.py | 2 -- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 614bb08d..832076fd 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -72,7 +72,7 @@ def fitness_function(_ga, solution, idx): def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: solutions = [] - for fitness_function in fitness_functions: + for fitness_function in fitness_functions: var_space = dict() var_space[fitness_function.surrogate_model.treatment] = dict() for adj in fitness_function.surrogate_model.adjustment_set: @@ -97,13 +97,15 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: fitness_func=fitness_function.fitness_function, sol_per_pop=10, num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), - gene_space=gene_space + gene_space=gene_space, ) if self.config is not None: for k, v in self.config.items(): if k == "gene_space": - raise Exception("Gene space should not be set through config. This is generated from the causal specification") + raise Exception( + "Gene space should not be set through config. This is generated from the causal specification" + ) setattr(ga, k, v) ga.run() diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index 98b18715..22e92020 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -275,8 +275,8 @@ def estimate_unit_odds_ratio(self) -> float: class LinearRegressionEstimator(Estimator): - """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a - linear combination of parameters and functions of the variables (note these functions need not be linear). + """A Linear Regression Estimator is a parametric estimator which restricts the variables in the data to a linear + combination of parameters and functions of the variables (note these functions need not be linear). """ def __init__( diff --git a/examples/apsdigitaltwin/apsdigitaltwin.py b/examples/apsdigitaltwin/apsdigitaltwin.py index 4977f87b..6c7795d3 100644 --- a/examples/apsdigitaltwin/apsdigitaltwin.py +++ b/examples/apsdigitaltwin/apsdigitaltwin.py @@ -99,8 +99,6 @@ def run_with_config(self, configuration) -> SimulationResult: "mutation_type": "random", "mutation_percent_genes": 50, "mutation_by_replacement": True, - - "num_generations": 1, } ga_search = GeneticSearchAlgorithm(config=ga_config) From 0243472e26a6586091bad6fea5ec8dc8bfeff232 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Fri, 1 Dec 2023 14:08:05 +0000 Subject: [PATCH 13/60] moved examples and implemented multithreading --- .../testing/causal_surrogate_assisted.py | 97 +------- .../testing/surrogate_search_algorithms.py | 208 ++++++++++++++++++ .../apsdigitaltwin/.env | 0 .../apsdigitaltwin/.gitignore | 0 .../apsdigitaltwin/README.md | 0 .../apsdigitaltwin/apsdigitaltwin.py | 4 +- .../apsdigitaltwin/dag.dot | 6 +- .../surrogate_assisted/apsdigitaltwin/dag.png | Bin 0 -> 37359 bytes .../apsdigitaltwin/util/basal_profile.json | 0 .../apsdigitaltwin/util/model.py | 0 .../apsdigitaltwin/util/profile.json | 0 11 files changed, 216 insertions(+), 99 deletions(-) create mode 100644 causal_testing/testing/surrogate_search_algorithms.py rename examples/{ => surrogate_assisted}/apsdigitaltwin/.env (100%) rename examples/{ => surrogate_assisted}/apsdigitaltwin/.gitignore (100%) rename examples/{ => surrogate_assisted}/apsdigitaltwin/README.md (100%) rename examples/{ => surrogate_assisted}/apsdigitaltwin/apsdigitaltwin.py (95%) rename examples/{ => surrogate_assisted}/apsdigitaltwin/dag.dot (55%) create mode 100644 examples/surrogate_assisted/apsdigitaltwin/dag.png rename examples/{ => surrogate_assisted}/apsdigitaltwin/util/basal_profile.json (100%) rename examples/{ => surrogate_assisted}/apsdigitaltwin/util/model.py (100%) rename examples/{ => surrogate_assisted}/apsdigitaltwin/util/profile.json (100%) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 832076fd..89ab579d 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -3,11 +3,8 @@ from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator -from pygad import GA - from dataclasses import dataclass -from typing import Callable -from operator import itemgetter +from typing import Callable, Any @dataclass @@ -18,7 +15,7 @@ class SimulationResult: @dataclass class SearchFitnessFunction: - fitness_function: Callable + fitness_function: Any surrogate_model: PolynomialRegressionEstimator @@ -30,96 +27,6 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: pass -class GeneticSearchAlgorithm(SearchAlgorithm): - def __init__(self, delta=0.05, config: dict = None) -> None: - super().__init__() - - self.delta = delta - self.config = config - self.contradiction_functions = { - "positive": lambda x: -1 * x, - "negative": lambda x: x, - "no effect": lambda x: abs(x), - "some effect": lambda x: abs(1 / x), - } - - def generate_fitness_functions( - self, surrogate_models: list[PolynomialRegressionEstimator] - ) -> list[SearchFitnessFunction]: - fitness_functions = [] - - for surrogate, expected in surrogate_models: - contradiction_function = self.contradiction_functions[expected] - - def fitness_function(_ga, solution, idx): - surrogate.control_value = solution[0] - self.delta - surrogate.treatment_value = solution[0] + self.delta - - adjustment_dict = dict() - for i, adjustment in enumerate(surrogate.adjustment_set): - adjustment_dict[adjustment] = solution[i + 1] - - ate = surrogate.estimate_ate_calculated(adjustment_dict) - - return contradiction_function(ate) - - search_fitness_function = SearchFitnessFunction(fitness_function, surrogate) - - fitness_functions.append(search_fitness_function) - - return fitness_functions - - def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: - solutions = [] - - for fitness_function in fitness_functions: - var_space = dict() - var_space[fitness_function.surrogate_model.treatment] = dict() - for adj in fitness_function.surrogate_model.adjustment_set: - var_space[adj] = dict() - - for relationship in list(specification.scenario.constraints): - rel_split = str(relationship).split(" ") - - if rel_split[1] == ">=": - var_space[rel_split[0]]["low"] = int(rel_split[2]) - elif rel_split[1] == "<=": - var_space[rel_split[0]]["high"] = int(rel_split[2]) - - gene_space = [] - gene_space.append(var_space[fitness_function.surrogate_model.treatment]) - for adj in fitness_function.surrogate_model.adjustment_set: - gene_space.append(var_space[adj]) - - ga = GA( - num_generations=200, - num_parents_mating=4, - fitness_func=fitness_function.fitness_function, - sol_per_pop=10, - num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), - gene_space=gene_space, - ) - - if self.config is not None: - for k, v in self.config.items(): - if k == "gene_space": - raise Exception( - "Gene space should not be set through config. This is generated from the causal specification" - ) - setattr(ga, k, v) - - ga.run() - solution, fitness, _idx = ga.best_solution() - - solution_dict = dict() - solution_dict[fitness_function.surrogate_model.treatment] = solution[0] - for idx, adj in enumerate(fitness_function.surrogate_model.adjustment_set): - solution_dict[adj] = solution[idx + 1] - solutions.append((solution_dict, fitness)) - - return max(solutions, key=itemgetter(1))[0] - - class Simulator: def startup(self, **kwargs): pass diff --git a/causal_testing/testing/surrogate_search_algorithms.py b/causal_testing/testing/surrogate_search_algorithms.py new file mode 100644 index 00000000..038a089a --- /dev/null +++ b/causal_testing/testing/surrogate_search_algorithms.py @@ -0,0 +1,208 @@ +from causal_testing.specification.causal_specification import CausalSpecification +from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator +from causal_testing.testing.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction + +from pygad import GA +from operator import itemgetter +import multiprocessing as mp +import warnings + +contradiction_functions = { + "positive": lambda x: -1 * x, + "negative": lambda x: x, + "no_effect": lambda x: abs(x), + "some_effect": lambda x: abs(1 / x), +} + + +class GeneticSearchAlgorithm(SearchAlgorithm): + def __init__(self, delta=0.05, config: dict = None) -> None: + super().__init__() + + self.delta = delta + self.config = config + self.contradiction_functions = { + "positive": lambda x: -1 * x, + "negative": lambda x: x, + "no_effect": lambda x: abs(x), + "some_effect": lambda x: abs(1 / x), + } + + def generate_fitness_functions( + self, surrogate_models: list[PolynomialRegressionEstimator] + ) -> list[SearchFitnessFunction]: + fitness_functions = [] + + for surrogate, expected in surrogate_models: + contradiction_function = self.contradiction_functions[expected] + + def fitness_function(_ga, solution, idx): + surrogate.control_value = solution[0] - self.delta + surrogate.treatment_value = solution[0] + self.delta + + adjustment_dict = dict() + for i, adjustment in enumerate(surrogate.adjustment_set): + adjustment_dict[adjustment] = solution[i + 1] + + ate = surrogate.estimate_ate_calculated(adjustment_dict) + + return contradiction_function(ate) + + search_fitness_function = SearchFitnessFunction(fitness_function, surrogate) + + fitness_functions.append(search_fitness_function) + + return fitness_functions + + def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: + solutions = [] + + for fitness_function in fitness_functions: + var_space = dict() + var_space[fitness_function.surrogate_model.treatment] = dict() + for adj in fitness_function.surrogate_model.adjustment_set: + var_space[adj] = dict() + + for relationship in list(specification.scenario.constraints): + rel_split = str(relationship).split(" ") + + if rel_split[1] == ">=": + var_space[rel_split[0]]["low"] = int(rel_split[2]) + elif rel_split[1] == "<=": + var_space[rel_split[0]]["high"] = int(rel_split[2]) + + gene_space = [] + gene_space.append(var_space[fitness_function.surrogate_model.treatment]) + for adj in fitness_function.surrogate_model.adjustment_set: + gene_space.append(var_space[adj]) + + ga = GA( + num_generations=200, + num_parents_mating=4, + fitness_func=fitness_function.fitness_function, + sol_per_pop=10, + num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), + gene_space=gene_space, + ) + + if self.config is not None: + for k, v in self.config.items(): + if k == "gene_space": + raise Exception( + "Gene space should not be set through config. This is generated from the causal specification" + ) + setattr(ga, k, v) + + ga.run() + solution, fitness, _idx = ga.best_solution() + + solution_dict = dict() + solution_dict[fitness_function.surrogate_model.treatment] = solution[0] + for idx, adj in enumerate(fitness_function.surrogate_model.adjustment_set): + solution_dict[adj] = solution[idx + 1] + solutions.append((solution_dict, fitness)) + + return max(solutions, key=itemgetter(1))[0] # TODO This can be done better with fitness normalisation between edges + + +pool_vals = [] + +def build_fitness_func(surrogate, expected, delta): + + def diff_evo_fitness_function(_ga, solution, _idx): + surrogate.control_value = solution[0] - delta + surrogate.treatment_value = solution[0] + delta + + adjustment_dict = dict() + for i, adjustment in enumerate(surrogate.adjustment_set): + adjustment_dict[adjustment] = solution[i + 1] + + ate = surrogate.estimate_ate_calculated(adjustment_dict) + + return contradiction_functions[expected](ate) + + return diff_evo_fitness_function + + +def threaded_search_function(idx): + surrogate, expected, delta, constraints, config = pool_vals[idx] + + var_space = dict() + var_space[surrogate.treatment] = dict() + for adj in surrogate.adjustment_set: + var_space[adj] = dict() + + for relationship in list(constraints): + rel_split = str(relationship).split(" ") + + if rel_split[1] == ">=": + var_space[rel_split[0]]["low"] = int(rel_split[2]) + elif rel_split[1] == "<=": + var_space[rel_split[0]]["high"] = int(rel_split[2]) + + gene_space = [] + gene_space.append(var_space[surrogate.treatment]) + for adj in surrogate.adjustment_set: + gene_space.append(var_space[adj]) + + ga = GA( + num_generations=200, + num_parents_mating=4, + fitness_func=build_fitness_func(surrogate, expected, delta), + sol_per_pop=10, + num_genes=1 + len(surrogate.adjustment_set), + gene_space=gene_space, + ) + + if config is not None: + for k, v in config.items(): + if k == "gene_space": + raise Exception( + "Gene space should not be set through config. This is generated from the causal specification" + ) + setattr(ga, k, v) + + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ga.run() + solution, fitness, _idx = ga.best_solution() + + solution_dict = dict() + solution_dict[surrogate.treatment] = solution[0] + for idx, adj in enumerate(surrogate.adjustment_set): + solution_dict[adj] = solution[idx + 1] + + return (solution_dict, fitness) + + +class MultiProcessGeneticSearchAlgorithm(SearchAlgorithm): + + def __init__(self, delta=0.05, config: dict = None, processes: int = 1) -> None: + self.delta = delta + self.config = config + self.processes = processes + + def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: + return [SearchFitnessFunction(expected, model) for model, expected in surrogate_models] + + def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: + global pool_vals + solutions = [] + all_fitness_function = fitness_functions.copy() + + while len(all_fitness_function) > 0: + num = self.processes + if num > len(all_fitness_function): + num = len(all_fitness_function) + + pool_vals.clear() + + while len(pool_vals) < num and len(all_fitness_function) > 0: + fitness_function = all_fitness_function.pop() + pool_vals.append((fitness_function.surrogate_model, fitness_function.fitness_function, self.delta, specification.scenario.constraints, self.config)) + + with mp.Pool(processes=num) as pool: + solutions.extend(pool.map(threaded_search_function, range(len(pool_vals)))) + + return min(solutions, key=itemgetter(1))[0] # TODO This can be done better with fitness normalisation between edges \ No newline at end of file diff --git a/examples/apsdigitaltwin/.env b/examples/surrogate_assisted/apsdigitaltwin/.env similarity index 100% rename from examples/apsdigitaltwin/.env rename to examples/surrogate_assisted/apsdigitaltwin/.env diff --git a/examples/apsdigitaltwin/.gitignore b/examples/surrogate_assisted/apsdigitaltwin/.gitignore similarity index 100% rename from examples/apsdigitaltwin/.gitignore rename to examples/surrogate_assisted/apsdigitaltwin/.gitignore diff --git a/examples/apsdigitaltwin/README.md b/examples/surrogate_assisted/apsdigitaltwin/README.md similarity index 100% rename from examples/apsdigitaltwin/README.md rename to examples/surrogate_assisted/apsdigitaltwin/README.md diff --git a/examples/apsdigitaltwin/apsdigitaltwin.py b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py similarity index 95% rename from examples/apsdigitaltwin/apsdigitaltwin.py rename to examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py index 6c7795d3..1877223c 100644 --- a/examples/apsdigitaltwin/apsdigitaltwin.py +++ b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py @@ -3,7 +3,8 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input, Output -from causal_testing.testing.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, GeneticSearchAlgorithm, SimulationResult, Simulator +from causal_testing.testing.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, SimulationResult, Simulator +from causal_testing.testing.surrogate_search_algorithms import GeneticSearchAlgorithm from examples.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label import pandas as pd @@ -99,6 +100,7 @@ def run_with_config(self, configuration) -> SimulationResult: "mutation_type": "random", "mutation_percent_genes": 50, "mutation_by_replacement": True, + "suppress_warnings": True, } ga_search = GeneticSearchAlgorithm(config=ga_config) diff --git a/examples/apsdigitaltwin/dag.dot b/examples/surrogate_assisted/apsdigitaltwin/dag.dot similarity index 55% rename from examples/apsdigitaltwin/dag.dot rename to examples/surrogate_assisted/apsdigitaltwin/dag.dot index ff50ae86..ca54c473 100644 --- a/examples/apsdigitaltwin/dag.dot +++ b/examples/surrogate_assisted/apsdigitaltwin/dag.dot @@ -9,9 +9,9 @@ digraph APS_DAG { "start_cob" -> "hyper"; "start_iob" -> "hyper"; - "start_bg" -> "open_aps_output" [included, expected=positive]; - "start_cob" -> "open_aps_output" [included, expected=positive]; - "start_iob" -> "open_aps_output" [included, expected=negative]; + "start_bg" -> "open_aps_output" [included=1, expected=positive]; + "start_cob" -> "open_aps_output" [included=1, expected=positive]; + "start_iob" -> "open_aps_output" [included=1, expected=negative]; "open_aps_output" -> "hyper"; } \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/dag.png b/examples/surrogate_assisted/apsdigitaltwin/dag.png new file mode 100644 index 0000000000000000000000000000000000000000..af043f9f64ec314624a09778bcf5576ad78d9359 GIT binary patch literal 37359 zcmYIw2RxQ-`~TgN3W<`eGD4DwC>e#Mj7p+mOR_UFvQi|oBr8fGl2wT8og%VBW|2)O z5&qw^=ly;D&)fUHJ&pUmuJbz2{LC?Vg}737uT&gPeGQe z8-*esL`5l^4Hl_KC7VjmN%dsK^6&ikAwlr1V3>B9EoYsE_Oa`FM>SX7w``AGch%Ei zqG#vQU`57zE@98M#kr{MwULXD?P$xr>|z#+(#wqkMCj$HId9X;DOKC}rjJwq`#Cjd zy!a`N|NWGmZnpq_AO99YcS5Uk6}?=n&c-{d8T{7bhhZjjSXdbU-n~suIzr)g^>cG` z1{M~z(h>&_+=`3iR8dtmxO$c9+_`gsZ5!|{jm~GE$x)l0-@SXclCrYArY4iEo!$NL zaNmLgsc+xD?UI$<`RC6c{(bxIXb2oPHa1R~)H`uv{ne{iGcqzfv$GG5jg9G>n5@S? z?HwJ@F2A+<(VEJtt)pXLZr(Yv$VYBtwD)YwemNz(n(ww1)z#eM;!HI)HA2q{)ouP?CI0@$;zF+?#r-E?mgo+CMGUkUMdOeZUt>^wuzqN;H)e$4K1zm zh6egV*WW%VDSKV!MiOiqd!C=z%*?`KcI2<*%a<=h!ou$04+`a}dpvpkm?l|cBjv}J zH+%&J1>~z_Wo2nt#HfRUf-VKc;@h0Bp1~UbS#LzMnM0~1Y_AqQBcpdj1TzmGpLg{@ zU*1rxcX04#@|VAVU$=AT&e@TcO_Gw59d!9E!c|j0K66)8R&ojn(Pd_4-VF>qE+@D8 zWlBolw{L4bJUqx3EiW%m&CHCdPd>qy4Ikw{%lYl1)|oS&{@eHWcjej5&(zD_kBr>F z$jCUquuxWCzggagru6O>ZhcG3)bTUfiLYMK+?ePpO}+e7OKBe&T@>0{# zRNUUcIQ8@6{^ut3u^n__m@PF=FZ3~ON{rJhcJH7gPh+_D-$zd4 zPT8cK=nV`E$VXaOT8bEd*cz}?mcz@-OQZGo9t{nR-jSA+L(V^Wg4hn<3J9Rd&C4?w zX-;HY9mA%nwpv% z@9H#quis41#N@MPJ>9O|yVtq9yHnHARlU0ShM$uMJ2%P)d*t1-{(eJ?ukW@UxiO)D zy`}5^$9V1f^+sj4H#D>)bn<0yuVNO`&Mnz}Z?zX?X0Wb2>Y#b=w{P*|Q{;RG1(1(; zaO>9Un=^G1?KzgLvvYIPt(UD0ru!?!YVy~-udlDJmt6_4?94gF!(g!d_s{%P`R2a< z{%7M2|32U@`G8wLf7-s%E9Qunu=dH!6lr~R(cAdrqtc2Bh0eJ9mjw3i&9v^3`8(;c zB7gNNFO%Q}PkiLx$h4P=_=sk2e1c?tkH-r8WYMBfg`r4mnvQ>ha=6?s`fEo78>ti2 z{Tdq1+?@S@_2n~oegV(EMQQb}|3R#pkCikKreo~kGJ4h{MCxG$=98f?Kay>!BFBWFx) zj%BB?VFmU4pFe%Ash620?UQokxl*56_Y~c|cdu;h%Nr5P4zYrj<;AD>^YGkajUOh_ zH~Lw0WNnx%n4y#T^No@1h$9UX6H^CW^11nul=PB;Y5Zs?biHBe+q7%Vnl%=kIka(c zaqaIM_CDILr&hBO&(4HNHIXkqw7tMtR5M*SFui!01q+y#Q$GA~VQ$p7tW55Tv9alq zhl=d@aQv<$(|%p@98Hg~;nL#^KVNZ?ll0!JmDXGMySqGG&;5(RgRMA(G&>H&f1Eho z$*UT5Ti%z>VDZ;@d;X2|oN`k1JJ?a%?XMq*q}jAxsA7J4V7l9N5}()@#Ctk6;6{(z zcATUaX8Pnd50l#@$Cge#d27Doc%vh~!X}xOzjnFr-Z@6Ziv7C^av0WYs;l3^RrboR z{Ml?_VbO8ZW81cE7v5U2cYXcJNl&eBYU;Ovk+-+Pk1;7NjcTgCV)pgx0}98F+k`ed zI5^~u^M{>clex)!?AWo4w{Ly#-``SDSeR-2kts<-Anj|WQFZQ|vlPdJ2M>hZ7VIxv zylC@L=I>Ow;_o(XKR-Wa2`e@f@U^Q~DF!fmB_$=fbgD*)`I$>x?>+i=x_T$`L9?yf ztNSbcFSDIE$IQ&EU+zOoj(C56e}}yu?o-LPjGB|ZHCEa+0u7AOzjFL_ai#xu(YQ13 zQCZ4AeEb-x#A7&F_?u%*2~OhE&#mq45$o2jZO)IGnz~NqnW|SDyKVdSH#mej4x^e? zLo;7oCkwS+Ea7%{P{eP}Uw7;)Tc@e1S!HNw7!edix0j#)n!ShW_tDXhxNuex!?pea z0aw(MAAZfX@%}SEFYM;#<~ZGdJk9ddsg3AdCaCmdwUMGK0%5Ona}SA|HS=FJGAgH` zp@|iEGC|A9?LU3 zA3l0y`0>#`>Wv$3`T1>F_%-g2Quy@Mt5@fZ%xHqE%YS@+Zu95ocgL^$u{WQolKtkD zkeFCjUjF>+FNv$Q%-V%6+e%AI@A~=m`M3f@a0&}&)O7BUb(g}EIAfBRp1vRLa;mq~ z+og^5&~<(Pek(e8dHI6*et(va*hHUGFH3ag-`Jd&mq+nzdU1LUMKPEo^>b@g71hMV zge8ZGWo&FLu!809C9-lIa;OLF5!yn&M|WL&yM)6=i_;?yAfXtG+>7)70SJ3 zt0Sp%sh8JVTU%eaaN+vQ;ckx=H_B0GXXnDv9jvUfbFS9{T$YtBb8>Pho`3(|L~l7N z5LSk^kH+WFVK$%`z1r*PS65q9P!#(Gw>LhxaO*=zyly>p(tW|y_c+A%JfQG;}W~25jQ$|YG53AAB9eQ?mEBjr- zhlYmM%pWlj5f=7LPTsS1Mxj&1^Yqm;B~8m;;(x>iM%hk^6(;k+-4ZkK?H zZU+aS&YuiNlm5&E{5f1w6|lp*y}jL0TEu-c_3ibQWjCxGL5X_qe|9q)ISnU-i*-n^ z{GCx!Q&U7UdiMPJ{YQ^{;^ckhfE&@s>HQSQ&fd6b6KD8G8yq}~o}$7^n}b=!g9pUL ztA2j}Xfxh%@L^b3-(X#IgtV(T8YU?AWX3nJyqQP$C<0Z%#R;GM#t{VZ3CWckG3wV!7oI_#&W!Q z^TudqsNQ6!`#k5G5{fYJ9uqrzopk!2r25cbZ4rU261JU=_V!oW()CQZ6$A9KrE1^5 zze~GQ=J9bw#ny}a_w4Z_H83(VQpD?M$={`=FX+b?aE2MF*Q$1}G%WyfMWWbkqNBUY z$;oLk^eJWx4@giz;NEUNK2bk~MypIb6VCB(;=!3rG)#QAv!c<%YojGLZx_-o$uKC- zVy9cxJ263*ryUd!5Fp;8Wo9O@@Ov_NyROqJ)P8TQlCg@s5dM zf0deQU}Ut0p!xRpX6L$whKxER0^(!K*vaExn|S(3(2pP2x{5vY_HKI-dz|Y&H4Z2N z-&mKoe?LDzaox_Y9FJ$Z>9lONDb1+_29$GexwxY$Dl~YKd2P)nmEGNC6u7n!kj}#^ zE2}qfaBeC;iVzcP>&>jJtoZ5@Pym#`F1tB%98m4!|5_P4n2M6gnF|-Ffsu3b^9O=u zm#?)wsH_*buLMN4UhqN&Wo2p7!@$`14$5%B%%>wx^HZEwUAe2YwA#cwi%%t#mEGC2 z1%MByNoH=z{+sRLQ>RXyL38Zt(gxdU{-e7jB`V5z{``5C(aRSul$4gP`ufhn+uM5$ zYH#kO`F?J!#idoGMKb}(klo%pIOuckg<6ZNN3hA+b03&I(9`C>g`SQo9KU;)n&1jx z>tkwaboc~ebcVmbi&u`P>lR+3w`@uZeFy;1_qNBq9MG!6LT}C>z()E2iq!kMy5m?T za4}8>Ke0#Ohlb1_Ml(^bCG8hi18htxC&*H3er-v~Q=Oje04!>E2+|5|Z@&=0c9@~i zWv&dUWFsFRpQW$DrHiJfdk8W_ksSc6aQgE-Je$3daZlI>$qlNwyq`aRjtGV-@vw|u zx^$_myIYsr+2;E7h6`80?k->M)XWd2BVX|kOh)67sKiKa!_!CkE#I47@N7IpX>Xpx zC=)gaIGR_v?`EU-A)LTl=g(_sJVn>-ci7C9b`kG4?jE$ZBV3FfwcZ`oJgPhXP zQ35M3q8kE9uc4qB?H9K0mSW**s(CEwuyL>EtD`{r1UD8ePUJW5ZJ@EuD^N>U8BgD_ zV+W}9K8@%nXRe?Au;zCZTYyD020i>}-)>*?4${rxu)(7SFElhlkI zN7hM!uzs|9RI2q>Jw<_@o|KzYrRyVv4Q}PgbMHHI_N*7a(#q)~c38~hajq>|IbTXj zJfpdru$Kd$QAQ@&+1Qe{Kctrv`*+mIBJL6tq!9>vSt_28kkHi-cJJQCt=qRh&&t{= z5VkQbEsgZmsv!1FIyzr}ZCs43WG>G(Z=1qB_u)Fu2!ut*xJf<|(z&-?@8;RgzWd^d zXZS-xLuV?k@JdL8o9nrDtGLzBd{qgh{}~X69{;HyOosWe?G^u4b}1)eW+MO_FQ^;@ zdlTwdad*qZtSqj%`a`&^^_w|RL*;sk+(4KD00>Gz2|76+&?#`;--RlIo$>zDC(;Bt zWZeUB=b#HMdsge4nY|Q=SCMXh<@$T#x}&4dn>V5~w6xjtK_Ct1|J`^Tcu;RIV98(J zY^{3Li^cBJGV{G_C`SQw%F4>Fe|x{BtH7BNkd^s}qu{Mux9GQSMFsQ~_b*Nn*t6%D zg2GzDxxoP$FJ8QO{l*PnT<7`o+t5*c(P+v(epK#^d!VMI)AA~gBhq*CPB}p)yYZIa z(aB%A2S!FVN=Qh&yrM`wtU|W5=*2fzX^R#oj%a9V%4Ei%&I4!?{(bhjVoN8lv^0D1 z(l1uECo+MccmM+>ot>=vi~k<**}OC8m^angUX_tO^V`4Kis7fg* z@meDS0I3oywWGVco8G_j(9L<_9ILPQu&g*a!J`*N4jgFwnZHZt5#Uz zc2w3^@M!IK_8Uw-JtkMOW#`V#Rm`!j6NhOyq?q>?&2td|Jv-j{J5sVH?d3~3LqjgK z4=;2)g%c;ZIXS(eXGBrMiN=9l@{Q3t7X`r&46Gk}9*S5A;KY(FVL{FvlDelMn-1p*Ds&&@aqN8 z&w|-TWukuo$v%DlyoqWuAfzgpZ@DeJB0eFZ53OB$%jUWGm4ye)FLf2EP5+VXvz67= z1Nh8C*Lx|Ajg7s1eO{5GCeJc5=%F`A1N!IYK*`b%Fs(gds){4Kd8bUp(&F#D(0Ocr zPV620niNO+ry75p0(7-LRG!P1FQc)UjmHZ8r|ZdESqa^c4T$11#);C-&f<`EDf#&E z-x^F$_dUpL6_g~E4Ncf!v^Dka-MgeL-SYO%yxu3rVs%22El*q5Rm)BrFsM+zmlx`|nRci5>o4S`h@pd^6Tk{;=!5OBdW7f~SZfs}kh z+vjmuSXiLd-Qs9x=h{!7mJ{;$ae%bzFa5SO9dx4dl=Px|s1;O0Oj{a1Z*p1Ch4=5@3|%+`cwsZPC4PjRTIZ)w zHfPnWqDT>os7PXDcjz~Q(hQcqTak{Ajv&m!LO0(++(DRU>Auz)BJUc!) zdHJ}M6#pHfSM|Yeagtgcuieg)ysxBT&G0;S?{gW|%}&s*-xSn&xAoJf zPaSl@^r;ZskKMd^GbT0y$PL_T)qK8sB(!ZX{>vsxkvK0pW1R5=q$uT(ngRGWfC8;_ z@}yUO{t;b|C5PI_hZ#{xTOFO^89&z4lrAqVPJg}8!^y|zdy0)weKRhhI(VmD;r@d= zlarIr@NcD1K*yoa#@(3g{?YVeGo*|VpJjWMqHZ0HhwD5zzG8tdw+?GD;t$_devT(A6l z4k@P++I`BWPeY!6_wF4OkM$BaYnz%vSv6bHUIa2#ltu|O7*y9918FU{E?%MvJJs^) zp3d#8ZILQOJMQ(}>0Sc#G!5-^02)9A9zG8GD@ee@&G0B5goft%zT2L7)z28d3z$p! z*RSjx92}%Ezj(n1gW>r0G-^{B;9t{wt*_dTE4iKtKh%xJ09jqwsEUSAnW>*Y8?j66 z52I*n_dke;xH~`Z!~?|zH9tV)$MA4zT^%hxl&rOsv~=EWA0aOK#$EVK=LbFlj&<0e zIEB~^A&Oy8eI~mL+g}@RYHVuSLQn4rPr|S<{-oHa*bs5eix>SKKHO?GAK}7(@#ak_ zkR;Dy<-IAs!7XfxgTU?Ie{A?HiO)?QM^7MK``6eP?{D9(%#M9w{3nA7X)A5ZS9)`>cvrjg2dJ3+d|WVuRm) z`&MGx$6fUd_@Y(ILVsA8m^7^`(*P-7zB&EwA==egNA@P!zf)`QP^KG|_Ztx58M@O_ z8sjr#5CQGRXA@q2{{HedHX;%+8oVh_A`Q*vYn{IBy z?!xh+&lT=ST{hDidzqG|2yy287*~A(ZtpL41wq7+cz9JJ$Wn9d+I6%uZi`&(E^cn` zQ3vO#-c{ih55vP@29 zQwfKhrgo9mjY?DwssR~}$R)df-nM5zR~;T6*6x3%X2`(6aOUF04QM(}E-toy{`g#6 zFzmUM@rDhg88kX1?{ysSd`mPqjZFWLtLoT$Kn0L5rxr%jjlO>f!}Dmy-V0#@qEMMWxT zbbh_P`ha|qgSJ~5*Ch=VCozTt@Adu~?}YaNC*e5u<-x(CBxQca#wy%*j9%S69j$cT zJK%6w(0JNrL7Tself~tDpmsx_Dm$%Jk212dZsX** z%>Gz~BAQzGmgyBA%D;n-nQQb6{Wq%?FdT98kA&k ztUh#=<15Qc+3YN6^Cx*2(CO9_f7ae!4ElUXObjh9!O7X#0IIU~>ZX>V5J(i<0s*m-g`BlX_qDMk*R+PvG*n(DaLX+bIpDYK3k2)WZcl3>)a! zC2zwTA_lESe+2z%%KO?{1qfacKZ%%0b_)D?Lf7DoLG2TPz*sOd7GhoH+R0u!W&A zp1|T@kHP2<0OcYMr_=W^t*o=t_i?^$n$z09kBvbZzp7t)YaJRya85b*?W_{;mJBGX zY?AiW*b5OyZU||nX`kHYP0g9~>J{F4?rdz!R#1KC~TbYHH8C zJZJx3z#&q^p%xR{ivE-Omu~q!a3T@5V6ecTTTPi?T;%5Gr(U&c6*D}56Kt2x@2_G; zVWMYe4}{Iti>@JOV896qCt}rgs9r=<@`=5?+H7ll%W3p$^qcp>qe9W z^*s+yB`97-^98pFG(N~Dlib2)jgw`7gJfi66cH6=mOq+Whx;~$VBDtbN^3V*8+T7f zNBj73=rGjJ z>TuBd(dmzZ(CfMk?-sC8p-$$;zNYv_&qCo{?9DxU7WFZ`$Oq`6W2# zyq4AgoNMBUsXadAg=hX8r_5;}P(s-Cmj%v2c1zlmafLe~J|wX*I4H-%VAHaV#RdHY zhI_z~WV6#FJXP)O%&WaTQ~Dbk%25h`{rUwsK6;&RQ6n?OoWZ~H{rmUiwjjxdJb4m~ z58AqQ>r;?8Xll4d;HNp>$yC4kA$k6-Wwo_5xKOCG8QIx)0t0EG8Ta+}QXt&gkG9eR zaf-DivnMOFS$}!Gj_-8rZB*{T+DPy3AH&Z%?2=Pr2gSjTtwxKb2X}_^`}Fl|21q&B zPbDFo^1=`WAUr+)^bVzgr92Gr4?t&n7M2B#8N=rDn{SM@ugcS2uHE*_FF3fG@Ccyo zcz9B1fe`LW;KN;BWJ@#W6M&3LDjTo{)Ps=dXnzp2y9!)TVL12iKRs~eB*X!nN!r-I zDWPjoXW>0w_w)AhTGh7O^vV?t6K!L+k7(N8Y@NM75}6qC732dY9tLbfcty4~>UFjb z**mj)LZO_$yD>=zEpx)WK2{pBlZ{$hTJ2xHWRKJr%A9`LJ_jo45H-Iz~_kQ+-3&GP8v_e&U&s_6Lc%vmQNPWm~?aAC3S~ z*uY(m0|JLXP72P z6uU1W&amFZ#AIKwciMADzX2Jy1yX{cvP>M1=cDGdl;;iQMC&4J3QL2Qf%pX$jh%12nlGzb4IVC>O?(5$_Y}3xYz6xoPeOosk z_qLpyoqhV^g|BL~IOm~5xt~Tsz31DCm+$QK_`3!DbAIei{nUrO=~>3}F}jY9;&O6w z87B1{Btldj#Eyoeq%6@U!nh4EBtD*JA`!6WY_98XA#!ohF`$X!rk^2m(xnAp!6MLb z4@ChOWDP=Xc6yUv^L-I8`0?Y%)6-96SZ>5-XJ=PpD-$1#6Lz0~qk}%&kGnxZidU}i zJbU)+`k$Z1P*lrGOOFkRvWs17I5(H%a2~l2?78pDG62|{5S~DI;JSlDWzyN_#5)RL z7P-X`{sO+f&HR)B2_+#=)_aPrK2p^8-fnt1mTm0pv>qNSyxHHUok*_2ePNgn4|O#K zsxeU?C?yg-H@yJ_8l10+qG*z|4vB{m0UPq?YPq~qeg(i) zPoE80XK+nKA~vcHve_M^m3nC%o`j_@lB8Z+4rSnLwkg_1tz=yECBrS(<#r@b=(o9J74NF#m zKRWDm%?lpo7-$+1R&kd={^t@^;`$12)6FMnxD%I?8 zQzv(`P&)ego2zwYXvEVfzJ?&Ssyf+A3}Ij;3JMClq@>v3CZQFz@b8mw9J_Gn@Zsze z{PBVC9tRo|l)D<9fyT-2qSuEB0!wE#;!6sAHI_U`f`P(SKoEk_Pldks@3MJSNF?Qa zdgBA+Byaora>FqB{P{Ee#yb(>mMmZQC*8rhvKeXKLv}G%oWw?nm4prw?X#Cb?u@1; z6%df5@sxwT{VqT|`1I!!vj90jjL?-GC%QJEA`>GDIBWKI{5ArVU0$NH2WM{#Puj6+Q z+<4*)dNhxWQ>N>0D?r$^ifNqWEHE1dWniuAxP7FhN zx<4K)&5px28T6Iixj<|fSd~>#J*&)(k?|dajcf7qTaAjI-jm3Phz$5Nndpw9nHRb` z^0{MFVE7)f|3M#(J3*`fHnFE_jAYmyU*9)?&6d7x$&i=doR;}>`%6d10c7CBO+K+< zSr2RK=%kJ}C^72fI|ddm{pzf(tqq8bih7MBlL=+g(zj&~qP)LG(~A$FSW|iC<>$YF zmMfa9@scF|EPufMA4BQerz?B#U>MB4*Z7w=@Rv0M0>pU4#2ya+_>lpPxgay~?d8-H z2sErGt)DP%aOA#nUn8U-BMumERDH=i_W(MF?Kf&TB>s3lg!cP0?JGMnS+E8N1UF%T z?a7U>4R8xdMs~Z5?i182dhC8RCPIO$6s}1IB43d9jTUBysW)sm2IoDC{nCqgAsaRp zmW|j8M38k_9Dmz4Fwm4{lVq*1@vpS?$pm#EsJNFe@8y?T!#D$HU^ePfJ!hV#)iV=i zIqFL1xmTp8VFv9{v3lYyOt3=9q1CE7iRtNjL)|Lb;X8mOZ>-%;y|$}v5Ta)uXF7mL z@3a#sjg!W8OFLLtY_%F&B;t0n_kasu|IxrnqD#0%QW&DOa}|Wbxp(cNP>{%N5s==4 z4pok*&%s}37ZZEd&QIh|5@Gu>@We^!-qW(Pd%*+sJeCg9u^pl)D=QQ27g^Nj20HN< zfig31OzI+L!;Uf=A!Q5YS_>%Wl7K>NPVS(m(e2SkrClCfR zgrgb?A zx{r^~>6@zmMbs(Fe;%9ho>gcc!Y98ZRXjSM_C?CYIUMIWj&U`y7ov!AVU9{V>MSAtY&IdW?L`z>qDz&OSZ7x1t(o^_&BtJ66^*O39 zCvjJ#xsE1l*cRV7=$hT^*BGjMZY4-UtFC+3+~mWB-)UF55+=61{4x{2tYcBflfumm zQye0U*rR0E&%A1n@4oGJ*~EP!=+(?`Q5hKyxRM|hn+HF|bdK1Zeb#sg3>{J_IeG}u z+PSGlJxO}_@ZrBm0)$HvcbpK4(fPBJ-_!$r65VJ1$MaxfaBbVG+50?L#$}L~RXIVO zumQqLV6GzBMD;S_7tQ^eEC28uL|X$8fc8RH2{tyiEcQn*Ndq7;!6C-dtp#%h9zj$L zKgds%c)IM8HSRSws!zB&L{}x9et>`{Vo3&OW*=g%L&3VvcLiZB@D9&2{NK{#qG92c zlu!~A6Z!Y=CzO>~@87&M4j*SNc`oP_eqSLbD7C%NzK*9bw-zV(L-yX zMn<{o&T3K7H@vi^qfo1%l8?HnqLJg+{b)(q{z&lREl8+B2GLT^2XL}&u;aw@s<-VH zP<>-mP50_T=4znExv#7#&VC<=+=im<1ulUKiMshrb}U{MY6=O(!z%TLQ(1%pZ<3hy z2jYe>()QR_2aVrR&#E6f={7jz@1_V29A~VQ>vRBUEGds zQ6`PmM7KaC`LORzMn+PTqMO{gH(KRzL8+*yEE@-)MEe2!5T}axv{2c|4MAods>@I)bI3brcqjvdE#(O=&Xh!`@w z%dbBw>E^n4Km==JrCCX`1_jgxk16mEPpTh7SmIfMFan_qoM-467}OWPC+-%o&0|9> zy8dmsc2~VEw{2W^@7_H`*Y3hT6? z_sUW3nTEi@90Nnc)lg0d3h3%$BTxVNpxxZ>gW(Q2n=Zh$Bj*Xx#5<@41q&l7P!~fS zdBQ)!#-hVU$FEv3zt9l4qb)S#qT4bKLXtbVGC#06@XsmLgy+x8m-bnH6A|Q=*DiKvZ)s_9 zny%bF{l(;o&AQdiKrdL$Te$sQXy4F-2uFp|)sU_yOL3^b2*o1Par_c+<>gg}#z;He_RaIl~GQ?_cu}iyTMXq~KZ4{vT_|W}T`iNV#h&B%;E<^?q9@iBiA)(reircFioPLaq8~~aa{QkWX zp_NaGYB3++99MzNROeWHDI@Zn<6!L$$KMk=k0K+#y!1$sWX#k41WfV~V6&Q%G4#YE zcDy~)8`)aIMoc}6iriz0o#&*%3z#WaaWYVx+F__(gb5K{nIQ8!`j8D*zRR3IXw4Nt zOp+nkkO^9K?D;r0?~W(+Cdf&=dJ}^u^#-}x<3^FlQHzR;U&Xx2V2;~B6)hsc<)~X< zAR6T5K-{Q>Zv=@kGnDW2fn<#j;9KoeedQ!Vif=J1w87^+*2%r@g))+f$)iN<4Lrh2 z(|ohRnD;qA5#;86{!97olZYf9Ufu)H09uPZJm5sM@|H?(%8E&X>ILiX1EN-)W{Yi! z=x>AL<8=^hFCxv9Q*L@5(znVk`i$MmkL;5c8q2ha>=+A${6?nBGk4zcMyOy5r^B^T7y831Hj9dlzE0VpqCzb;IGE8yq4_3PgN zVM~~^!sD5p;ftq5s!%K<0(4s#A~}^OmLL;NBLf2ZYqUr5DDn8(5;K$Qmq4fvKB|9w zICNFRq2U!A0%i&x>w&-=2)DJLKi@~-z)bM>+?+9`9h$e;qn=`qC#&?aGrL|p&kTxC z93I4gdR^7mFM$v_61WI~zRsg*9}j~cGsSVZF`;As!N*AuOoc&4#gVPiymYA}XN#jv zhnmJa+!fd0&`>oA^2e#Ief{Rm<$^Q~0b?jc7SIkHmibLtcMEuoDph|4?21fC*sY^r zW`wq70r*zi(h{~xA6v!(u9&K%Vldm`G7yZB{8LKQrpU&uW2y0bccT)(9w#G+Y(Q!E zzqy}WP)01!7c5z#7rIXKFqjH@tStTBK|%acI`BKfR^&k;supNz(Vh|aDW^qRU~U-n zYfTCA;LJt=8}7(qMq;#Zcmb8Io|1tFAZgPU@H{nHLjbFAhA+}E;H%a3Gu`ONCCu_p zPV=H#rhfJ}v0dgtz;=ay%ygPIwDc{HgdIsD6sQ>pIAmWOb)WC^IR`WaFq1GMMP2(5gM1boekPH4#o}tO#@tS=|eYjl8UVwfc!!6MB9_>Om+)x z$*EvoWAtiyW7(XsIAFQaO_|~%p$NygGyLrceqO{Lk3eTa`_o6KGXXs`IhCOP+!ASC z*P)mj5x7napBW)@O}LxGyK;o2+Ce(o!3-lH`HG5(U4g7do&1aTT}yrt}L^Qzj6&RrJW1n6z|^>Jyn5JWG;rhv=d; z=~R(;M(>NNl9GOU!Ox8lyp0qU6$OQbLy$=qpLaL=H6k`=(%4lWFO6dmH0@O2IR5PB z1A}rO8@tt%4DkCmkk)Lp#94mMPa7q^cu|eU)S`A@d|dQvu8onm&WKs6vxOTPP2fa& za+J%>Rtsn#)sV2ifK0dlnHd_jc#~YM<9arDv=9?0`ICzmAH~GfiCeUnKt2Bi*@6|4 z)e)QiwdayGw4;$IK+VYQ(StSWxG-zFPf=B=4YAcqaqI3{NHEdavB(H40x@>*zbrl| z7Nxl4k@m~^rCgjo%!%68)<>ACU}9sd1=iP{%mF|^IgG37Vax82762Pl(b5V>HIY8w zWw{4o&{g{3`}NY6)R*LoSFc{(4zkrL^O{Ofw;%#*NXoyauI?wq)-gKt8|NPSPnwP$ zFE!}2D8&K>;VbmHZCkfyK)5_0^Q0bATZNJ$V&akf`0TvgTwydACJv5I@SPa_6a;yA zZm-gh7Pox)CN)td$xJ{`Pmg! r!UVz3GdMbfdE&k;W`8vIV!M`d|3z64I$hr&z zRBdstP;jO+B-OmvUVJ$HZ_55rB5SZmXAJ@}SC)Sl z(@++He}+itl?=1)Cqc}O8x9?w z`T3LiNqxHLpBK_(-5_U5eneO>)n;|e;|+2k3D~k3pTyzn)}g(Q1bj8OIPe+869o<~ zgk*Z)4$;JJg@iqfFQZN^DYLM%r)Cxu=`}`)G^=SuEr^4x4z|qfA8}-IX-sq0Z{}0b*igc(|6^{uWgY6u5dymT65+6-fNURr(l& zF@P3FOSjK#Vr|?)l5dWfMr>+aje=ZU$SQpXzxdC;o4PifSN;6hsyIqS?ECigq9-TM z$ue5L|M0<((vCpS7x->3TYT{a-h6!7p-ddX)rc^Efi4yq7^qsvIM|_XC)4zZhi;#A z&ZT$7QK$wo$zLBIwnOWM7GZVKE;#=e5k#qu8XaQ&4Ckt>`{s-)+77GKva=Kisg#l$ z#-Ak`@muH;`<^n8S97pH`xg0>WSU0-d*YWEu(Gw!?!flPk-?B2^=T z>61BI$c-K?+?H`@#Na)w1`9C7u5}NfFtr__$xBMQn+g`r+HZkbJu=nnh!6r_6|I=d z_RtpyL6K=6Qqf0U^Q@jdTYmAPPI5sY>>l(vp6Q>@{x1s<9LRR~0p^H!>8aa+-M*ly ze9At{zg1O|QzKY(|Gs_35KW9x&}+N83e@t^ZA>es;u&4%ifCzRqi5YOJvc8hc(TQ} z`8Ne-!vv$zwNLf&d!G2^0C7=y&J8!cI9beS`G+`RpIxLlf&)3E-|6_e9~=^OQ}1tx zs@6P*fulr@CxO}LHhV8Z#>q9Vlb-1CUg%f9-13&MZfIa&rb6MkTc%-ug^f#6Z_c%75e=_sdBL3Ucl=$F zf>q)5#_(R6Rm|IG2AH5GqCc?MEgXe4=!g)=fDJ{v-F|<)dfwwd7JmDJpbr>Ma`JFf z(RZdT{*`k=_4W1Zr|M00j>UImgGVOmci>Mm-zyCEy&J>e=sc`%n= z`$Tr<%5RSqUw~?PJ-y->F3WC{OV+!CR?-g!rzVc2i>OM=u$m*Xw4MShv+criR<({f zYBf?xmKm#QO!ZK`%+>FtkBpc-mU3pmEG+fr&DL}E!6H7;6Y#hYP{0o`XowGT@P|Jl zW`NN%)RVV#lr;4acq)(*5vN1t*q*Pzm3*Uk#IofCTxduRb47qw6byq&YxSf;a-~6R zT3B4%r>IBid>2J_xEW zFMqDKPA2SBf=%@TVCdhuHa(E;qX=2Py@?P4iHee0;C`iDOIR zdZ@2+Imvd8Jz#YB-T|EL3-9brFIQjKkMF9es3<{*(Z$6Dv!i9O?U8p30Qs=p*^8z^ zd^JoD8=xMubP{bb!?IHx-iKH90C77JKe&)-ct`OFh7zAKBO?$%LSiq}V#)x{)21)s zIc{NQz6-RGeXUb^G&S6F|x9}SI+kS$q*Zh7d?A+cO53@5*F=T%)&$?C)P!CW9& z-b*xregBkJ{Qv{pXN0VW4CXg#W7setK8b-2)3%7HnMl~}fwE&{ViHUw=h7@!&D~`0FWrdlbASr;X=}ML2fQHkSa_;+6gTP z8ho?B?KlD2&#oYTf)P{56Y^SG+wocf93BUT4Mb9Fw2k{-T@ zkxi>1?CJ-=@PGR59SejR#CH&s_QXlk7HU%}V|gcxB+{C=Gt_(c?j=$~W8;IxqCq<% zJYd!f_5+6O#$BfGW<5sQ7u_O#+T*zl6gjX(?EX7xX+jv)-NMSc$xq=~wt_LZ%%MLk z?fJiRV<%TU{=U^p|1`8l@`df)Pdr>a9}IX3v_!Aa*+tb2e?$0p9hOnGaEjC}D2q_9PzGK~mtiUcI9kOIJe1MEk;ME}* zR3W1kT;##!84>3Y#n_;`zAmg!jxhA{e$M8BL#T88;RvkL?VRYHC41 z!lvkdvqT+R+Kv=T)WL`(ur$qA7mi{6121|xfhi6#ydA+xW>SLvOiRR2TQ=qs<1sR2 z>)^2a{P`Be_3DMWj_W8<5Dece^VYekbu?+|xgEzmB>Po|i6N7v*O{4|dU`!=YCn!D zXk5H_5>`i+{cxk>-Jg+zQZFDDis2m@>SxaqcccTl;qUw$C+otO1&J`Hk1 z&)p=3T4WCaZUI``HRuLe`lYMM%HdwEEV(#>Pawf4<}!QjQtoxmeR@TI!;H$X_3zdX zHNm4{(cX`+!vo0O1na1tYE;Agn%qs9Cq(^jV`nMnHj`S17bR6f&a=TwW0Jc?Od`Vn z;2@HLCcHRIi#r7buO=v{)XG(4^(IHcYfM(E9Xp3?u|(fADSJv zNTePrYa`u@j~|c*g{Fal7BVl7+3B6Ig-IOg-GM#G6QgHipy!)7NgPH+$wBPLv0(W6 zG9n1#4U$_s5zHomQcOBP@h34=jKE>0>^TtZ5nWha(C&Ru?J!nLEETJB8X6A`RpXy0 zCi_GGZCX$}l2TI2 zvrLzXj%a8s zgsE;D#DfJX|1Z(+zOr<~)VIluEE=Nxq>I+ps&w=Tvp)oydH$Rm2EPq*X76iiP9BMW z%d&+G#vrXlayVp);#<7C;N3@$m@v2UQRi+{WTedozoMdLR#o~^do{2Z z*S&x5i5Eg)seq7j<16t-36fdN$?-v@&Pth5)C~)0Zrgdy<-kt?#2oR9xaBodg2$3m zQ!CLg$ndFBz@D3t$Vrir4VW_|&4|S7$G!DpyBPeYD!1$P=1&%3zMXXnKGebQ2K!HI zc$zG&V7kzcBzdcMdOYmtb@6fTJ}tX+8)b-$tzp`d>PdaLqf#mX7Ld&mTi=GD%~az> z;fS;pm{!mR@n!>p6(Oio!I(!d-0sHEXmJz*;_dCZ<$MQ*>Yw;xn96%cYkV9&1xZOV z^M^NBg+E644~EZkgrSil|0X0j$`~4Sz=*!-FlLJfKz%Sf^5DTOh$|$%0^AUC|Nb$^ ziv}3;hD3olN*MwwI82RnI&!YO-07I%^1&j~Z^1MInfNh47pjvyfm9_p{t8KoL!gW} zWMi08(b@umKp#AsnwGW_#-9ilB2i|x#c$)(m|7zozYM&M1R<(^|5Nh>)BMyr293O*LP+dbJ;)K&Eez{+Gzm!0-X-EqDbQ;+@zJPt(&kLsqhSLK55g zk^!s|ba>5z_6^TpW~#t2e0RRo4l$FQFjvU9vdGw zu(JB}&Rm4u2iJ(%sV#_t<83`wnhFR|o12@{uuEWF;%G~+dGJK1?+Xla61Zs#WHUXqB8qCtY5vcO}mXLWHn8EjmE}(x$M@PEK zHSD`6>GQ}KysO|slK3YORwW`lnKlE8Bx{u$guyu!O$E#j;p@+yKfex?4?IU7G>Qx` zpgh85xTH;0t|X5qMJDVqn2e?7M79d;k??DzTHm(pkHzGtKV*Bgn1Cz^?6!KDMH(`$ zk5nk(<9JiR$qoaDYnTo1V&H^!S&kkJIGph)Hq9A(49|fW$&iR5Fg}lFOzY~p_=u`h zn3fdmwzlnf;oNmwTQX0Fd0qbTjCkaPnIS=XUl6P!?==8y^~CaUtzNSx^J*OnQt2e+ zfIOe+Z)PkRW;m8UC=CAbXY2iP=aYY`NWF(OUbph~VoFkylj)6@3#&3oq7BU$M{v5s z$=~1K=GWLo67~!i(ji<5cng_TD)b|i=C>m{T0X=4Bu|D62j<|paW#03sBB(XGqhc7 zz}mV*V|{&31ZJKhVA<8SF!?1wixwMiGYM$`yR8AhBXJI39ru)~_eIIrLHfArd4N9b zd0&|633Vo02#OL` zT~Fa?_`^6^Pk@%Qt7|9>1f;z}@V>i>cU0I}S1c`6;bo?4uTUs8A3iWpUgzW_0ym%2 z(TP#`10{GCLsFmd9zRS=1|gJVigWU9bW|Co50S~?2k-%ZK=~2F$SA?Nlq(h%DyTao z?WcUCKyA-~0}o(2DJd%YAjnkH))pQs<0gP-uE@ji8KS-^bR`~cZoGKzY8Fy%7=E)# zaquz!aGd3DOXHQPXe_)M+?nT^idqns?T^a zK|H&r@59HBRS<0Cq!2VKiE$@C1uj$H%B%;VZuc=QiFIO;xp@epsnU^nO>QRtJve1J z3#udr2*aug+lJg?x~j&=+lAu{i?hS>MD8Wdkr<{+gbqhSmRiE8)wpy?V0`3&iZ2%) zCEUU+Y}Eq?5ALRbf2-l!63`?~ecOgXTJGYNdCC~dj(_^}D6$dHu)LvX_#Zl68|T7H z0gC?qR6Zaf!AJQvHl{+Jc1vq(j^kPD*d(RL`*?j&e^l^yfpSd&679#@MRFSXn(!*B z!?3*{!ctYnd_sI`sw!%G0?>x3Z|j~eylD)rLJ5i=(lhd|zsBM~rN}xQRaK=0A`Kuf zEpqE!zjm$Ckynm(-`?M+O35lVOVNvbx9p0dCh0iFtgObIxC3J;!!U++16}q{JAIp& zP(xQXZMalZa0vmyB}gLV*MNn^#Kf*48Ah&8Kv3{o(CAp|J4hP#aGMt(8b;_WGgykTCAZFUg(}CGlV-u6N$M37~A@!gH(8G(= zINtWjE%W3wIW34sJgUuUfT4Zu%WLEKSFcn?Mn=9349Fw)AG0iynk|bv}H6LY_n?$JLMOJ4~ zN>w9-y{=rjk_0^~3t$$n$GGo!Ek+Fi@Fmp!ZzCh8fOhe0YO1Pys5Wz`Ac8lkA#n~Z zAFnoOLhSwr-rmQ(Z{NPOZ4+=e6A^RHYI9Ij@uzB%IDxmspmcZ>um`~%1^o`1KY3XQ z^1gVD=_6VicV)jyLrh2YfoYO-FWMo}GoB(wRVC%+Yv7p}`^RTLfE!ENJ(fukgLtUi zgmYuTF!!u{L~3va+F2hOtQYtO2J`)at)M=wCp{c#pRDrGdx&wah2SW{&5Rl0*7y_~jO$2R*;~|=o zTTIbP=K@SY0X&NNNpMoHlcE2wtuv3xd5zxwV;+j5OdUj0W(gr>Rw^>j8fZX?M2U={ z5urjUNm0fmqKRlw<|su8QJIn{am>HZcFw!r-&*f$oj;DD=lKrzz3;uR>)O}un!zJO z`@LYm*2DyqhvElGXDw--5AF&CDP50`_MkICKMNd9fYvCL6>QwyoQw7^O2bnI9ig*0?=OUbC?5#I}pZh%Fy91X@o0EZHS3oq-rJUa0ZeY_xV*{k0WUceuIaINrn?3}`rxbPA zew7CqBZEv%PCjk_VaUc`U%QbZ_Orj*!a2;Aj8-(GoKVtej{Cras@*mvb7szLBZxm6 zKNA31czReo3H-<#<@}bH%6hf5ojPJh8ata__NVXOJt<2t@-^3LeeV zpeP8mjO*8%O0N~&wn0HttA0m4-u_8-PupP|2M;Ixr~L!t!3OVpKRh{Ut{85ih@o?8 z+I5vxeGrTbz(@Lk0BhKQqnuWPSe$AE?wG3)<6Dz7+lE~K-PmcGy~-$rdeBM7v$uO zEs1h(O42?%(Ltqor{UWX|BIw+7wm=mZ1c7UX+EZy;{)8UeVkVb`%msQ%F0Tet&sX7 z)dTAUeQ0*sy*DoMsBCS7R3 zx_oxecFHtK?*Bub!6lsU9>02X0C%#Qs3Q$bY-=$!;2m?3mSdgm?Mdhw>EMRtRx$J1 zM1mQ4Q-Fwh31PzcNIEW-Gj!eLpKMpN`P+vv(g@5G5u-OSN8+0_Fw-I?-M0;!cW8`^ z;@Fp{>c%~L_7t%x#=b5mM&(|)5)~WUko&}6wma@H(r|+Wny+gx^wWoE<6$ z>Dfn?dCqS4vX~taNA+SM0!TE=&;RFz!1GjS^y;NV442B^0?E2CJOBX*)yAb=inzxj zLxlR264!LnBomzdz|On`-eHXm7!d5_d=F5SiXdCG9|MP^Pox_Qd`<{gevT8i4w>v~ zgbcF_mN(iH7Ixjr^Zv%xjq588Xdk5<7{g-~r$v&pD4r!e-0BmLCmDI6f?{pc&D4bb zC&2@JnAC+u5k=X~f|Cl5CUb;0^+Xt#Q*<#3YcOf=m9CS1^qo8UX1VptpV;cK)rM)5 z)Z!I|nh$M{7ZSL{F^gxlAGAbzlxj-Kc~Yc4TUZe8R!V2B;ugVfzYxKs4hpz)r^cy#Ilqg!#`|PIwA=-XT+_@5s~R%pSL%l#3pSA zYn^i9%B&GzKGVpU(!)O@BEpeJ7qR$w`YF@iZ3@uKa?JVhzQkR%@5IwIUmx6CZyLm ze1IFeteiG;W-LFO4{-1b2c0Og3F3;X#iE1Fe$aGT|KOPTPSeP~&{S!0dH|pS-%ehJ3-n9=ojV%DfG|{z!`A(Cy14duTwIJ4F37)3(b*80C4Z#5 zq(?&NlJT!+&X{2=br*_jPy|)1Z|+Khr=fAs_}!t{p^YrgFM0aR&E5U^=lhp(6uWlp zn0Tl*Q>nl<>G0z?s|_a8aB(4U>(Qm;=$rqRlmV_V02XClTG2F9Q@_Nl^%jXgKV_@K z$qfXx7DPkn3-BXqUa<(=a2ta~Zgxv9L14wtAE0JYBu=0d=+j0|o7R%5NEG>DVeLYO zI5{|&mReQTd~KF`o2qO@y;GixZf&hy6<_uGwTA2VjQ^trpl8X4g$2-d2pA72MtHc9 z8ZfRgMjNlR=@@nn=0=c51)= zu3;dDA=}UFX2+?Q7A#yiNbSNAf;JK65W0cMvu62%yYL3(Hb~p)*Bj09v~|&Vf6{yU zgb4xPQ!ZSvj83uY-YOdYY2d(tDB4X(tq{49nRRZ}l%*r)3zCM8z}BBKZ1 zrJhn`k=w*x3=yj z-7+x9<2pRqpV|kXP_fMa@+B9I>=bMl&~(_5J<}hQfeNqW&5+e|l&%m*Xnoy5YVQ9t z-s!Hx#ETE3f9$B>FqMm=tonm}xRmUf+&!Cn|G;OB&5Z%Mwjpfc#~(b}ZCT9i`?V+f zkSpudS-l({_*uh#Y;9p_Av87x^SM+ut~o-q}d;o(o29$%M zaWd>2FNJK3?%QnbNqwEs1w=$EjpNOMu87Xv626k!iH=H7{NPBsBa)6APMO7n^rj(| z*C`YNVj`U8O~@~ieDkv9ebZsLX`~`*vLA|udqqX#v|8om+1K@4+v(nkOOBHx?>H=8 zjA`JWhtO=YsOa9+nR5Jh`UamKV^hHZwi1au)1m}i4aqS@+*s&}*(CCQ9zf|YK9%F^ z5QiHJAO5ky`!~^pXj8%tHBfDJ2QqR2Q;Ul`)6%TarT47E@O5gELN2@+nxP&Z9Hg+fQ4A_?vJR*oX}=G;FD0l}4kNxJw{E43nh0v^U(LV)BX= zXA;MF9Q*cmN5INiwI4ooR!GU~9Erjc>lVE=IvnS9N0Qr>rYx|xH`QpNr|%B3Uf)yv zt=7|%U*D6Rq`iYktz8EVeR7+^55~1tK^zi25-tr{X>rEBNu7h7u9MNytUge%GfrGL-h2S?v)y5{YVMh-kpN_{<0 z?-UCfMjQ@B^-+te!Y3Jdw{Gou5P$4g@yoB`xl&eaFunFgn{8J>{c*kven7K`LP77c zgUf*zX{L3{=DxgQyZr3yZY>(01`WjqVdVJn+j_0N6*f?zaW}ene`=k}vkj)R=Aqbsc~TB#VV*?<7=wCOev&G1Tlmueo6y^N_#<;%V@|Qly?ek^ zrfT^3NRrrY`^CDK*LS?AA8x~@IvRMOu(s`4q{^P`IW|mZIM9-kxh9U>IlTs1&{}%xolhCbWeH)#JZF6U+>$bZ9IZ8 zWFjc(GI#?7M4f7vb_2-*6qjiX()tY4R1TwzhjJ$5vepJ%D7~p&{MUmFNMX);zwf)A z#0qxJNq0v*Mf&ewYdKU*PfmRl2FOBfEKU2OIN}&9ZQ#0KLFM>P_wS`TE?%rTWXRu_ z&yLE$30`hy^-$|Amny6Dz6`l@|9&o4t+nIUty@uxatN0aPpOVs)4ciMZ2d9_@Jz}t-pR`qEtL}6AN!lG32b-4FAS&YU}^jiMoyK9E&QfRgA^ z*+#s2MTHYX&?hsAhVKZZ%O^sA`RHe*wRY2K1JwQ;Z^V+ovO>Gem4G13mk{7|F)YF)f~_-yZi%&EZbP;CWxLNurbw){4;xRsI4Xmbr zZuwpK-UlUf@(TwRO|f~rfmBr=b_>5KD%pnQViFL*S%H6&6SM5pVw*s1yuWSuZlso~ zKqhcRd)-a@*ph6Kw2%#(8Sb9>WzhZV8L5jeZd&Bz+uP#2GA}uYS~Yh<;`>3w>r8?v z<@{EboI0~ob|j8@{rc_cem!qmY`WDUM>k@gxEpuu*qrFe86+|Yb@xHKKChY*U^>Bs zimelPvWHV}t8bXR5IDp*DLW_Uj8@E+`Tk97ifjh>_A`G`h>i9B|ExNtfENT4B8ikY zzj=U7>UK&CF;`yw<-2b|d*Sd*}loHezh+#C@tl@=_5?sazvh0QT71vt6Up*AQYgEsK z3&~x_+dbr53tN|d{TbiAtQNIbYFs zN!J~h*{%AHT5JLA*Jb*SKR}0TbnWV6(Ue4YT1nKoeFOlJ&{9wJ5*ujIx^V*KH{a&? zvN_K}CypG8$qLdZuQM-QnHmpuuBkGGckJ3Ucs@H*9OTtLR zVkL!6+~OybpNOR8WU-jFOD2i%N4(=HP3N(M{RJV9I z9VVPAR~d1qI7rCdA__aK-lk@8?otF_5FSkz^Fl}82RdN$l7|Xi z30F0nb_Ce+Fdyxn$~6X%q2%;ggb+YB5UEBOv&|>Z6=9tm2|ps?VsUKfKzb|YHPT=t ziyVxT?qhKz+bvXM8&|NEh1_7^^;YN3Xb8N(S;#skgds})dKJmKfszYZ`}$QpP%-uYAD$t z?Zcx{9tsnQlsYoNuzDSNly09XwDdw;{aFX~RGNB|;(5;Qz4w9ST#7%EnkC~(zinK< zc=2Mf-=v{aY@?8EjHD}MeZ57YT~WA=UcFkM9f!i(#C7I)zA1>&7 z9UfSkN@YeEOEs6rD+b^~iYbquuJ(MwpKBEIw2ZLPIvJ%hY#~}`QmQ~kIy#Klm%nmcc`E))* z4`fhrsS3TreB;dJFuYR>qX7KqYuZR?)TUMzBFY4h5&`f z21G9i-7y}e9N8kQzc?&my-2<%1uj>?MQLhlSH|oNK{+0omrF|r07Yt>ZY`<41jPhD zWxdEm1$tjKX`E}{6KjdT=a2sArO_iG2a5nUD6$Wu5jO)iqzcIACAauOBs&WOfNh z^TJ-Q`!759?eO9BjT^-8lAEx$*6Fvm;wx;X0Qh}>Ih}a-e4y)dbI}Tj{WT(Tw>X3n+5@_33-}xz zZF}eiRrDD2l3&j9NZ9wMX*yyjCrt}zM~#_$Ser+LsuGsNKik-ayffQQP>vHPlt3|Q zDjo@E_G~Q#4VH!h509KWb^Gn@O9$C?SLLuM{_aiR@(HkAAvp$mI2}N1?w&Vqn5~&Q!rL@6HySBvvcM18d58h3R(XX$Oga&y>7_idfB%gs| zf$~hyQHo)9|~$X>`mn&`JCRpl?AGE}zXaRbWZw zZL0Omc`;@+7*LYWi7QIlhkr3FAM$4s6aV%awLms06%#Ei+M~h?HvNT22t&}T>lZjJ zFf(EAh)Zi<&S&wvi*j2llv;w=VQ55pIDS76wAOXqfdgfB<(G5B zS&8@i_RTqxl3ab17)@N&4BU+*>kxDw&ksw_a#dAYJit#bd@wKqYP~Ew?XF3ur*t^1 zsao`0Q`0{?2J3&8ajEgQns>h9gfiNiCz_l*TXPByzs-uGUYSZRi>(7__}iN3;a`vc z9(wHe9E8uBbEX02I|SHTy*5Zk=+YLJek`F-+;ku{&lY8#@|IW~$dIkJVp$~VEX~3a zcUav4Iu6~yMKFvhLrA)G&l&I0ok4;m_56?eB7}2k3I*RqP)W-$PAAF$lN)~8ZUU&y zN%2(mItncyUoZOne{8qFphcCM5Huuqsv?WEWUe*8g zyXfztn6pkzkhKE;s$hHp4QslvaS9B-_KM6#WaMH2 zOdpV#M&cFV;^yXsx AYTSm$Gg$MKS>t+}w&n4$`?w`k8nISEsy;T4JJJu4&80HD zZvkU*k_2xktlfX z|BN9f2&oItD_Izj0eFTjnkThkYxEcx*bF4O=wkFfScMk69QZDo6DxsQ+y$tR=QAXh zQ8_2G^}~0ZV*Mil#K;^%ObmdDRlNGWyLFWddJ6Rpv)B%{D4T33!J}p&ER309@7VG` zaiSrxmS{zb?5HIv`4cdbz+Fs^JVLzWHAvKubZsNcT$Q3TeyJJOIe^}xTcj)`$Bja@ zkKFCg*xLRJ#`%T~0sPe%K0N*IVWaWmm5BEzS5hs`G&T|w!VS#c5G9G|Az?fF{b9Uv z+=l*nSo4WMuxq~cXB5c%;=m_42p+s~MUvLVz+kTJ=Q!ex z!eQ5L8T-R#jI&bru?cNREE3Pr{aAHDz03w-7k47gPg6s^*qT4@e%k*`Biw7n% zhc3UU`M_eb#e9$A(Tj@OVDO*EEwd&7GGrOcOj?$q%%1zB$%O5cknR|BGbAjir=rclzk z|Gx>g$t-3cFfNeRb04cj_kOk4UV+)VU?(xC@gAE{cMbq)_~EY`(jD4YJk%F0YBXum zBvE%ceg0;xTwpG%eaA_n1{+AiGGZ_B9d-1*7b8hH%d?KmKG5vd>%=2F3JNbm0O%W@ zst~A$+;?uW2|d+?WsL&Sm>2*=vmI)HuxxjIcMs{2CMTnAFIagRK*iuV>hbOBhHWTL zBmh{nzR&=QR{TdPu^v|M&+&vB@J4vie5rKk4#_+(qZkBG`Sgb_RA4?R6 zAr$bABvHzJ@wLO1B6nf1{o%?T+Y`IYSBveenTZ*Xm}7GqxP@xbjB?KE{vwZpkAC9( zZJUtX`q1Ky)EUVTN2JKILZVYs=azKtr(EDqyGI$oqqND2LJ!xdr;It4foTE{5*`H+ zLr}TMtZA1SCl48BxsAlfZ7h-oBqGI%D}za?eaKDC47bmWii?wuJp6F$@I*6cX3(nM zrRMWW3#!MZs;CcmNjY0Oi^^HZm8V(0<>32SNO25qHtXl((NXf~Eh1i&@zx+Hf2v11 zi%ikaxRhP4BTWjAA3v@xNxhM4Oymvu&G**_I8`^v!MZfS4w3e3WR05gUA)1ugd*pRodo%c*tCku83WbWBnbD}}0(NrJLtj)9 zdU0%MtfYh?Hva{Ou0Khs+il{0Pr7ZT2qPHNic91eA6-?*8JC<(x1zL(ak(N^`@H^h z@uK)X?^as(K63o{xMyET9%r9h+?v5L4S2V7JW4M;?6t?MA;H1H{V>cW{g;YBY}J*N zoK~2}WuUY{Dw9c0TXN14;Qc&IS_gd>1u&r)XVQ_8;8@UA+FWR8l7n7+BRBruV7{-v zw{{Qn7bDrtYRCqtx{Y}hPD%ZRn?m?;knK4YU;WDG&+|ETn<%`!yi7Zsh>jjj7zjAf z4TBlQ9c*34qtJ>Wzom^LQq<)16+9|i%~q>B5=}uXRn^fvrB*5`qtNt{`dV99Pb&;Er$MS3!|Z?kDlAPPjy3-*O?;#f!sdnc#yge5mBY9yS069s9|Ig5R$-Svpd4Zj1^J$z%dfNHJT=e)3i+tA5yo}0djpQQ2Y=a(lT zIYl%5uM=JxE7V5-lsur3q`$5%)8@6WBSZQ8u(()7(e$ssLiB%eq}0b!;mzajInSY> zw0N@^usNjdy`j6^4Ss$qQ~j1cNsmAfX=Y;LtJuXyVZB;FJbuOcoo;2-r+V_=u>-H1 zJ~dCdU=ucL3Qiq{(m5pcr)1el9JYO$=}k*hS?bzHtS*+r;*TEK_}+6!5_faH;~AM} zz;I`>%|2VdPY?}Cdx+<%U*nV{(Njyld1*R-hQF#^UgzI6DC0+3n;@MAvTB2BpQ8&m+ z>^Ug(fQ^B?5&ZXX4&bQ^JVaI|-g>e@6*3fFETY7E=A#GQ-tKQ6Yy#Qs0`|xgR=pH| z^r*w9c|i9S`fDYGG|_sZWTv6xqJ%=^Gx_f7BQ_ zfub(S!UM7|040M!4CIodfnq_4>Ah}BVvaam9>N2p0K%A3h?p5#`Z&EV>Ia3d07 zDX$TirtQ8~{y{|DoJI1S*(eVnBxItGw3ikhpP`zYz8&d`Gfvtxj6Y@VnDB)RqmY#3 zq-Bcgz0KzqJk+$A_Adp^&4Qb@)9S@CQQVndjdjGGmtO&H>I@w}yJQ+CEsDvKbLJES z=G4A0W-2XPZULK=Dg?y^JDJduAp;i}Bf&`e`lOhf$UYOtEL1yYYjsI2wq8O~QX>@% z8;(D^PFyeof>cP07y=s@ziul-@obfNY9p8({+_L#k4Fp&wOd^SMIxq;#d41^`gO4=T@1~t;W$WD{* zAOqS?c0C4E0ajHUL#2R3Tqp3QK&6ntoPU2Bc7TaFd$K&wkv96;;^I!CFCaZ7=#r!r zn0IfdR%^n=Q~5qfTh-UEF(BPuedqsQr#)zu9ot+iOGFtCtqxJ5fl!zb;I!PbZ?AU~ zp!8bP3YLZ7NF=x<32X|<(y1q9Vp@vM{8}LU_c%g}-iT(@vE<*+^CJJ^Epp#g0d|Xb zaLr6eKV&tpi*nWlOAt6bKvSx@Bf%<3d_wk1Zyb>O6$}LtCI4PHA?dcR)!yAEH3O7u zHls-nmIXIq7-!0OG5zF=gQ1Wdot!uvTK>e>Nvq8^?HvsLjm1DfHW=ft3Sjq>pKzex zi8kJLl~^c&jAEZ&!!eQTKmT1vLs3yyHWH4YG%s&ifX57vcvt9;h70KS4gS8KXC5~@ zVKbSGbQw|bIl#}lQGLEugU@-J0P5K4@4wABj}CK!{Y%=8CR9(@acm$@WEY}?J{IRs zVo0Fob2Kq=H=FDwx0gtH)s7mE5h?-S_rtYlqko)>k)(9MP`(wn%1|y?(QjyU8@CRN z)M9V3Or%u-crqKZP@_xM8i)SBFM-;Z0mPHLS*N)VwhKK)8;=6`CYkb-4+&)G5v@M_ z^7NC}_Ai2Rna@GLPn|vr097K=g$Zx}WM~J;VgBrU!dFrJyuQ^gq0KXy(Sx40lR! ze!1myRy@6Qbxq*JBP>!oOlI!ho#JF!9CS@fLEuR|e;)1sC|pv0{`^V6yHi}u*_eCg zi>qqjjYF+l2ewAdwO$6B4*gQ<3isJI{MO9sAp3nxEhF*ihz-Oo{GTTto37a|0Cu0@ z8$}GjeGNTKfaOXU$2wBO^L~|fi^lWS8v5Aa->p+pG1We1UwC+zjyqchjNKO&X1iiV z(Cpp|tEnKC(K+3wcJRm5LrL0CUHm-P$!XcLBWKSBb7L!)ww<&7Q(@o0!ZI?~a=ap9 z$a!gm$bb0IqB^Kz-XQW%yPd;w{u=S)G7jfisBw}Vw0lHM958U8@z9Qo_!{h^D=H2T zOxZ?Lk!`=Kw9kU#*dq*{>2hp)v#iTiwYF+LD{ofS_Iz-4#tfa1wheYEec7X-x-)lQ zWMtB9*R)667r^Rnfh)~M8QTx!aofq81XWA6)WIOis^f-#Y<_;b+3o`g;8~7Hs>C%OlY;cl>Z7Gj$Ts`~Iy7%I@kq-f1!gFHq05Uf zt4yc}EqIk(x#FpN+H!-4dxeF4s@9&i#;s}V70p=3pj!)Z6U4YF|COHi=!|bqtb2Qr z;_r1cxL7e{>7ek|(+4jtu@3-t+y3NT?u!}~<%V$=>-zawZr;50>f%;AbBn=J=ze&? zg0H|VppruMezylF?xA(hlA?@Gt-saO=F^Ff?0II|$&Ff(dGqvSdZc+l;(4s27!(t7 zqtB4@?@tU@qQCH&o58X8_#+7kDrt{)+*+9R)I}JQ8jL>ubbY$^(UeEdtYG0ojvaHnR^Iaqe-M|y z_3F5T2Q|;MA;%3vD<6p+jvo@vk9WrW^?{UP#lxjZGt%@@F8}k7zQDPi*1fM&US9s` z%a^lG_jR08D)%l9Iqy6@bNrRe89Tgtwf1biS<$P$)iI@6OLd0253y4TTse|;+Xq#T zc>Ymnd7}2)(%w0FQ@6-R?)0T1jP`2Upf%$Onr;-LEo!vsT2{wG2yaij##UUxL zD{DODdXN>-ys`lAF?PzvjU$k(mSWA8Y?38VsSKINXj*U( zTk6MBw}J^?SvJlWYiDw`@4x>n)LD52CwHK=%^-n+rdpX4_}OpX;3qeXfA}`#MaRsi zr&DdV`T70WNFXXY`|6dW3>~M2X7n18p$I127L=5XgO8>BEC)Y+&H<9EyM1wp#lTl? zVPFRzY5yIduYa|yOam~!?tD6;0B9my^v(4<%)HuCMJ2_6kFMCrVn!p&wPc3vnly1@ zT_QWr8%VscWR<%+^=0z&FsI{|Ukh=YlLo=8tSsff{;J_w^cy-9&*=IJoI6D#GSjcB zsw%7e!}OHJDk^~kiBv{9OBtiUwajh#<$A@%{5`4Z=?7>{!$aZRx%203)f#TN)M;n# z78|u0)jGpRjF`lz_0(a`M`ahDcJ)tQ%=I=-cJ;qPT?8t3hzaKhuh);0JpJyWn)t!6yyt1BNu68eXGa#Pn zcW8cMM0Vyv*F8Grtr=Z~T6<1X!7DwCM2<}EK0BZ;|6jahFS~6}xb{BtAXBY2D~H9$ z#}mrtWE{t8 Date: Mon, 4 Dec 2023 10:02:50 +0000 Subject: [PATCH 14/60] Added contradiction data to output + black formatting --- .../testing/causal_surrogate_assisted.py | 20 +++++++++-- causal_testing/testing/estimators.py | 11 +++--- .../testing/surrogate_search_algorithms.py | 34 +++++++++++-------- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 89ab579d..94bffd78 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -60,13 +60,18 @@ def execute( for _i in range(max_executions): surrogate_models = self.generate_surrogates(self.specification, data_collector) fitness_functions = self.search_algorithm.generate_fitness_functions(surrogate_models) - candidate_test_case = self.search_algorithm.search(fitness_functions, self.specification) + candidate_test_case, _fitness, surrogate = self.search_algorithm.search( + fitness_functions, self.specification + ) self.simulator.startup() test_result = self.simulator.run_with_config(candidate_test_case) self.simulator.shutdown() if test_result.fault: + print( + f"Fault found between {surrogate.treatment} causing {surrogate.outcome}. Contradiction with expected {surrogate.expected_relationship}." + ) return test_result else: if custom_data_aggregator is not None: @@ -90,7 +95,16 @@ def generate_surrogates( minimal_adjustment_set = specification.causal_dag.identification(base_test_case, specification.scenario) - surrogate = PolynomialRegressionEstimator(u, 0, 0, minimal_adjustment_set, v, 4, df=data_collector.data) - surrogate_models.append((surrogate, edge_metadata["expected"])) + surrogate = PolynomialRegressionEstimator( + u, + 0, + 0, + minimal_adjustment_set, + v, + 4, + df=data_collector.data, + expected_relationship=edge_metadata["expected"], + ) + surrogate_models.append(surrogate, edge_metadata["expected"]) return surrogate_models diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index 22e92020..df7c1a3d 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -457,11 +457,14 @@ def __init__( effect_modifiers: dict[Variable:Any] = None, formula: str = None, alpha: float = 0.05, + expected_relationship=None, ): super().__init__( treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers, formula, alpha ) + self.expected_relationship = expected_relationship + if effect_modifiers is None: effect_modifiers = [] @@ -471,7 +474,7 @@ def __init__( def estimate_ate_calculated(self, adjustment_config: dict = None) -> tuple[float, list[float]]: model = self._run_linear_regression() - + x = {"Intercept": 1, self.treatment: self.treatment_value} for k, v in adjustment_config.items(): x[k] = v @@ -484,11 +487,7 @@ def estimate_ate_calculated(self, adjustment_config: dict = None) -> tuple[float x[self.treatment] = self.control_value control = model.predict(x).iloc[0] - return(treatment - control) - - - - + return treatment - control class InstrumentalVariableEstimator(Estimator): diff --git a/causal_testing/testing/surrogate_search_algorithms.py b/causal_testing/testing/surrogate_search_algorithms.py index 038a089a..ce543bfd 100644 --- a/causal_testing/testing/surrogate_search_algorithms.py +++ b/causal_testing/testing/surrogate_search_algorithms.py @@ -33,8 +33,8 @@ def generate_fitness_functions( ) -> list[SearchFitnessFunction]: fitness_functions = [] - for surrogate, expected in surrogate_models: - contradiction_function = self.contradiction_functions[expected] + for surrogate in surrogate_models: + contradiction_function = self.contradiction_functions[surrogate.expected_relationship] def fitness_function(_ga, solution, idx): surrogate.control_value = solution[0] - self.delta @@ -100,15 +100,17 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: solution_dict[fitness_function.surrogate_model.treatment] = solution[0] for idx, adj in enumerate(fitness_function.surrogate_model.adjustment_set): solution_dict[adj] = solution[idx + 1] - solutions.append((solution_dict, fitness)) + solutions.append((solution_dict, fitness, fitness_function.surrogate_model)) - return max(solutions, key=itemgetter(1))[0] # TODO This can be done better with fitness normalisation between edges + return max( + solutions, key=itemgetter(1) + ) # TODO This can be done better with fitness normalisation between edges pool_vals = [] -def build_fitness_func(surrogate, expected, delta): +def build_fitness_func(surrogate, delta): def diff_evo_fitness_function(_ga, solution, _idx): surrogate.control_value = solution[0] - delta surrogate.treatment_value = solution[0] + delta @@ -119,13 +121,13 @@ def diff_evo_fitness_function(_ga, solution, _idx): ate = surrogate.estimate_ate_calculated(adjustment_dict) - return contradiction_functions[expected](ate) - + return contradiction_functions[surrogate.expected_relationship](ate) + return diff_evo_fitness_function def threaded_search_function(idx): - surrogate, expected, delta, constraints, config = pool_vals[idx] + surrogate, delta, constraints, config = pool_vals[idx] var_space = dict() var_space[surrogate.treatment] = dict() @@ -148,7 +150,7 @@ def threaded_search_function(idx): ga = GA( num_generations=200, num_parents_mating=4, - fitness_func=build_fitness_func(surrogate, expected, delta), + fitness_func=build_fitness_func(surrogate, delta), sol_per_pop=10, num_genes=1 + len(surrogate.adjustment_set), gene_space=gene_space, @@ -162,7 +164,6 @@ def threaded_search_function(idx): ) setattr(ga, k, v) - with warnings.catch_warnings(): warnings.simplefilter("ignore") ga.run() @@ -173,18 +174,17 @@ def threaded_search_function(idx): for idx, adj in enumerate(surrogate.adjustment_set): solution_dict[adj] = solution[idx + 1] - return (solution_dict, fitness) + return (solution_dict, fitness, surrogate) class MultiProcessGeneticSearchAlgorithm(SearchAlgorithm): - def __init__(self, delta=0.05, config: dict = None, processes: int = 1) -> None: self.delta = delta self.config = config self.processes = processes def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: - return [SearchFitnessFunction(expected, model) for model, expected in surrogate_models] + return [SearchFitnessFunction(None, model) for model in surrogate_models] def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: global pool_vals @@ -200,9 +200,13 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: while len(pool_vals) < num and len(all_fitness_function) > 0: fitness_function = all_fitness_function.pop() - pool_vals.append((fitness_function.surrogate_model, fitness_function.fitness_function, self.delta, specification.scenario.constraints, self.config)) + pool_vals.append( + (fitness_function.surrogate_model, self.delta, specification.scenario.constraints, self.config) + ) with mp.Pool(processes=num) as pool: solutions.extend(pool.map(threaded_search_function, range(len(pool_vals)))) - return min(solutions, key=itemgetter(1))[0] # TODO This can be done better with fitness normalisation between edges \ No newline at end of file + return min( + solutions, key=itemgetter(1) + ) # TODO This can be done better with fitness normalisation between edges From 7c5e84b11d2468754014b0959101c2f5cd0faea2 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 6 Dec 2023 09:12:21 +0000 Subject: [PATCH 15/60] Additional return info + fixed multithreading --- .../testing/causal_surrogate_assisted.py | 18 ++++++++++-------- .../testing/surrogate_search_algorithms.py | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/testing/causal_surrogate_assisted.py index 94bffd78..26aa1d00 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/testing/causal_surrogate_assisted.py @@ -57,7 +57,7 @@ def execute( ): data_collector.collect_data() - for _i in range(max_executions): + for i in range(max_executions): surrogate_models = self.generate_surrogates(self.specification, data_collector) fitness_functions = self.search_algorithm.generate_fitness_functions(surrogate_models) candidate_test_case, _fitness, surrogate = self.search_algorithm.search( @@ -68,18 +68,20 @@ def execute( test_result = self.simulator.run_with_config(candidate_test_case) self.simulator.shutdown() + if custom_data_aggregator is not None: + data_collector.data = custom_data_aggregator(data_collector.data, test_result.data) + else: + data_collector.data = data_collector.data.append(test_result.data, ignore_index=True) + if test_result.fault: print( f"Fault found between {surrogate.treatment} causing {surrogate.outcome}. Contradiction with expected {surrogate.expected_relationship}." ) - return test_result - else: - if custom_data_aggregator is not None: - data_collector.data = custom_data_aggregator(data_collector.data, test_result.data) - else: - data_collector.data = data_collector.data.append(test_result.data, ignore_index=True) + return test_result, i + 1, data_collector.data + print("No fault found") + return "No fault found", i + 1, data_collector.data def generate_surrogates( self, specification: CausalSpecification, data_collector: ObservationalDataCollector @@ -105,6 +107,6 @@ def generate_surrogates( df=data_collector.data, expected_relationship=edge_metadata["expected"], ) - surrogate_models.append(surrogate, edge_metadata["expected"]) + surrogate_models.append(surrogate) return surrogate_models diff --git a/causal_testing/testing/surrogate_search_algorithms.py b/causal_testing/testing/surrogate_search_algorithms.py index ce543bfd..6e84ea1e 100644 --- a/causal_testing/testing/surrogate_search_algorithms.py +++ b/causal_testing/testing/surrogate_search_algorithms.py @@ -76,6 +76,11 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: for adj in fitness_function.surrogate_model.adjustment_set: gene_space.append(var_space[adj]) + gene_types = [] + gene_types.append(specification.scenario.variables.get(fitness_function.surrogate_model.treatment).datatype) + for adj in fitness_function.surrogate_model.adjustment_set: + gene_types.append(specification.scenario.variables.get(adj).datatype) + ga = GA( num_generations=200, num_parents_mating=4, @@ -83,6 +88,7 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: sol_per_pop=10, num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), gene_space=gene_space, + gene_type=gene_types, ) if self.config is not None: @@ -127,14 +133,14 @@ def diff_evo_fitness_function(_ga, solution, _idx): def threaded_search_function(idx): - surrogate, delta, constraints, config = pool_vals[idx] + surrogate, delta, scenario, config = pool_vals[idx] var_space = dict() var_space[surrogate.treatment] = dict() for adj in surrogate.adjustment_set: var_space[adj] = dict() - for relationship in list(constraints): + for relationship in list(scenario.constraints): rel_split = str(relationship).split(" ") if rel_split[1] == ">=": @@ -147,6 +153,11 @@ def threaded_search_function(idx): for adj in surrogate.adjustment_set: gene_space.append(var_space[adj]) + gene_types = [] + gene_types.append(scenario.variables.get(surrogate.treatment).datatype) + for adj in surrogate.adjustment_set: + gene_types.append(scenario.variables.get(adj).datatype) + ga = GA( num_generations=200, num_parents_mating=4, @@ -154,6 +165,7 @@ def threaded_search_function(idx): sol_per_pop=10, num_genes=1 + len(surrogate.adjustment_set), gene_space=gene_space, + gene_type=gene_types, ) if config is not None: @@ -201,7 +213,7 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: while len(pool_vals) < num and len(all_fitness_function) > 0: fitness_function = all_fitness_function.pop() pool_vals.append( - (fitness_function.surrogate_model, self.delta, specification.scenario.constraints, self.config) + (fitness_function.surrogate_model, self.delta, specification.scenario, self.config) ) with mp.Pool(processes=num) as pool: From 727fb6d69f130e9bed67b8127add6a5a335fc6b5 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 6 Dec 2023 09:12:46 +0000 Subject: [PATCH 16/60] updated aps case study for multithreading --- .../apsdigitaltwin/apsdigitaltwin.py | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py index 1877223c..e3505d33 100644 --- a/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py +++ b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py @@ -5,20 +5,24 @@ from causal_testing.specification.variable import Input, Output from causal_testing.testing.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, SimulationResult, Simulator from causal_testing.testing.surrogate_search_algorithms import GeneticSearchAlgorithm -from examples.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label +from examples.surrogate_assisted.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label import pandas as pd import numpy as np +import os +import multiprocessing as mp +import random from dotenv import load_dotenv class APSDigitalTwinSimulator(Simulator): - def __init__(self, constants, profile_path) -> None: + def __init__(self, constants, profile_path, output_file = "./openaps_temp") -> None: super().__init__() self.constants = constants self.profile_path = profile_path + self.output_file = output_file def run_with_config(self, configuration) -> SimulationResult: min_bg = 200 @@ -33,7 +37,7 @@ def run_with_config(self, configuration) -> SimulationResult: model_openaps = Model([configuration["start_cob"], 0, 0, configuration["start_bg"], configuration["start_iob"]], self.constants) for t in range(1, 121): if t % 5 == 1: - rate = open_aps.run(model_openaps.history, output_file=f"./openaps_temp", faulty=True) + rate = open_aps.run(model_openaps.history, output_file=self.output_file, faulty=True) if rate == -1: violation = True open_aps_output += rate @@ -62,8 +66,9 @@ def run_with_config(self, configuration) -> SimulationResult: return SimulationResult(data, violation) -if __name__ == "__main__": - load_dotenv() +def main(file): + random.seed(123) + np.random.seed(123) search_bias = Input("search_bias", float, hidden=True) @@ -106,13 +111,38 @@ def run_with_config(self, configuration) -> SimulationResult: ga_search = GeneticSearchAlgorithm(config=ga_config) constants = [] - with open("constants.txt", "r") as const_file: + const_file_name = file.replace("datasets", "constants").replace("_np_random_random_faulty_scenarios", ".txt") + with open(const_file_name, "r") as const_file: constants = const_file.read().replace("[", "").replace("]", "").split(",") constants = [np.float64(const) for const in constants] constants[7] = int(constants[7]) - simulator = APSDigitalTwinSimulator(constants, "./util/profile.json") - data_collector = ObservationalDataCollector(scenario, pd.read_csv("./data.csv")) + simulator = APSDigitalTwinSimulator(constants, "./util/profile.json", f"./{file}_openaps_temp") + data_collector = ObservationalDataCollector(scenario, pd.read_csv(f"./{file}.csv")) test_case = CausalSurrogateAssistedTestCase(specification, ga_search, simulator) - print(test_case.execute(data_collector)) \ No newline at end of file + res, iter, df = test_case.execute(data_collector) + with open(f"./outputs/{file.replace('./datasets/', '')}.txt", "w") as out: + out.write(str(res) + "\n" + str(iter)) + df.to_csv(f"./outputs/{file.replace('./datasets/', '')}_full.csv") + + print(f"finished {file}") + +if __name__ == "__main__": + load_dotenv() + + all_traces = os.listdir("./datasets") + while len(all_traces) > 0: + num = 1 + if num > len(all_traces): + num = len(all_traces) + + with mp.Pool(processes=num) as pool: + pool_vals = [] + while len(pool_vals) < num and len(all_traces) > 0: + data_trace = all_traces.pop() + if data_trace.endswith(".csv"): + if len(pd.read_csv(os.path.join("./datasets", data_trace))) >= 300: + pool_vals.append(f"./datasets/{data_trace[:-4]}") + + pool.map(main, pool_vals) \ No newline at end of file From 8b0987e037546be5db7a0a545aa27283e165692b Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Tue, 12 Dec 2023 09:46:57 +0000 Subject: [PATCH 17/60] surrogate assisted code moved to seperate package + updating example code --- .../causal_surrogate_assisted.py | 6 ++++-- .../surrogate_search_algorithms.py | 2 +- .../apsdigitaltwin/.gitignore | 4 ++-- .../apsdigitaltwin/apsdigitaltwin.py | 12 ++++++------ .../surrogate_assisted/apsdigitaltwin/dag.dot | 5 +++++ .../surrogate_assisted/apsdigitaltwin/dag.png | Bin 37359 -> 55421 bytes 6 files changed, 18 insertions(+), 11 deletions(-) rename causal_testing/{testing => surrogate}/causal_surrogate_assisted.py (93%) rename causal_testing/{testing => surrogate}/surrogate_search_algorithms.py (98%) diff --git a/causal_testing/testing/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py similarity index 93% rename from causal_testing/testing/causal_surrogate_assisted.py rename to causal_testing/surrogate/causal_surrogate_assisted.py index 26aa1d00..9ddbc088 100644 --- a/causal_testing/testing/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -11,6 +11,7 @@ class SimulationResult: data: dict fault: bool + relationship: str @dataclass @@ -42,11 +43,11 @@ class CausalSurrogateAssistedTestCase: def __init__( self, specification: CausalSpecification, - search_alogrithm: SearchAlgorithm, + search_algorithm: SearchAlgorithm, simulator: Simulator, ): self.specification = specification - self.search_algorithm = search_alogrithm + self.search_algorithm = search_algorithm self.simulator = simulator def execute( @@ -77,6 +78,7 @@ def execute( print( f"Fault found between {surrogate.treatment} causing {surrogate.outcome}. Contradiction with expected {surrogate.expected_relationship}." ) + test_result.relationship = f"{surrogate.treatment} -> {surrogate.outcome} expected {surrogate.expected_relationship}" return test_result, i + 1, data_collector.data diff --git a/causal_testing/testing/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py similarity index 98% rename from causal_testing/testing/surrogate_search_algorithms.py rename to causal_testing/surrogate/surrogate_search_algorithms.py index 6e84ea1e..acaa2913 100644 --- a/causal_testing/testing/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,6 +1,6 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator -from causal_testing.testing.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction +from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction from pygad import GA from operator import itemgetter diff --git a/examples/surrogate_assisted/apsdigitaltwin/.gitignore b/examples/surrogate_assisted/apsdigitaltwin/.gitignore index df652f9c..76313cd3 100644 --- a/examples/surrogate_assisted/apsdigitaltwin/.gitignore +++ b/examples/surrogate_assisted/apsdigitaltwin/.gitignore @@ -1,3 +1,3 @@ -data.csv -constants.txt +*.csv +*.txt openaps_temp \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py index e3505d33..881373e5 100644 --- a/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py +++ b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py @@ -3,8 +3,8 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input, Output -from causal_testing.testing.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, SimulationResult, Simulator -from causal_testing.testing.surrogate_search_algorithms import GeneticSearchAlgorithm +from causal_testing.surrogate.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, SimulationResult, Simulator +from causal_testing.surrogate.surrogate_search_algorithms import GeneticSearchAlgorithm from examples.surrogate_assisted.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label import pandas as pd @@ -37,9 +37,7 @@ def run_with_config(self, configuration) -> SimulationResult: model_openaps = Model([configuration["start_cob"], 0, 0, configuration["start_bg"], configuration["start_iob"]], self.constants) for t in range(1, 121): if t % 5 == 1: - rate = open_aps.run(model_openaps.history, output_file=self.output_file, faulty=True) - if rate == -1: - violation = True + rate = open_aps.run(model_openaps.history, output_file=self.output_file) open_aps_output += rate for j in range(5): model_openaps.add_intervention(t + j, i_label, rate / 5.0) @@ -64,7 +62,9 @@ def run_with_config(self, configuration) -> SimulationResult: "open_aps_output": open_aps_output, } - return SimulationResult(data, violation) + violation = max_bg > 200 or min_bg < 50 + + return SimulationResult(data, violation, None) def main(file): random.seed(123) diff --git a/examples/surrogate_assisted/apsdigitaltwin/dag.dot b/examples/surrogate_assisted/apsdigitaltwin/dag.dot index ca54c473..660a504a 100644 --- a/examples/surrogate_assisted/apsdigitaltwin/dag.dot +++ b/examples/surrogate_assisted/apsdigitaltwin/dag.dot @@ -9,9 +9,14 @@ digraph APS_DAG { "start_cob" -> "hyper"; "start_iob" -> "hyper"; + "start_bg" -> "hypo"; + "start_cob" -> "hypo"; + "start_iob" -> "hypo"; + "start_bg" -> "open_aps_output" [included=1, expected=positive]; "start_cob" -> "open_aps_output" [included=1, expected=positive]; "start_iob" -> "open_aps_output" [included=1, expected=negative]; "open_aps_output" -> "hyper"; + "open_aps_output" -> "hypo"; } \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/dag.png b/examples/surrogate_assisted/apsdigitaltwin/dag.png index af043f9f64ec314624a09778bcf5576ad78d9359..2c6525f4e6aa9418cfa0121037f8f62f22c81cac 100644 GIT binary patch literal 55421 zcmZ_0cRZJE{|Br|AtD(SQB<-eB_lhdWJkydA!KD$Dm$dI8Zxu@s;o#tlueV&jEpE5 zh39>A-M`oK{PSG*>%Q+x@aa>^w42#BQ&3RQs+>^JqM%sEOF^+l zjA|o(g{pg70{(BKxtg*9#VYwvLSN-$e|Xv09N4g7L-^m=fPjrWJUop* z>%w+)JW+{Y%h06K6jwRD6(6^r!qe09{Q2{D9z9ZXa1g)`jf2_Qe8Y>CdHNgfD)+qK zwr9_tLtt=LZYIt z@Wa)s%WPSiR5KS3o$nUJbtRcKHZ<(!=Jp>NI^W*W@us1nZ>p!9eDU48Gu3t}DJgW> z^4i*r_%`*EC!>T+&!2xH+1=e;S6xlP&(BXoOB*%3L(JB%prF8cRVXzzH9Whrq9V4U zLUP#C%#4$po4fHRuDfsklJ1!^CuC*U96fq;8xxZze$dm~uBfPpAMQSWeA2{({lS9= zI{4@FslmE1VR7+#T&<|6C@wv}px|X)-9~S3?}^Dte9_Ccwyjc9mHU=f{OMbU?)_W# zq_{V4-f#*C++JAlXm4*napJ^oF)`gz4R!UsVqztqZd67`Z^u7fhD?0?{81zGFHD$} zG7a~$-NVIk$LjWoh>AwlkByBzdGh4W{re}hw6sd5TR(hYAzwN%kzZ2sx~=W*w|9>^ z3N6(POKkmfa&p+_)q^%F-@JvVuNjlB!;8NY6c)xUAdvg%Y8U^$eaZ$V8X9!RjvdR+ z$tfr)(Veotcrnu3&d%=b+qa$1_2={r4cTzfwe|H8k&y~oS`0HYGmUn_!ou6OZ(m!r zdRE6jI-38ip7+G)Xb=|m;k;6im*Rh4_gPg{_4(C5ho3q040Pw5{Zio>asNJxLvO{K z$H&|wixwU}eE8whr}k_E(T8Go0m;c0CAbgo(9qD%3UMxBVa-MNiq#jqI^*Nx0}~UW zAtBVcxw+5F$_54oZdO)SKFnfHYQC4iMYg!#?5wl7x%u;J%MzNJnqQtf$#3G>v}u!$ zp59th8ykbkiSmdD#&?gz>+wu*tHs6VdmkcR1@ti9aa zYpTLKGB)!&I5<3i{=9E$>h_HrHy$2jmYdw0PL4|pm$-OLdiws{b5*CX&1M&HYmI%b z%F4=y&m3~r-}8112new4U6;58TSfK4COW!(l9I7OK~&BzE^VEiwI4q|aOf#(tMqm+ zbNj9oxMkPg;#{!QB`?C+l9R++-Uc2aPYQ8M7 zdw;}drGmkPi;HW&pkRGnUBHX0f7+{je3s_EE90U%jdST+BK}>sU~Wn{QWg`bo{WNa|}zg>gaT6Z(eqGzIU9fGczIKdI+;1!_J*=X9nw!UR^r0 zxVZS?Y9bfg+ zC1KNQijA~veyBWUYD@zvCrBY{`+*}(&&sND;llpJ#KhR6E&*yW0-6Q}!5J9`nwy(F zJUsS4`#t%&BVBWkTMU1#i>8T5bJRh7DjJ%-LPB>s(loS=Zc&K}&)-@pwcBt}e*S%acoKZM`F4T-J&`U`bJEBxO`_VKX)LQLkV7|NNe` znC`9I%fn-CVeu;8>=ldjl_Mr5CZpfKXO*jSPVFNlTVeTLM8qq9YUU%}zxhU5;z#~2 z{zOGcNJ&w|b{cI<4ipi+R~$*D`B*h&0MBCL^RsX%@86gDYc{yLx{kij;~bQSQ8{_i^6QJM ze9P zadAs^m3Zl^7kVp7%bJ*yWDguT(Ep?V)y2lWH5zF7;z>?sZFxhFnOxJ%bJo zce)1ZF()Tytfc2X#(k$QVV~a$330k~DFGkK*ZA024YvY6FCX9emN==u zvrU3Kk6fa-{?eCH^x9t!rQ16XSQskU-y!89>G#mk(50ncHmBkw-@J*~S9+;`?eOq$ zv2B;?CZ4NHzY-NgwvP+wQq8Om4-Mtpy%)H)ILy3u-MY56HnX?)h1KJwA4^|dO29su zVo-dw_umtFr=p=T(wU*HDugFQbX?Z`};hy)xPxfVEO29>{HyQd(WIXla8ZN zD_VwQmZhI()scDYRwajIS6)$V^E*m01bpL_3KKRl*OLk(Ny`pFU@J!-DhEC%_!C4R5*#fc>a5L&ieK1FD=em;$YW%PgmAs+dVmZ-(&l~<(GVb z(#3s0_K4HXn>W{zrVQa?#0H2K6t>oZX-?g=g&JF9UWQ!3ize-aBQA(W!kPYW=@cha!@ckr7;8ehdf1!_%|v znd7;m^1OZv+D=YRd}p7A;((gl+rPuSEo1}r_I$VRdL~|D+|dHY zs_%1+yNsjLACXI`4dD_Ki(39OU*z~z`u5JFa==i#BwS5DD#{HQl~_dt5-Rlb@_jQleIdj!}h&rj6JpT3SVA<-nEYKcW{u z2nEpYQ2P1pU6*kx{lJE&7WG@Cmqzz6Ffu-n@sXacT9qy&691d|g+)aK>;Wu%`C>FX zJKNCG;xj%eX4AeND0SG%-2CRaq`O)CT~LKP==tbfPYlRep74c{nLVE@&e^0Dw(L>egMm-dV?Att+2? z@uwy>mVrD8jf^Kx{OsQ6;`dIftJ46rksXPf#-mM7Pd^_R z{QP0X#l^PYfgxYNc@qpwI50Hi2iRr*srco`kF4v~`LZo_uNWyRZm7S#%VMPY@!zHI zd1!R)z^Y2&?C-GTV{OTlVQgYrXRiXRer{}RTtIDG`1`YE45bS-vHr~)l^)CBholS6 zjNl%gqjc-%nQ#MO{rGXo;mdPbPtT)(l}C;o(K*)V__2_C%a$#`nC1Y1`wRu~iILrp z{ruK*>*#LC{_x=po?csPYXZ*t^U~6O;L5D5gV=GYx>9%XfRcgLQn|4?9I^RaN)<;yWnzY8n?-3vnz8fVH} z54;MXwU`_4!lue7=hGA=C8rMX=!}t zm`)wG|5OWDZ_`_G6pgO-{rebzfh;Et6vUOrrl!H1XT9dAexy9$I3Et$ek~CYpf)54 z0I|CIIIh1g*SNf)xjEH)ehNiyt)t^~y~9>|x=h=w7a;74)7San7wgyn|S38xq^LM{8SvwbWo3;gCoyX z?BNk7^RO-SKAM(RR)uIj{X7c6 zzskCIW&%&IjXpz%_uBHD{)^QW@7wHBx5C0`u~96>J2RY9Gq$Isr4^xN>E)T6Jb!*~ zPr17_>cqn0qHg(x6u?kx(mu~RR=!40SorgGwHb}kQZIk$Bq}AX3N|}hw?$i$qEY$f zFkITe^mGf5z|dL)+b3EXM?7cuX6faa1IUmXFJVPRdy_dFy}gD&iU>iD%8jp8jjJIk@leAL_d666E7pzD0u2V9)@V(C`=wWUoi zE-u^o&Rnmms#>qY>F06#RJ?RpUY;Q8^u(u<5cKM-^1U-d0&f}&ILAwYPb(@bS@CJc zWv-k+4p@J+6B)@nckMEGapfooXASm3smlmCIXHHm#t)%wuvB`_e*>_%|L|cup!eme zFBwz!YPGQ=6r^;~(Qa;_zU1yc0%d^^0NNdgTY=TwVtCknQJh!*j>wcc4{t^p{59Sc z0^kr46O%U8j;$t}UsN<$8_XEGL;N<`pyAR>L5+>7ZZX_(*IkzxlGs&GoG|^nFk|%M z$^&k-=pWr@9aY{ZdN}}jVa;}EYinaS1Y~7p;fP=@lWo7;diG2lC8YcO?NX3 z7$pEY*4EW!V)e z1Iz|_^+C}H-F0lcjEqcowt;-I()end7!)Pk80}^6@xwkVSEN_}y7XTAV}C04NaB<; zp2< zo>Q?W&YWR^mN71%LdD6p9s-hHu2C>rnPb)R;k$S5{{0!h8d+7~7W&tYO0#|gz%%|D zN|7HBSy|K8l8HalRcT2{H?a>+^rV2V`HJGaKqjcCllfbXlYKxyK+j!efE~vp{S{Eq zBIwG4W9|+)M!L_?ur;I?{Vq>_LXisuP9|IpHv)o6vAW~F$?4P8fWVQmi=P$Xraqr%p@M-7o^2?K2(s_xAvN}53K!jfdg?LUTnDGWZ8mOJPK?vd{?V~%@J#&HUB zP@_T}&#PLIpxm;v4-^hX>dxcGIaBu(n4;*DnwO@1Roqq%*E)w@sk|W_oiw}g?OPobw5%6@rjF?X ze(M0OuB>aS0=45k-UX=NRbWxC1BEiP$hsvH8lx_BF`u%^`r)ame_u8?{q>w1Se~wl zN!&O#vUX`{X$G*LPJK9sesrbNU_c{|m!-9Jn#7fbd|p984Y#>*UC`i|OrR-VW8)s{ z%wd(t=;%y{dphT?En4Q-YvZ;LL4?rz`0-Qpc+@56uOq$IMyhib%Ubk)?`g9F?{{epf;m7X1n$eGd z=W$WM(0!AWVK;89Av^-DZ`-zQlJ6g`gg$+G7}y(NynFnzKmCB>z|_>K6DO$9ib0{3 zb4#=+fw_~mlwVu+$~l#B?0f?~RFj$c`4K!3Rrfr=8?eLq58nxhM_2Uja|8GQo&s9& z7#)F<^;^eHCKy-LfvsbkS6WtPZfDm726p1qsf~SoFMTO!*d?ETD#RM3fa{Byj4Xfo z@&$sOSB|4TR&_6c8kjEwZf)t;n7pfNahaM|B6O>yEluykWk%u2aifG^sS=+y5U#?aBqc>tOG^$~E;`b+(K*{r;Xl#Pd7$|!C@S8)bLTjKc+UIXAbM*7jBoEc7V!A- zE*ymZ#;Aki0%QK%9gNhPfvjw7(uI$7PLdYh{zQdb*fqZoQ9g(x@)WJ zQnIo_5)(Ngs1J>d$k6L$M(3Qw<2_{4K6Imdr<9Zw;GpoKL$!eGU^Kse{SrES7#hoI zPX}AuTLlF(>%_s$Ee#-7R*4D=_cz?#6Urub3-VWi&+0WvDXA{wZfYV5yPZKpFLfQ$ z%CT>$t@XoEYKN3U@ZRs=si?2eoxn|B*vcwuXoQ3H4toM5curT&-#={6X5Ae7^3$h> zDBC(`eKa-OwBP?upIrR&N2bkd_UBJDOVttqxBmWqXsZ1jP**S6^|8qiBH3`205u&iL zumQ|!-@ZW5D$4G!P)YzeIH(Tk%GSMoO9LbU?K)HHX?}ism!1r#UphY&QFN3T=#7Ju zlkWiee$38-&kJq?R_io_5@9;r7zGK<)ZX6s;OAIji;Z8e&J)R%8(+e4ZD9?F*!4B* zGWp+^x>%zC28D<3b)yN??<;9TQqinvc1;YI+phngQByYk~D<(ac* zBktXs?ff(Q1ZV975UJ1d_j3?F2SxMp5)zJY;>V^X1UGz|_7M5k^XD&p@8*Y1Q0hG| z0HhEZ6{UMNJ~j2*9$)n?3+!h23Iy=Cw==l7{Iu?+!|K>T1Gv~z78T66PvpwNrKcA^ z)-(xLWlt5Fu&`yOjrrn_Fu?)=rxJ4croOT=U-f?Bcdk)s4e4e;2f<-sDNciRxSs$N zPGRsp+n&6nq$H2QF!AReGh9|yR#$$tDT5m^UJnhW*)Cwz0C57G@g+nP08OIN+~ZcG z`0?Y1>Ei4#oxQ5hhrAS-PrdO5Wv&?_(sj+vw?WE8&_l2NnH1;cUd5%!W#@ zxy;|CqoWr95R{>wLB%6r+~@Bg$CahAG$@%bx~*0}fui8>4WoG-k&x(aJiL0s!6BEy zD6>YbYdM+S`uusFvw^6Oq%`3Mu;vHD(en!m3$@}tfA~O@hKGl2l2R*~2yRx>nywG{b4{>p)R8?=y9)W}aHccs)@ZyE^<;$104zsbb>5@}7H|N6N*t>VHPI=X{ zgAe24OiKFjsKOpUHZJ*N_3ohxbr_^F9sejXyY77Z&xfJ00#37ukRwG27X}u@18k{( zXW$uYer<9J&KE@Ob1SpV(tjpI-@bcS>NUrUWAY7;@9*!=j?n(QF3>`Fx9qL(+$nwK z{*85%-J?&(i{O5NC8CpD8@hVs%9WDi7N~OIK0vAXJG$NzP#Thx_ZS+AglbdG?8aez z`#{v7{4)RNXO7mjL0jpJVNk3+dGeiMZm#TI!2PQQ-t!Juu3m*M$oti2wbCzYb=v7g zAf8NQ)$*jHM#U6SG|_I*(23^-FbIKh=A(6dTcKrBTKeyqt3Tg~R{+AG;R<>Go?z?g z4mqBpbvHUX)h`;t!oc8QFlq$&&GY9+APYsYwxOsJZRF+48(6&lzP{_ezUu%O8(mPT z#ipa^<&5Fyh+4hv`?|U!P9!n#!mi#ti2|l=+xGh8#m9j=k4I$Yi$Kr5_IE~k>_s~T zSj#LfYG!dH{^#VguhK%Vi_)>t3_u*{85xs{>%RPv%&D+0yb~ne8hE>9%`GGOUhg{`Qk_DxI$(%>%p)- zmbo3oA=dMp}DOSSm)^Jmd=+z}x@PsLCDESvJ@gk>cqR%}U==ir$ zbhWfXdjaZs3<>mh4!Pzl2< zfQ`NuMzjJk@H$dyCqI=WqoXvsjGf{!&5RBDzbt@4Pce`+xbKhZ?>Ml8MO!vSXC!8N zX`n&R8H0-yS-sp_n+_i7 zIV~+MD^`}iU;J38I5s{f1QCz#+>2P?s7`Y?+M9U%Y{d3E?AZI3`P`M4AZ{Lu!!e#X zg)_6W_k{Uuv4M_)roCyASq%ki;<)l-&C1G3dz7+d1ToKx0quwis-Uo5+_CpbMFlqU zE{HmT(DVQrT}NBjkgF-az*)uU=b=Oj2lh4gA@5)dyOO-TFS-XKE9(JmCK*;D@F*ZB zaq;Vm$6zIK5&JZtb=K~1taurDKs*2GQ$2T2!O+;)o#bkP8HsE#H8oW+*L{}U;vt7G zXL^Q%H$rg@#rgn^q`Gp41O@HJdOC~cGt!eo1L0j;5>N11sfYF`ZH)%EF$NrPzN=&{pU#3fvxQATXijMZD?3S(C2p^ zyBrFE78nLnUm6hmmANi`NUm=lbf&rf{C08+hxA)uf}_quK_c$g3=CM)PCZU4EWAh5 z6euknpnHpPijN~6KYoWN56-b0>hva2^d}0aQGIZ}zLZ^_6fG?+#CvYWUP)=vo!sG)azSZ@A@x7;T3GllHYZ?H*7CjetzBIycr;e6o}9)z4-kQ}|5acqk`7R-Xm%dC0~JIRD*Z5x zD@X>4)_DlAz;^>E`tk7`pxW@FHQ#x2)pDqvIMD~V0O#f|`kvERJ2%n|6#37drJ*IG zgBp~^eBB}UR{$n6U^YDylfP<;T#8m=PR=`&)mvdYK^ThCH!*{Nlcg^5b^XW*`Dob|eq|IA?}Knm-YIASXT#V2NDsKf{uM_w8k8w-w(?&iq-4vGks zY}E{?L$Kd5Bh!clzKDJbJVPaS5pRhtpp^!?_^GBtKt>WFLTK zaBiR%m|wi84j((SM_pY#&h{82kcr;PM|0obyYyAB0sej<=_!K5KpISj7SBAtbWiYp zNP`A8o~1y`(3cu-7-0X8c4ja_-Ash6hw`q~!%eMeck9+IbRtuTwBYrm6+&c-EHH;7 zQ#~!kzH{eJa)E08t*E033dB zw$z;sUrSvmqf`~Kmu%XS{GDe?GtXkhVH(ypHfA=*o;<0hwg<)#)FH}T8g@oTd8AV) zfM-dH=i4_(p8wnyq6ZXm{7*3l+qwY&B`GQSV}9Oxpf*^CmB(O_xK4gzf@|d#!&mDPb^m??lp!4dZvcS`3JRw+HE#gjgTKS(WkQkv z^vqEM$_OG_8BvTrpxCvqUmxgr0L;M1&fW}7fW%!I8#ilcXn4&}oy;?-9Q{<1RVJ|i zAnAv@`T1$Rz5mG2@Dd8Y&>zTWP&ib2QB+Vc9QBf6*RJL2H9o-bg|VrrJdhj6MzppL zOiAHJEC|NC*@X*1s9UpV^!1q+md8KqBDB)h(UJW8`F(_FhUYR@N`Xm$9qu7-20s{^ zV`zBzE&_H4Vqw$o-M23yI$Ce?55i|YyNyb0yEY?O0m2*CcH?ZEI(QHn&9OuD15P9l z!@y93qWlpkM?cp{7T1er1npp~D~nmm>sLZ{HjTD+hXix8nyM-lEiEYv*w&2jxxi;| z1R?mapv7#YV-NNBSG12pQX*(c0L%tRr3i!pfh~4cJwgSf7l$GYGcv&Lu`fw3PF0l{ z{4>pQWCBWNvARfEz#7m{_Rzp|bMo<2VN~#02@z zKFxvX|*7RBpw`i7X)Dth(JN5fxm~yl{uctqeqVzj8MRo0V}%@ zZR$c+4yd>Zc;YS0%oA#A$vCjs$$f)^^2oq^pIc~#B}tt7Ejy1Syfq&i9`;9d2XV+9 z%jYKdI*wj^1qcPI{8rR56Yoi}73e)qtKr4SRDQkklNOy&S34y(xcG4?u{-HIj!6x3aM{X6j~rD7Ik)_#keTkB`sBEjvG$)@|J6l$W`Q zFSD)z1^YHY1;r<@)|{sM<>lpNWlw^bjj(x$ebZCs25!)YI2|q(p&gKRrJmugAc(`b zi)s+DqaHtY<2x&$vsNts{`?DW2+!dIxp%mzRv}8qLOqEYpHPz?fy78QqUbN`)mBwA4|-6{*^+kK*2aVCZ?VMxo?jgkrAz~uMb2FU~p_~v7rT} z6Gi47R7sY@7tiqP9ke~$b*LWCb^r!CvF2>r_x@d8c5-&sa`%+$%!Eb-EA1V=_sEq6 zA$ZJ*i2V_64z@KoJgkWFcA@=AFtTuS4S?ka`KD{mKE0R-$KcA!;t0(y37xWDX$3gf zL~xN={>8{|SWFsFi&voz5fC-O7Y^#@4k31sjEVq{1}T-w`tkWPBuEg(9s644!>_|T zev71TKqR_zjnyD65X}v$>B7=dV z7zhNE{PS-ZU`|c$GbX9V&XXxxPQOMqaWf=5gqSHgXX?ep(OZX;pXcQX5!EzX&ZOmD4bkJN^x-{Yz=?n$06ed1Lifn2Nvlo&kym3 zjHu!&@k9<^{7{V^qM4zCMAA+aoQjHyPUARwH&W#HX~dEEBMwN!ojcb{OOKLFS5p%; zLNs7fiBlqqwD^l}ABl&~=0hP306ZycvK7Lk_M9wv54MCtB6Y~I_pg&QfvcaQ zk!`G6U3Pr2vSbH%PSShw7VGBoz^d8C<NV6ROvP$i>)Vzq_kOv0a8>+V_3M6qB zVYWDLmExQtPQ#^3La63DC0v6@WGySJiI}KhMXX3gxNgm?Cy}L<a*YR*W8=ESw1EJ+QgjwBtJ(a(iFuSQbISCW$m3s7PQ4vvNkfj=Q zdNktikqAlA1)CRZ{jt(p9Owb-97}R)!^5v3!do6L@EZ8+ST(#n|J51@0X=2l!TV>3 zJR*>jBPqzsdv7Lno@g>iXMhVMaWkvTc!oGdJUYe+P6$S=r=&cne{t^T&j{+-U4m}bgA|fn8rW7~Gnyd~ zVFx@LTUxAv?nvK-khBf}q}Ov~7i&J>_+$7NTXsp*!ub4vL>1r|st?(fF!H*{me#|T zrley(4}MG@Rz-y*N)R!;kS%!vdL|O_!G!_r10Ubk-EGyGu8Eo*h2@m$oFm2E7DN#A?DfvF{T>ZOuVevjI5A@Gnom+XvolZ4#i1am-^*7icL@^=m zJhq#&qQy*2Lt~>kE&dA-%f!g@}yu4gE=f0G5h0y&F^T&aS~NIE9q(t(2qEpT)f{`?WW)OURG zBfTw2F(XI>Ckj_YhJwI0h&mA+t%a;O>J-7Zn5!`|QvYx4rc3U4SK;+p>A!-KIypN| zck2)Lw%k${6%tCneA>a>Jn8RLYkPZUw^S}biz>Au&=}U(Biml5q_Frx+u9vFb`Y-% zt=Q#D$av<>Mi^JbK)>YTGKl6K$}aT~(Hdt&#w|a>D<(;3$dL%kS(eNZv<95eKQfY5 z(OqDnjJcFx(0jO$udvmy!$s{s9fH(+6tD?JYQt^>?m!6<*C5UUFdQfJ2yD?u0P$P5 zZSzY@8(HTz4`4tX1XpAzAb6ptDk6^;2FQ3&e-^;#Zd}|oAC_oj(BhGxv;-U{L1p9+ zU~8O#@_`4Y?`|X4c>uI@;!9}+LP+#1EJ3geAWslzf-w_7lko%y>gYBHu#?5b^*v)_ zVw52@y?*_=aS$vLzk^RtNly<#X2>=?>n4G*;ocXLaU21#odZ^T_1580vtc9()`m#e zkIHHNlG7J1Trf2^*RsE=JLRa7o|KbgoO4hs3iT)%DH1d>bRv>*ha){WKHiAb2S$M~ z@}y+2(yJHZ-Q%4=S9OjuH9!AH!HbB%jLlaxxDFmv^E6_j&-$`sp>Mvb2uGW)r+c|% zWMo8JPjBOKI4j8AU7dJ%2t9Ol2W)rZ(!z6*Y`Z3vo4IZUL1ZEL5yNw6&-pkiwmFP} zbR1Tw_*jbwsPX%o;tx#5IzWGkBs*lJYj?2lr7fS>C}RqiF|uFzIkual-7zxp>2 zU|W~^n69>M`Sa&G)ZV&o(_gFNQAed$1;25^w8W@Q&$>Upz(?$pD71eU2E((BO78)D zuFS?%{lFu9vnGZQM(%XcWYj{x9$gn35$`H6I-nHP`xnb)x!x zb@g?U1Od~8um3N2Fi3tjHBE?(%9HTO+suY1p?W`k^RD^$l)NlR=!D24B1YnGc-mgS zC*Gj;ctAMK@{eY2Ls9uLGqd>0GV`VpjvO>>vfDApBnlTfWryy`bcsF#FbsN#h>1Hb+*ss;=;%>5DZ0XI=y zaut#cv2)4%*Xy*b@#cnx)8pwj(4HWx$|{D9x%(D#)g@%3PDgc|bsTZ9a?#!yRhwJ|Tu(Z}sNG zk3Kx+E&wTD5OiV$Ha?Gp#KVMy-GJEht#f)M13fM6?f0HM*$cG-GmF}%s3fy3^BVD7 zBNMbz4qLl+c67*ttpaRP_F`yY{c8E~LHDm$m66Fq2_7PR6}Ud4s7g+9ow2boM0$`< zZe$9uK}lHc$8gL!-j1Cbu}5Y~X|k`j!34Yma~KGPm6a8+?qEyDzd6X=d-lk0+Nrzl z{B#OtA&_E$+um3S4HS7Ax(cg{7qk7gz+g<-Am2%I^5jXNcSIkOAT|M$O4jjlZTsVPpXqgaqfg(x}O6GU;IWjNp6{ z@&Slg>0sm_r-p~8TPZmFq`rO($o4Ee7%~zA-S|nUsONoHa3s6{T_Ck$J5+F#K9nZa zXVtn1HuKCCepnFf9xwtuI7zw}e-O0sIkFv)TrB`_ z;9JS|q-<*I6SM6+aa1(+-n|+Ch%-f2w7`5&rG3#ig`o(LVNPS)Ft{205Oj!!7#aEG zi?yYtt~WoR5S+DP7i``-NN%McF_Q^gVB3O8&`c?2+IT2DsH>JB$B3E~;n)&V2!R9v zw&5=QbN2b~C7hj{`oU9=?uaCAQc&vDZ|K4j*g8TfPNE0 zlgca@2_`g)36c;*5E5XCpu{zkxK8UfLBDik-Hx4(h+w01D}}OXPNyC{g)s$bX_Hc2 z|J#{oA=yA>%K81^e@Y?%6?O=j6+p%T|2P*teVW@PJ(#qDolD#jaE!aqFj`++^(Ns7 zOhQA4IPLCfi=$avePdlI`qR3OH8NR9H0V8=Vm5twa|54Y@#u}O7O;Q3O}jCClV!$B20oFhBn)#gMrCm`VfiD&YsKMkFuUwHtF(wO3 zC%r%+D43_*O)L4E#;e7=;SMJY3rjvA%Ac?I4615F`iV|A-{X$wWV+U6Fc{~bpY z6MLp0r?8B5dWLn1KoRw_&&&kA2U6gL_7B7P9r$3~>(~AOJwlk}`12Jwi7hvC%`9m1 zH{=lir|$8n-)~L$GqJht&i|iICo?~_8Dtkq=cPZt?Qu&G(bBi8PgM%|Vde&v@sM3N z4J9QdGRK&4BxaoF3Ku)Yt+E_!Ic)U;+b&-C$-#(G;-}YNzj8qN#%*Lp3!ZBx>K6F^ zKY{J&Eu1Ed08HzYe|xjcrghpA?YZ>wBpW1lx<8KvWEBrd0?849!Dr=XJcg(T2M1rR zE`MEZ!>k1um7j;|^ZvHh|H}dZz~MdvQR0X~bn)UvGDZhZ6MprMrlFxKR5;99(0#$; zBpMkTDJxT=7+!1hMBiY$*(|LSA}+Y*W{#&dPBe zLqnp912JHS$lLd*z0WpyMMl|>lfnicB5E1LM;{Na)Wr2s5ST=f21P->x;m$w3GPp; zjI%@Z7m|^KY>oWH4CW7|T`xsr?4rzRa3c~A=FnA85sBaenB#{eKkki=7GvGQ2&41r zR~<%TSFU0C8A%dlWqFuEZYO*WU;Oc@{SG2D*jIR=jSKUIK@kxV z?PUgC?H>^~piH&L!%T&UN=gjct3JP2oNrA%z#3eT3KOoLiE0<^(o)S!Y6{5$tVvZ zO+@uY!DraE4R~GhN7apZ1U9gjCT`G^DJJAv*q_T4(GSeC|7&MRR2eZ;GXjlS7j$Ec_pzuml*PPO^ww}5h;)C&*JT0V?6TwF}O z_Q|NRu+T_VTlFL_laTf{P1#7rZEMzSP`G=Q zTR;z-;HP$nZo5v_m>T!AF!%l(!Nj0f7MMx=@Z4z+K0S&*4Br|CQ;0mUppcMP$X8z) zY2F20IUzH1^Uf`lu-fqMi5qBOsAmsacJ+*;KY6kq4ss3hY<^G)s+*g);4w>}@NYy~ z_~^)O|CAAzmA%X7>`eE`3=z@s(Jlsp+Py$dY+x(@GE2gw`9DH2ZyK_cFK zP%oQes~J%B0wy}hRL3Ariq4W!kW?@ zWMgL_Ui?~mmwDz)nFLlMC^S?7iUi_*?rE~ z!Bz*`3WIJxe*TmP-9gpZ07yz^u^)eXS<`+B63Gw1baOa!$Y@q0Q-SH-fWsQ)_m~-i z*W+;q1qaI_+EPFy-4jZCco5yHA^qJw%H!h4gjAX<$7dRz9{uwqeO;VCKeX z2z2d1kQ^11mG?kx5k3_seRZ=y3^f84(6>a)P;OP-CO?08R{+!w<;D-(?G^sKO&L~a z4+b+qJ^#@FE^h7(Fjnhy%$iy!iD>@pssAY)Qsgfo3F*O9m^E=b-xZLIz# zzMqOA*bhjvp{py3q7~`!F_=p_`ub@XZ7_|n%8{X?ik9sP71t1@;N$1dj4@5z6v$(c zS>t37oYRPkz$QCzcFIKsxow>P2sBftQX?d386G1oh_gzzcFI}Ktg8C39m*7-&cPUI zQ#CJG@Na$uhKAfj3RIJO&_0}w>|A@iuC8u~Sg+Uq@MFF?LV?Tdnl{u(N%n9GA=%yZ z^x^f%_!@0DH@8Raxj8v0ME~-W)wMhVCv2FD9$07dLj{JOsgHPNp|+ZWak#F#_W$D z36YVJTZZ?+>_(AO-nk{v$jE5_-n~!cN5*K89%#owQrW-l9&!h8mmCF`q|ehbZU2`e zKz1huH*q&JlNT4F1(mJPvLOGLEeTe`S1JZ8dTXO29cXu1atcO z`gXr}Et3GyA|H;hRw&A?86>bXkRdTC6$;C_;UMc}VX2UT^eWlpk`gf(*Z-obKW|u> znb~YW+k;2GXYbziB*%d~DV{Vg<2Hy$4KjiV&=3jZX{UfZ6k2%41;{iY6N7BT8ntM? zm%xJ`pKLvZDLXQwutSUe*ku|tgv@)9Mfv*!5w5JG)`#j3$e^jOPYL9Pr8e)YzVV^Z zk{V(E59bBA8Iy(EFq^<4Y`&h70^Np3Ch)0n0uR|3gp*L9|LUK1P(tongIv(Bq z`FuBc!@g6o$B}u0m_^32pbMCT0;T8X)}f$d&_C~D=_HSo)GjPFdA$wBF~jkFsN%5v zDf23S8(L2%zCwL^?0aG_$rzHHGODaOMx@B9)sIQl1;`8}ObAWmCBOmRArp9p&w)OsY1n+_@8Xfm z6D)vt_qG+~H{khZZ~PODCYGF|_jY^^K2z8fc!=%Gl=fcZL!f9>(*Q zmi=%fYT!_`PBtmq+w&t(8i2g;Ae^1>E$Q9omGrJP|I`9~~wm&e~5xJkfCP3StQA$fZ6C_+ZopXF`G3(dB8SF0(}X zr)1$wtXscfLqF675>d^}$`W=RJxzin_eCr>BNXy=a>p9uD0~%}-8g~~Jq50<53}?Z zuSf&BLudbxa(Twe%#$4(L;aN)QV7rp<&~5+E?kiY7i00Q1<68Tm}*_b9V=g%eX|el zL=Ci}+t4&!ZE5KyDp`eDvXmfy=&X5XdWd!Hy zx7%-!Z?h?6EIPjXdO(0un#}|sSo;i$M<1SZYe&Z`M27NF?gT#DRkzOp8y&}E`RU{4 zCYG6*`LNc`$iYL>#QSpTq;P}R6v6ngGy3t`9Hd|^X(D1wyi_uF9Q=J4T2g#@1N*{P zD;I(d@nW%?k#Ax~x@=IF?wr`06NlHY6}^7<6F}sH%}doMGZzs3rNJXWRFxL5X0pm0 zh0r@x_Ttd7V>^g}4Yw`;1;`xe3EAPGFU@&WK~xy15JaE#2d<}DzZS?JjU8`DV?x3} z8E*k177Yq*Kb8rMp4bQ4^EMYQpn9xDJGu^T552V&A$Zi!B(TEMC>1yvE|X|9ySHtI z?L^b}G3W%kZLPU5a1?n%73>n6d@dOo4hXAwRnKu)B^`<80P`&B!6fNq(`Sf6FIG| zP2OKZ{#at-8qAiC0EsW3`l=WeMBcmu+k+Y7W0xRCz@5&&)Gudg(95T)#C3P~zI~L) zqv90>N>~6GJ3uahh*hEcAK%N62%wMls}5byv%k2cxHus~Lu9;`r=Fui0UU_*e}?Y^sUR>#C-%iR-xxP6orVatzSNO$-d)uy7Q5}a(<`iXm7uH>z1nTtN=BHD}O{Ke^&9G0tP5oU8g=4 z^x7N}rKpk$06R!T_27&tNPy|)7++ssxzt-qii-UhtW?AXBQxLRZ5aMfpI%N&`@==L zJj&eyCZ{wR))P+vgT&?Aea>M^AnIKgl4S5byGkQQ!f+0}I{()&fCmaeWpWNb;VUsR|*9!tLtfW3jvzP{4@?w!95ulv8S_Zfh69CS7oMMRy_b8`Go>^0{* z|8boW<|g|jjRctSK=A$*vzs94@HosaUZg?EfQMg_KIz%n+1UrTwi?ItE=oJzNO7^Z zf)z+{B6i;7I7VL)ll%dMzL7-L(OGG~jb!0k@LoSS>OvwSH?j4=@e-4hU$HP1Dk%;Y zwzrR(4?6GpzJEVIKU982-?}r<`r&QfjgJq92M!V{&XUsaT&nJQr?SE_9#Sw6L+&qY z33MwAoIp8H#wZ0`-ah1bAu^jm!0vf!{bC>9jJ5?^n`ioGbf0r6cpr(K0j7L0uK?fz z{M&u-AT@UMe!O~0ZgADoO%5wpG{Ugi|756#0fbRH9FhldhSKqL_>D_Pc8-pYQcyr> ztnRlLUiWTu1-W>)xSy}Q}U<=7LEfod9uLmx>Uu$5qpYzV# zvFd1QN&)kTWycP@$z#6Mb6kL219B>?!gw{GvaT)@;ZE??H-_|D&Kt9W4U+D#(BiYQ zjy*gOT*ee{pHcN;7AIK+aKD$3HG@$AC@!=|>u74qBBFx_cJ7Fj0p-nXK8n?Y`wC|d zWQki^KBsIIrHMj)A?gT<7{=<<5oT3Jfnn6Z%PQg2(6UVwDplOQtGO<#=-6E0+7{m8 zrDPfGGFqAiW>8}Is08o}9H2REW`bd<`@?)#}OtY`>O%plZe^sGRb1SG`) zyR8QAX>l9td3^*)QW$(2ZXB=v)!oF##}NEPWiUk=@Bg@tqzs055fBIgd?e2URSvt5 zcHlSO=Vo)dDb?@L7EGm%etsrs;nyHRDdFu`d5~*|7^d``sjBuR(SO~j~M3wqLyN_chGLEo{0KR}i1j3|0c*ZO|$(xlI zTkr%{l{9PyhWookmB2VVJ~-?|%)fm2Q+P9KG$*nc!r$0-xEHb?g%f90ppSw}LMJE@ zBwjyaeVmvS(J7@u0qxt<`ho)LlzsfsynJF zWO)#W>e^k=z+)*gAB_$|C<5-YwaUkbASaZF`xtRNmXbxyh!=pH`W6Ps}A((a( zyEsUJiv*6LB!R9&hNESx5%Gnlc7GEV5CsXrwKcdDG>;Yu`P1|Yewe=o3KkAX&i<)! zxF!mZ5L$|e)aU9V&9tS*hv$@Sj(I1#;^nb0(^-;-KtoYtNP$OVt79QPb&rGJC@R?* zxI92PWVnnBec4REf>1jE&0}MDy`9^`G*k6c0gvRB1$u?` z=yF&X3%DP!VN4}?U_Di-{od@lFNi9fL&l1xW+}i7cM84?=tct17!@S%p#rNMZQUjU zDhxpqID^1uyh2J5F;-Dc-(@}h1P`2 ztcitf_m)C>cZ5ebbjKjBm!zEiW^2gn1*CL{vIF~S0C1G3SKD$iQozEG*IGs|yi)aXO)lUFE&74mJTI<+rc~Un)EqVb9|6129^!y~gV{Y6wcTv3c|20~5_w zY~fuxO8_13){$LpHOr+6Xd+2qT%#5njTWO(5JRU0vwOK}5!WT6OgqLaB1CdDh@(403y zihR(~+R9$8+95!j>V$Y2qIgGT_{?1+C=pZB_wk`f}W;>(bx@k0a- zPYcIokF+#91$m_{BG9Jj*o5~Nnr~+{FJQs*C%&@9eKNUau-^{)0H_s+>5VbkXV`@( z1oxtfT$Kr1L3DIuvU8ynKYmDnWd=I|f+bV6sCXgNl(&Dp@h2+VqeQn7PnZbVcv}T2 zteBxbpz{h%Ll{~mmP1Tf57Y`T93VdL0Ud2R#ex7;^i$j$NmOBH(r+bicF7c1lho^v|f&%*?N{E|V$5nG**n(^qC|JaaXpH{w z9eO|w=1<7n9A+jl<60%p6jA%(l zh3u?CWF=8XB@#&kksT4TDkSs&I9=c0e;mjEd)&wUz3=OyKA-pdHO}+7PUtDgoH^C# z^7x&FzLQpgkpSWAsHw2)bfz`y3sNU^RCl}a&DV@K$-*WbiQt(}fyPm%OP3~`$BLNV z8S!ZaqQbwhr?!q?#Euv`G|^}S~9)+b$hE<^VSQAZi7400cM=FFnQhqiICBF752{?h&LpWovF3FTS0 z49_U5I=$lOlI3Kf?K!$%qo8-lO=Jk7(1&3x0sYXl^feR1DyltqL>c`&+%+73Mmofu z_p34j;#E^q3xJl%2C1g9UjYGCvP78`8MVI>hKAz9}zgo3c!m za$IOIc_PG+u;k6P4}HseT5x;czI_U`XrkT*3$Eae(N(##VKtF`fV_nBSC(`*muO-# zXApfFjJf&6c{HeSuPVXT(@w@wxp|TW-0$*}t(!Lof}l1Ncm}b}ChZ8|l|_==Y|_fBYf5dY!uF8BN`VrVSrWmWe5v}We{*f^PA3h< zFVnX=j=Ddo+QXd=?WbE29r-2;4 zE*E{xX@9!*=y8bKcO;E#qL&Z;439UWdBWxMVquCv*(f4O2^dDw>AqC3NXMz}Z!IVk z-y#A`qFe21W~QTvJ$Z6CxL`;5ptgFEiH1wGM#gW!Ry2?V?@}!103i$e{RLe+LT_G5 z01?%b)8#t^FtRyrba!f*`uj@AIy-{{U<1*dhv4)Dh&hc>#X=Xn(v67IaCUWdBz9&$ z>scrVBnUq+uOB&Zz`D3FwKWBh1wdC*l{!giXi``Gy_{%xIy;g|8aq@okODSkt2U$` z8Z-^#d-{nM6vATaoHewxntv1r zCqVW1gNFEqL5}R4oHhVQ>uy{6hNWePS)59^*rR0kj_0D|z8>3n~b}0pI`9gTXh`s&i-h0#N|7ZbL$WuyULs0aNOE!sH^!Lw-oxK+5 zfvE2su&O_48pf_o9ovgl0j-Z-|GmRO!VYrOdpW*aNVB@O|DW%kRM{TuH=`*7>ewQD z?P7Z{gtr{vpkiIj2j>8hb}%zLb=_wzt#cs^;o7{iJ1ojK*sk03zfWS0y^dN0=VM>J zW4;gP9icty0}tCuTYD7U`r9*k{SQK^-B!C^7%q`Loa2$0E$KlEc!`4vr7ccuzHxNN zIPuAI;$=(vIqHSk_9m(ZJUv1L9bj-Il=f(R{D@?OIK8b7Z($)U{J+h9QQ|DH8VWqN&2cU`>$fGIAgJ?d8YT0&HqJh|jQt~jdsU>c!2RcB%R zlG`8N#%J>8sES1fD}EbxxR>KdGFrMPfs`j=B38(1hH>@Csna05H9;kZ5G=`qfJBkR zkY#s|te|hfI<=O9^tjuF$@tJ|9BKm{^I_s@le~i;LT-|%GZ0u#ei}!?o(@N^ zOHQ>?dJz9mqrW5*?x@&g3p+4kh^wlecohV(A!s&K3H`kpr zXU-4cpZL^N7e4(JphIs67|NaxQ3)h4QC-xjr*g)7(-H=2lL zNShDfi2CL``EfsWy)~mdg7Az<-lDPQ`pc3#!v|br0S4lKo^o?b2jfniMkSl1M9~A$ zac6+`0`8fSOpt7#$XoKzXVIea({Ft=0P6!GETrDrj;ItRz*bZHkARzkm5ZaCeE}F> z7wa4`d&aI>j!gy^e`@V@cyc=c2#OVb2w?3Vv(p7=MlQ%zV^v%@NfRPjI3r>9Ct(dh z0#k=_MiDb7yPTSb>RDGuM}Zq(ukb0prTZ$#INK9bd}O1|xzR zDxgb^UDr*NLXvc%p$kF>40uq@(9}PV|nH zS4l~ui=#9rC3UkkXqrK+H!$rr%sWy{V1pPr>>Xe2VIX&h@C4|f@YFXzYzmyXFVX9g z89=H04Ni%TJgW5E+(y{5Mf`+MSG{%X_0$W-u0}b34-c~O0t`RD^i$hEzpE!pryjX; z{CEs}|5%J65JGa6c;&c@$^!OjE%Xd0TXOk10#B@|>)^M5q@UQUPFSV{q=|(s2yTVu zbc<=IJ7KxP2+^Hi`!X_F7WVlI7xWp#AXGWp%|7;q4H`K5lU_5B4o7~x$cuRcBa;nM zJ`;SY3u!>@O${^{lOv8mRhAY6Wc=_&&8`4n+fVGg?bokg%`IDeXhjG3W!2y6Ra0lp za-@*H3mH;L>KgLF9>r%Z@smGYQSF9qMQ8QEfusRipz9{|XpY*+6DKx`vkj?^9}IHAM_#G2^YLCD*6*fjh(;_A!_gFy?ck-<~{y-d~#Y0Ope{4LBW7kL}7=MT2VN~ zMyFUyvZ`hK+5=tUWAR4$^o47^v73JJ#2V_UI{-W!E#Y!{XJ#JEh-r}Ln*Xe5R^e0s z19sx>Ta*fJ~MmH(~BwhzC3hr&6)n&S7~*CTu}Is#5(c-C>h=SoS@oVZ>kj8sJi#U8)x8nDvSP6$ zicW&~oG^ZT1l2rM)<#ZRvW+(rTwg!t3YXv2=KwSq$ascR_E?K-wuMNLK{aEzA4|n|*b{dF&m<-K8 z4YzQ~l8^b_7cO3GzzHdZ1lop3iY*`&EtEF##DQY~Ia3oq??_VF&ZMs~|85GP06p0F zIPW8Q^zK@td%zvn380_*r%gZ&G_5*N@@DMZ-1Y#=wK#OPU-f7=w@cTSuDO^qcGlao z!O79XCs?vhRU0;pg(^THaEJqg3mn%)@E5}c2?b{z#4zZq9(KqVTZM(E%I+1oWfO0H6uCAT?Y~OJ5 zMUXaV^eAsJ`< zT~ZQf%sz6rN#fvLVPW-UFIjc@PRBKaI!N{2!x;aEJNN75CEjycHfR6wSEE{ff)#A^ z0x8dC{y2Af_sj5B9rXp z?7}I~u|o&#&B8Px&EN!&SOMtFGX>l5~_pTfFQt9y0)hzc++gsU`Z29Iue|GkQQj3aMQ z6dVA`xI<9D{`~SHo?WCv(m}6d2jAX1eEe0FUj;6W`ieH>BKQ9H%>MkU8qQe<6&L%Z zY!wrCOtm`Zw)%jK9AUMQ(m=oy)k||f?AOl=OHEyNrflY~-AU)#&Wi7kxpGBl`R3O% z;(sN#pI4dC^Pzu}KC^7&dhhJ?vtO~Z?y@EQP*We7-KxzC;;QA)0m#i=GXHg1nM~l4 znvZ+^K%HQdujg}z=wm!mD0s>n(ne}>#0G+{qh-43tF28h2?8I?C8gy1dh5{FA71-4 z*woI_SHV(8$oK-md=`a%Bv=KVzj6hq$gba)_^_Kj6O)3i#g_PJ_sOh2UMFJuzUy$9 zhUEGDUQQdeH8q9Bp$;c@bZ$lUhZLhZDeZ4wNl#ZO#2SrNZbG~ZJkF!$!%bS&R#5VY z@=u`n?tbx6FyQoH46rwrR|*?u&;F8y5Joi_6E-4G1XLssJ;nI#&68 z_PsdvWwzy4pdAm`ctrs9oG0kuhh>L5uG!eXJ@jgB%*j8CaF{$gu`HqghL^{V98t_z zwW2Vx@=^3f`jngxZSEBK>&$z&^V8D+)*dTji|N59gy7c^2#+#y0V(te=`f!mS@*zq`R*B;6JA>1&z9~_Zj@^NDl zOj8VVb8>2bPE%g)C zHRH~mQSO6P^Dp~9oR~Q=#4=~BYvHU7$DnnG;;iS6}_32Cihnjd+55!N4ZUh zv^h7lb<&m{)UOW6dDWxb@APS1ql2cOb|=5SaHjB+0~lx>F5{cg1=H55AVj)O7>5Le zqdtF`Qi1s5@{f0&1X<#=v7u@bZiI`uZii{^Gc2=4qI;(*KoWE!J#tS7mgAlk9wPkX z9!3?k!;`+I^{4^XFk{ZZ=RxJ&nAk5}8a66b5|(W4IBx(@fOzWnOm zF>lww0VJRX_pLJ{uO9*AGfp|*XDk*7M}7yda=*7{xAJ^9p6EAYR?4V4?NTiEatYpAz zl*Me&cGD0lwt3Z`P*pe*j3dYfQs~^Q_@hS;@QEr0#U)zA`JJ51d18Tcdr-K0eo>JT zY=0`)@0fApMgi*4!*5XJOUm3;qL&SU7^S6L^`|BjobCwbt#6;64Mz&#eJY}e40uI8 zA|K-c1NL%wbRYwWT+hCDw@3Num;)+NeM@CRm7UMzfkEKtTOuP1-la^s>x@{eJ$kHE zR>HBic02Fn$NYD*$!rt$(5R!ztu{2;nl)(?(B95HhsDDffx=~1k>!o>Z;$WbTZ!Th z=(qCMb^rw{4spn&FuOlq$IgCvXiYKcIouj;N+ZkqZ7h9HW%xaE{n$sF)|XJr12pFy zES^MjaRQ5jaLH>1fjb-!pg4N`_?*fMbotHp=6&iqH%t3YWSK3O5TLEgSG9XF%hucd z`D*W<^MAi3(ZkmXkbw-exfISeQcK@J*f_yv|JkL({41Zddb_lRLDTw9STJZaicBxA zBq{O^c(T+?=%xECk6~XgT)MP9G-DqYSm3AFksWHje^;YMV!xyvnHn8pG4Dta0ac-y z-pM^js)O2uap&x!C=$BU#KY9YBvvdmEAM^%`W4)K(#*<*xnMK6 zN=ATU2%bYBdI_E37j|~ntw)h_!M<2L(tgky2Kfsf*8=Q-cKkWS2$u+I-VzmMZF~29 z3P?TwUHYBR{&4Y}omFr9lf;vCn2@2Li>gP|l9a-k_q%(4>Nmmw2d0#;m?j+VIPd-O z^9Y!z3kGL?LH9?!jtz&(^qx7hke2p@PTYqJhrtGAK_c!uELSjglZqfu6s zRI!biznHT&vm~F>bjM5FFnmx)%eE&cbXjWB#kML$RlR^QEb(B0TXyeGu(JX}HtOF$ z2F!N`h}F&LD`qq_(hCwklkDH|A#PCzjy3Y&FAW=cy>K&3y&+R0&|Zfn#+h9N&9IP2 zBq9fEZ`(U(KV1Y~SZnsbTVnf^e%V@Q`$fj=K37)YT4icu(((>=T$_oV$HhC=et46W z?#)~mNHoLhCGRPF1^=TEa=_r@88}P{fQp@ec;cP%SC3z-bx$iii_EEQ+qNPWpan(% znLp8@Wy?6x-Z81h0~8e{V^pDrdJZa=52sq59^Vk?6yu;zkUfRA7UoK3M0Y`k{ z7VlaguGcJQ&RcD)4anUj+6>+b9ye7l`1GlyY-Gk}EJlO`ddyPD(d?j@WY=JC-gOXt z-KJr}C}EiF-O=-H2k{O}zT6tk;&-~y3+p45Ecy2A+JCf|L1q0U&it{PbUOcIu27 zc91D^t2^K6x991k!PF!(C~xHO_`x3BVtRN~3g8#P4eIFV!JIlR*U#PPyvlU5Np?}B zv6kPqt+#Kwj_Y#A?QKj))i7IY;s~bBojVM^hdkZnvG;fQ)J6E{1ibNdS^070n95Qo zjDW4Qw7SML+|>YhW6PdBPrS93shgf05x)sQ7|>)VdJ5UoZs)EXn9u1gump;tH?+T_ zl6Ma)1J{%;3xM@Q#JFD3F+-4I5{EZ}D(qFFSyEr;w2XgP`laeg&8DmHa4Yv8Jxa*^ z*KvX*0_cM44m$TnC*t^C?~B=&x=EI&tE+3)!Re0Z3iELS^c~~aW&fOzbqrlnktP@_ z#D#K3({Kmg#nsL`340r0Ddb43tZI^i$qXnT|wZI4?jjcFlyCVb|l zt(r_;$Ca{9>YBu)Bm?#kGTi+E0ReMM+HOUtN`T*BITa`ZM>3KV#qJXA!=3da*De_F z_`#g{(?4UU>w4n8t9Jd;g(-_+-iO`4o9VrGzx9_}3!f0j2bjAVQQrq*CF!~Q+CU;4 z0%DQzXIuq_G}~mxI-i`qh2{B>;2&=PC}?SH6P;X}hAa7f|E!x!%rNG#IS)>ir1naqUXs7X6or4__P6!tay4aipE#8Z;@!SO`&Yx;#dXs zK0PHviFiNJ0^E$gex>l0NaILbk}x$kvzog27)5Xl;`6+!1~Pe?Gzir7))+D26C~66 zdS7rK3J&|i_X@{Bp`<^+BG!mhtzIhHh$(bzOsHmn)-cg7dlico8 zdn`e2Ubw-#HQ8p&KcH+^Ik>Jss$MYx1h!GOv>x? zR5Ha~Cyd`B@Vnr*Am-26Z|fJ(3I;;n9pvug3C@)%Pe5jw_wKcTlW@CI<&LP6f?C0P z6Zvn)t!vh%zgmbYB!oqx{Bd0v29EmYeac(HQ4(<;2si?mQCk5bCC0{DwUr0Lqe28R z|0e*R6nf}Xic(Z2UvSjjftRbVs2^;N zo9%$Os3`wW?zX;J=+AW<2j_T)z(p%QMx6kGQ(1nCNwnb{8ku}m!}@jcEb!|+?~0Ed zMACJK`HaJ9qU!|sTDGmVPmWu@>l-|;Uv73fDHL)o9ngLG)y zZTl~`<8G0Cz6Arn4Hm;&qq|eB{~&(v4YiA#m)q>?hgbI71Q$lQb3?8J5$%&+7RKB4 zp~6{5-s>6PQ@9n5|Ei;yyLCNO>jHE=kvQ6@9K+SH0@#(9jWVK79!h+W|G$OsUk ze*S_5tHDA>k&$K>JDb1ztjBVap?Sy^ zGHGl+^IAH&jy#6M;klqCd^;;!(cU3c#Zk}%6PT1e@cc+!Z$?uownh_TCW%D57OeXy2c4QYlBPK=+2_j6Y~TQP0!3s zjU6UxMlz@?;I<^>?R|5r-Xazetp_jH8dX1NWe5bc2W0o`R^`?l{?rn6c{Q@_N4UC% zW2cr08{(er)p}#1;kWT?Rp1%VGf$QtYBi6AIuL`4CxqGhlCBgZ;0$M~*Tm)EK3~U@ zg?iCmzd#TGDks>VD}=aHN^p=7bEJx{Y16xRGr=_^^p=WnDAK$D(mP!-ZZkK(i-u3l!ax!xM-4Ks z)gYG=nJ48H3v=C;!jupPPxO~+Xoi5od9}4I+cp6)6#|JTCsIu1T;!i~^XC#&b%vjFsYX6aAz2*_8{`A)&XHG5d zkvP2@^b%)-_)`(*()rPPl$4g%dDi?# zg**+-&WULvZ0ve)z3^sGU(`|a$%|uXi?Qp-@a8RBO3vRksuS@%$T|=g4&tCXN~;rJ z`D`au=>_MUXtg0AnBig}AQ-N-F}9WKx{rQWg%(8?lNiVll_7Ae0}CTV**Gp?yF|gp z=l-W)vt03`Ikbs5Q{mghX~&`v_41P^ZAb+nD62gj)K=58_eFw9&yDTiIeUVB;hN}- zm`Fj=gG_{N)bhpY?0WyRuUS|$7n>ASDz!?$^ze4mZ*5;K1_gfR`=YJmhR0G_%W7U! z))+_ee3HR2vF%5+jk=!u6LUq-9z4?T-!p}zwzl@JxdV=e4C;;7K$QKW(sXdJmR0=Z zLMU9Ml|j?hD3qug_={7oz=%yg-+!x}0Tz)NQ!a|6oA)MstCp4-(4{6I!<8XZNHAF~ zN*{EW>2wPh61E+sw}*_WC%`N+WwGW_URR%AG8|2hR0UM_gwZ!-hOuOuGGim?_Rz6@ ztY{vxNZ^7(LS9hqpdqcLo^ki?7IPO4v|w8O#H{crZm}!;0L@H;jvZAw2Su*MkP&om z5|YbfG+s6%BCZ8jgiQYya}>9*@2~IXuzBQ}D}nsTvqnKUd((n?VoZaenvXSHWU-{~ ziQ1Zs@qa&z8FQR0QWfC=k*errRwLjzPm-HHfBTjNCP5^8{QHDc7PUn&dTDh@6e&`y ztONK!6sSWVq|aw{uEwA!^5~&?7U2uA4oOsB!=@AD=_CECY6xVT36+jYbK3eSU7HZ& zyJoQaeC7q=$_@ACg{GwV1r}%&Pxd=;Y_pw#*x)3pgF8$lUTz+3hInG`;VKKIJ+#(Q zdoI-?z^nhD>w{R1@@E01WMCfM^qyC%lgc$aZeWa>xJj@u?igG_)2&cYv<)qhp$4PI zqb%Q=kUBjA5f>+y5wE`MrFDua?(P|yn~=3i;qXLdX*Y~2^RWmXS|sHvm%%Md7xYtd znX%hHwOq8l#URdT&jctCeU1cRIYDf>hb3e(GcO}!|FsSqr3eF%|DXC1bc(TJ^0Am0DogpZAn)95OaGnVU|cUL~AHB zPJomj+NUftht6I{;GH(IKoNDqfDnz^_?4<6- z#(Y^|6eR@;I%>*4P;+DKN!tGOvR>PHqJ}8yeVZ zGbuTWRg>$93Xee^9H}ARK^iSutj>9Lg;7?(U~<#2yQ|I}Yn+)QiO({0jDAAY`5deK zCt~MLRlo=;Q_)SY9?=s)b>rigm>4f&N-1S2k~t0JP{Y7YxSIR@7CyC6HxW)y51$i- zd-E18%ztX~ztKC0{p3hOf-T4i3$DtwacFJoDUn2Dycgv>b~NfY*1ZaBi$wIaS)h7| z6Ps;{+1*NPAXBe%n2Qa9L*jK$ip}-v4|_3Lec$!1ulRMM9uBM3tX;cJM3yblHshL* zvmIBRMDyZgy)kGy(+wZ@S;Z%ixi{ig;j-P+J1T#Uq!wVU4y8R8X$s!=P-2ZP4tsW~ z;nwJg4T`(-d~^EWe|BwDa^J(q;vfWYEvq8K0rn6k2X5CU4M^mbvYo>D|1rw-?5V+g z&Ow-tWDTa#k5ij^t&|ZRS=-4>n)athPiwIS$WT{iEA`{-`Ap675*_^iOzOt zD=@{?+@!Q@?gZ;|rAfKMsrRAzJfFniYWj#?B)ju#&B#V&qra4(wX&SOgEo?SuLZOm zI8bLwOancSiBUI;jJscl6JR{u?b$b@882cHiv_Y<3~;ZJK%Fq{e)9fZ#KPFf6bB6%@^o(PIpT> zpx5iv%(};Yl2-gk8b0gzg?0D(gg4M|X}#~Pu}8San8&ld=Be4K>?-~9yQE8vmQ|;9 zLD^+KZYfc%huyBJ6Rk3`I~{hwj$K81&eSG+Io7Vp*N&qv4!L-zs% zjvWW?+_8g!Y7IgCnj8I1)2MwbytWb#*vh#1n#>X`lGYv5*=kdGt#6)O=n2UZ%puv+ zwI2=25QOt9$?KE&uH)yxG6Qj&X*Oxs|0)2c`Kmv^rX8!WK~}FsA;j_L0U|+vIdfy~ zN=>U*KHyR8t#yocgYmYzy>{(9+AFS!t;;rGeu7hrol8F^UQBU7jvl=)F0L-pd+ttb zn&SQ<&GOZ$%#!&AzyH=%S{>VA{QGNKRF%$HD^cKZQ=u_e!k~n;vew?bA&68Bwp{!k z{IhaE9IhkTV>EL0r@jc7#vFAzjI=Sv@L}Tb<%n9(@rR!1xZ#ihov!FUBj~mPfzgB6 zw|P$ND?Rt{nJf4dX!+XWd5J3aNtxJL+;V(OXs8>DD#~p5tVb{G_=M$KcASw&z4<`Q-J8z;LUjSP(l%*#CX zCxnB+aZmSEHJ=i;fB*C(Tnw9&CvW0wp_6jxU(O+>rcwv(ICogohjjzWac@ARpavYm zdky@s4mqBM*5|7^SF2&z|2ya+Z(Xj}-+;eKsXjOVM--*9yV z!H{+mJ*Nl|Su{#jr@3`WdJthb4>}k;m{F2Hj+fU&+#@}J{_6oD-%>1r2ETLbSyNJP zYtIopK!B3?F2A=04&?E{ETI|4foobbG9f-ba=Ckto*1UFVLCK^_aB{VS`#(LWN290 zRF1x9Kb`ZCsMBkIN3cSHUb|ksde}-bfgLa3eRSkMdm^&$frVcUL1KWYqRL*VRd^gx zo9z6fv(WsFFJJA6!_3~U>E0GC22ho63!gO~NI|^xC~skCUUGZq-@kty8vws(HEs2% zaYI|VlDEbND^qU2f6^g5y9n>8GCb&1a$RbBHsnRt-kVo%({)UYDY1ssoit&NF%9Y{ zLDD-6+Z|RLKxIR$zY88mP0vPGRyeAMwqfhfPnmdQdh73L0kO=%sL`hS3d23!U1ch- zJj#ci=i=o!&lSdPHEfyGtCBB4tq_|U85Zg}7V{!94Xugke-G6f%yQvHr&n#x;u$S^ zeJcY6=+h&P5700(U#S*iLrRdrbN!zr#Zn?(A=>~<$9Bq;=NE?BTUo`WU_?29{UZHu z_nv8QuANEive#QFW>xXJ0rR_@yPYlB)X1D(aHY#A6skva`t2T%UIkd>h95WVvNBp> zcl+pN-v}kUb0crQoc`L4b4HJ#Xw2s6+#-!4?dniXp~e&PU*;pFtaBMpJJ0qi*yr_g z5Tm7tZf!%T_=DNMKe-aGLS#aerN+7W79l)KAHP`%-IXK$vpkEfEd`OG<+*xg*~ARl;jYo#4A|X zXPihg`VKfDT~orOOIibdea?I|H)M^f_awO~DV8~C>+#A38KLcJ_t^JEP*O`;RC0Wv zjIINKjc|5WaP>-)#~0GRN8Z`DSP5P9qL>ZM6GCZ2MygZE>RoE{|p1Bzwj!Ft1j zlIlY7iB&Za!fy(w%L7uNHxKU$!KkmRTZ?LPFd&+NTc-WMZm_XK8-*i<`RlP!aPde4dbC(W3BuCF{wcWgLd&$Yp~TxKWbQ*fDIvzna!n6)*(-zrybtNRNIY6YFTe{Uvtc!01RdY~Y4xs=WhpXbwbrd0f>=o!8?OPR#MdkQ zGd5mv*KupHJJO##xfFcNDAtG5AOq2rNO9;Bd7?>!s`E5!Z_C-bv)PcLk5fC$^=4dC zAiV}qj+neS26Z}gs889N>oUr9_6T)X98J;`F;DnVQs+_kyXf`h;$&y`L>;fZ-&qAU z<8m$ovIe&Jcn2lfGe}Xu5|Yr^EMOva+3VtM73=|X8s7#wN2h$`vt6_ayL&BA0!zpQ z4R!~|fbe7_XI5EPDS$-({z>@l;O1Tj)qovm82jFcWq3_}R1HyUrO7`?)AMC_6MFLX8~bTRekw zIESe&60)+3Ht5kszit;u&B?iOqvrm7r10Ujrckpk@6tW&!b-mUX^A$a9yoOP@O66Z zVs2bb&-GiQqkFuG`i7dsEKBp)v16NE`pXDZrgz*}#0Ojg0R0+0DW%j7IxtQM&G3yI z=Q@+(WzI3&>I{Y<(yp6D7G9G^o1rX)Dj!aOk(LUq9TTONygaaFP}p@b~l;w-6)N zngU25|4+i@%aib9S!;T-YfMlT{3bo@R%T`(EUCi}A50K?h~C@7e1I@F#>(-Z8>x?1 z@n0g!y3W9(Y=s#WOEMXOMrtZk-L!NU8sa+Jz?6Xw#>S^`@?~EuOEZ^qyc@H@=C#+p zYUL$JF`(^G9ko+%>SjoP<$c8aU_#Di*BaJ(O)EFdq*t%;Gk1X0LXzLQal<~5NkH?? zuqw>?sAPB;PUwjq-G}s9(3vxI4Ba>uPz*lcYwe&Ud&F2tf4{%nOkUK4PBp(rI_n<4 z!zamwZrjcuC@+j_@QAU);gf@gsMIXwv@x-;*z%j)BzJL);y14~g;FGQ%xT!o%^pPj%7ItrUilpMeh zDlvB}IlLW|>#0?N0w}0aug4Z0KXGCjo&8Y6%>^0OPsd<2sGZJde)`3*`DnZH=-s<- zQP#+!>1JUum3za3-Z-&yxLTza9shd{>GzDa=^~oU^2aG>rdmh^H*Na#>%jI|q90m< zX|H9wc6qe-&t{2iYc^S#w+N9V<-aS~^b)cywO*}{IOohvT5-^riK2|10poqY$${!j zZUl1GSA2V#+@*2HE};RCv1W?*G{_@!KD>Xw9S?J-&YhzlW?^K|n(g}dK%pjGgYv+I zzhc>U3bX!lsKZg3E8g>yoK~*-Yox2Yzo5ECYpJ*M(!!%YOxYuHTEvYTC;?w!!p@NX z5Uq-g1tY{g+9xb1VjIH;Gl8Jk{@Y9>hQ!)>%~ZW)=(5(9zG}ccv=)LVaN!Wb+K}Gp z#-;D4FJE4{xmW7?@ci`CnJykzAf+ z2dp<2!a+>jTwYl334-`>@+3cbR;0R}&*H_gF`Yo41_s{npx#}_&FVqD_nom&5xaM< zhtvlb?Vd`n_hj%k?N+Tw@YjGi#tL7XDw3`MU5-`)vDFQq*QD#^I$6#@By>YNjS4Ta z0Ue`Iy^{R6+tRNo>sMydAsfPts^Fzf*k-5V*N8iYkx}O|?Buc&NQ#CP?q?l3WE_Re z##LoUM3va^S+#;RDqa#9Yu9LLhqhzC(?K zHlTnDrwmypGj(vVjtN(xqnu2!73}9 z`c&9j1!uQ_77%ciT2x9~u3>XzNql(?NsWQlEZ$o`QlFMd2sEIoJI|+6!ZNLPi>jGa z%cKBMZZsl6R@AiuZ?d7FB#5g}Vzo-B1yQifa2t29o?%eLxMpFt{GK5c*>dGlFL)wx z+unc0vQg7Dw$+C|q@WM@{dYnyga@Y=m5JBFnvYJjf+lW1)`~ga8BUM%2M@kqN;0@; z$_x6J21=(S=mD4@=_!A=mh&!OQsPkJZ87ox@BcW`Btsp*y_dDT|Eomx5=k;fx~i#?(?8$N#H7;%@mLX{8L<<)=$rH zSSDlo?17Ipt7N`rXeQkEv|E_2Xk6mmExA~B0DfurTV^45;7?=`#eC~xim?amxVL=+ zBmSPmlK+9>A#B*KoE?djJ*VkhZl)#}En2wn$Ne2|uGyU6>f@H^*#l;KnukZ_)0xZe z-TK0*HS>$LZ&pl`@$lfckzPH3lbn`7W8fAEcD$3;>HVmn$ z8>M=uVD>UG89H+Z0Rkknn zKM=Jta&mV+|CTs&^7Hf26x0?e%lY%?Eiy06ogpA4!RZ{w&%PL1bn931`LnO3C^z_N zwj)NUld2^r_C7@RiJdp7RYp_t3*t_%#lm3PAI5
$F`mjiUx@odO%9*^-S+GK(fVxC`Uk;b!O5MoT~y>6^y`LndN(ms#J zd4I6-Vx*!RnAA8s7j9@)w?$zOG8bit3I8;dZi5tuh7|i?pf^%2tg{`31i-5x>mE{e zwM_oN+lgSKIgfpdkKf;;d_LMt;zA;Vwu2o@Rvh9Y0cKOXbm$QIZjBM;Owo*e1!S>d z0>!OhunFjs|7MngsUI1-58*P^K0BkSI%4df3ByMUgv$?KF^pr{~31t%gpx^#L={DelM;# z8wOXMT3%DB99WxuvH1N1bsDRy7`%6VYcSV{j>m>f6sZir5T>B{Cv$4moVBxatzOZT zNJIr2li<87hc0dbd2wxP=^MahCa+dP5K+*o(u>@9@H9xRm?YV2@TW2jijOP|9ddXf z!?`P_x;^zrz*~J&t*WwXMXPtbb4IV(=%|5NyxaPSGBlKDMMbdh*s%tn*Z{{2VkGuW z8yK9-#9$zA5LtQH)Y?cPVoN9s|TR`ym`1*}Q^^2ALdQ#!@R#yX8t)FrK zLE?Wv>;dLqb1{PVfMZ$57B~Cyf3yHFvYZU$rXZXUDg-E+Zy{$V8HSXUT$}t4nS&>1 z&r!sjn*vD48f}c=xAO1cU#qZATZ8rCn~5qTyCLv+=7oBq?Ao5Qs zLmH)?zO=E|c}1RUATUiW%TIs1v~BCMe5~91YZ3a52X<-N&LnMPFXv09{r7h1y?whVf2!7U_QQoIQQ%`>PLNCSM*B~Z)lFA{vX!nWZB^* zQXHM8VUqrXu%;k({qczJir=rg)x4a(yuWIH>2tv!A^PUEgDko?#0}|d94z%lr23U-tk{b``FuH5PXD)AV>%rDf zqj=i%TrvZK1R)!A`e3)$$a*!OlCGGrt$jC&Y%Z#mhnuyNlqXsksXY1p=iVsVR(3{W zfe6VIdgo|-=L4gOgN)8jrd(pT2A5n6Lzt6#u}PLlDg|pQC{U*vJw)uR(CGbt ze;LKZNny`EHEr49h+1XG;W3sOyzJ9_Tehhf?qM%6mu(eXXc-Wf=-SYUaeIm+NWh+* z-5m?J2CEfXfFES^XV%Wzbzb8WO&v= z;NnlK{)Qs}Eg{!A1A@5vgqxpy7{TUFF|!w2R&9*&gzidoY(8^JSY~#|sGkGT%G-yu z=z)qJ!QAt*vU;>w6SfUmI}z8+HBo4Z7>!RL4JwrunyL-z`h)X7-!U~n4=0ZweT`Vg z=@4>KW5*hz8R$0cRyrLWS7&1$PLJ!;J{A0(YyBsfqJ_lHzEm|L)DTe*${jP_=w85u2M^_lWbAP&$>hT3 zBVC-4d8wn-VxwkKvC3y0J2pBYC#pAiA=7c|0ynRwsHA(_)hSk?I5H?W!l>zF-*zk; zdQ>S4i5i~}Xn#D@b1^}h@}hA zH_q-0l3F~SPzd4P>E7gNd|c}~Or#as+6}1e zWh6KjL4;(7^1BS7wPJXKvXnMlQy{(pWI5HlpJ>utRU*&q=QSnBMLw%0W9WZAHkFTq zT}uR&(y{@PN?e+J9C|@7K8}-K%q}a-gi|0OqLIYCybt|xV&_e2DT{cFA`1{5ICPRM zCPq0gN5ahj5;eDZdSxQ2#?C8Zac)1lt}jzJfj(603&PzX2gS`fAW7oo7HgZ^h|v4id0(prFb3X$X9LOZlA`G0T+>MB6$|}7zHYRtihd6c;()@C5dkUF7uIQ zArhYglqQdF!v-hEXltwBh-%Y5eY8kA7(y+Ty7=uHkzVKJopk!*X;^9XEA%V)faI^E zc#;B7GSxdjH;iT0i>}D6iN!~cHr2~31DOWc{l2+wbJ*%lMN7b0EOHwl0?D^FM|pwD zMv30VmgKTsfxy&Zn750yg(J;;HMH3XMT)4W1#6NH#A>TWPuW; z%49n>kqGwLKY<)?>1fwE_vSQORnPc{T45pUMO;Z9>=nTkxI8Ktk|vc5ZCz3=IwzF| z4K^=!L9I|27n}QFg?`Nnpxfr*>h6|p{}Em zdjJm4-%tWgK*`DD7*iA%*(Abud~$M7ZEu| zKxMVIQygh;N~VWZrVHz1^E4zZw1wO*VvmMQW+Ey_>MoaZ6~7JC(F~$_E)mi4K?w?z zQLX?EQS&{JrDg-L%%fAsyQ2)E)47>yY3&jR z*DQ&ETxq-@3iCIB|2!)uX!L|n+cJ2D!8}ua{o33F&|XTPe|dbkvoHpOf3Hz zI(${k-9rAc76G> z7@5Lz)aD^}1~V56XCS73D%KSZ#3NrsMI+*f%B{zZyNHcE_jJ`K`CKQ(9wj|EM<6vV!<&}`p6SWT zl5QOhG!Ccs2OXY)U^k+zGR$M^v0e(M^rLr+AA4Zi-c>Vjj>vRuskG_oaB8gi&to)n z`fB@KxZoPylr1r4L@Qq90|IOblk~dPDhyBR<-Ni|b3!6C|9#G2>1Uqr93G z)G}e{T+k=mr=l+qluxNB zb>_>8`7f^4XU=Z}ez^_J3l7uTTt$AZe#S&pGvHZ+5)yZ+*kxsKFWoFET2*L8sZnHS z@t+CX93CFwE|AiP_H;w?QEnfJE+=2&Qp2O0`?AF&ml6R`kH7Hqp$|E9FscDSoX9WU z@2Lj!6k`UuI9;l8enQlBj0}*>i4dr<-;UO{l(6>De1I>3%TPGP|Py@qwsr<`ikeDU#in>@9H({e7s1tx#e%`L_Ex+UF&Gpe<$&A!Fbh(?(EV zBZ1sq8Za5_9w0&y6Op{%eC&(6ve_F*4wh}Hkk)-7F zY~uH#oRbM6=U4pfA+8I81*LvmZ_a#XKH}Jg&*%O#K4_zKyqxkZ>K4^mLnOsf&PI)A z02}`sQo?Am1FQc2d2|0{J6edRzShG|YjE!%s0Gy^t#KWPNRPMAHmmtX^Em1ovNEIQ zPK5N0DvH9+wI2?r-A$86u^U8~&w$U>66c125nOXHk5SWeJRtxa>t1dLE&>ItwG_1k zjFbLZ;z8^4ji#*M&&818(}_CjpP~y|2h@zJQICjOAO~?q0k05Gq=J+lwRjUi(!j7V zd&kSi$Hs%ZaS}w=Nuk`-|h8t!_&SoGB2yihl0BgE_ z_}VqdB9uduF1DWRI{MDJ&n|Dk*2Rr2)q{+HLHC7^Za{W3o!Ad+*RGXXfI(m3p!W*j zKh?zAwUR4FM1epVH_tnySXl+D`Hh<}p@UDxGFeQOl?Izo63D^A?_zP=|H$cRY#d;& z499iDDjCT~>XHu*PKERE%r*mZyAB-0u_SZrtYMX?fP;t+V-Bqfe}N$5VO)(8Ejb_Xpe0=pv*L?-HI>j67xi8o*IvA)|e+Y_5C#XQ-WZ?)62ZO z_`T*?pJf6jSBQnj%UuUSNkThwAaE7*rJoH1qn8)JBjXg8vq~ClRij)3E_95nQrn@y zleu>?gN0g&Q#g};Ql``W&>z%a6KH|zeoIgk#WcPHzA_XV57ut3L|m=hprL!bE-nu; zDLCdt9X6tqUll7QI}}}~D&=F;neUuEa$}>*l86Tl?{hMvkgGs^s?elNoD75`!z|~$ z-Cb`R$F#V83 z4p_M!ErHB`SZ^*8V}x#Uxf@h3Zlmp;BWXPdYhKc03$y6cyIs?NCs&rH@e>K9W4nG?0`0_lS12Gzj^6S0&c=KhWza$9Z>_}`E5ucY4)$HmucV9YCFpLli4;q+# zJnqEKZ*7KvR1^(0XzEViFVa(Cc6=*5W)AYU5{D5_U!i~$XC~6z3N33J8`&E?MPQtr zM(Ufp2{)-$>7l7U-~>@%3eIh5sm6{SD}4yPA=zll_HjAF#>Reh3Ch;=YzY7J4>4f$ z9`!TOWU`zE$H#Ega*BxAMkIonnKng+H`-9|F>-Go^aPE(fB6Ri7%1<%*6*m0_!2N2 zlADxerWy-w=sTT8(1oHYY1OLL$Xd3`yR;qCK;pOZDS~+M_8+fpLX$16LgrFy6~8?w zOL0`c9|%N)Ji*ItnCpT~eNHtkbXbAN292zxoD7^UIjOA`0dP^vsy}W(U7hlvoo7Ru zOufh>J37spz%e6n=lldgVS#OCPMD<&RU%ehKt^_?mgQ|I3}@pypMNPW!Gz3b*$<#h z1-9E@{bHm8N=CX1XCjmlB|fOToo-w(^>S>BR1hM`>}UTc?~KsfBvIweznrkGz(klJ zV$k5Y#X(CKo7ma@1YdvusZ(k!LYZO(=7sE3B2Bnc#rrj2z~rXZFmEzg9U0FR#K3e4 z^Or8g>DQYuDW*`^pE=X4vX->4AoOzP2nt^GDyg@$tzJ?F30g&>#pUK+5|P$U zW=^u?Mkm%m%{d+ir*V0GLg#(e;K63+Isg9d+L$!E}kucgw_df-lY`N;&`&KInY7q4S&lwk}-03`l;(aX~mv6X9 zw{@EA0h!_l9wWUV>VAfz{2JW7yI9*eSoecuGhx{-{KL4haLeK_lEE;5LTqqfem)yn z>XnNZEu|4cr@^r&aK^=G>_t>~)PN`p`r!hWIXnPR;v++@!A_<7i&MX+qB~MFIuIiz zsAMvDPu>F?6z>avTi@FWu%1zBR5UVql%G%mAbmRaz4X$_L%c92zIfAcA3Gz z+owLc@7Ap;iT(UIcdizR`OpOeKq<_G`R0%jGLYW*>>psi1)S9)jh2Xv)@|AdAl<8P zU#SiO$8r}kcWOWv<*n4%QW=2>YVWru^*{DcMo;#G$E6H`;8#k$hWc{_pYOA>FM&?YXG{h;r@BI1k?>syf1e~8=qC_Y7 zXvUmRosLl9yr80$cm>I4zL81`4gzY8$%eZYwL$Pzgp=A zUF-tfb(OI-{}>(y2&68**>b~=L3Xp4xKfXm&mnu^+PB_Rk& zw|`Xe`SXm$Dqbj+1TNrWN8K~?^P(X`uAe*cdrojjh>`*oa9lK!L?U0+#p+HHfN*Lz z^6;4D!{Ww&MJFj>BBAMw^*DL0^PwUreNq)E<4jtNW>4ca2!kjFl@v_?n~8dsXcY9$ zFz)Q*|2aZgu>23O1_hej&cgVk#{k3t3IxNU9-Esa!||w+dz@#>c2AwSyjf_Icf+hy zyiH|53}6{Z%X9P#uvE2vXz0YYD5nao)@9b@@Mw|Y4bsqEuCuj+Lrwmk8 zOhMknOpVodySM!tf`J{*g#`@w(t~p{cIBiA6N=xIpP%^gk;dPad>@!$0klj_k1G$* zICp>$8b{On_wUbJY&V51eB)ZNY43NN!Rcw)PBR#onw;pQixsmVTle$rQ9yuBg8ZSr z7Xd!71WE@!rGOvU{b(8y+Uz@cu(lW|ImwmkISsT|sZq0Jf(}5L^qE9EHbx3j9;G71 z$5DWUgFGsjO4F8O!K}?@6`9D46Mmxz{vmLLp%MKfn>^)i>PfOIdP$ov9P^>!dmFQ` z5WP&nHY`JcpB~!w_UHCawlpn}NxUIMrt{Rs(p^%LK03F}<@U=8ttXREIVzIAe^q0P zQixyWvH$-`I3ULYc>6#)01BvNoK_sKuUl#;JrsgG4g)C_;#ZGbJA1^gzPiV8A2g!A zB2DFi$4?&JAWrjuvV#UlwwuB&3yA<;99MKNKcA_b$4me1TOc{I^4Q01_;{gdX#-$S z40V~%_3rpA|9gL)Zap9O=2i8$|Bu8l`o|79N-pHsXs~G&ih3+k^9qWZGxdq0jLx-a44CCyyAmeLiW0chj9k(Hf{_)s~Z))Y!G@k3Ppy5{OR>0H{XW?EVgr?i9bj`($vv@TP()GeZ@ z|DW#xcTXqlsFLkMyliz>JuTg!<=M10 z(rgPQ#I65dFI4(f`1YciJQB_BGnor0te;r z53*L(4YDYp#r&VURyqUgb-VYqWCfekpJSDf>A2|n>xtv7Jj$-Y1&Ue`(DAxoLLyjn zY$QK|JVX(MNP7>R=R7*L>dV44F5JDjtYwk>Le#frXQ5@Z_4baZ3p6E2K?}wp7~y=))6pSy7)}Da`zA^apx3Y_*D*K z9GwEPpZWX~H1-p=6~AV5%s=@K$UeL-3q)EW^2zvcp-QaCi*>U!#nCKp2kuyic5qBD zx0c>U7Vk84+mRL)O6BxiNO-`0iCsaQE$LD~OBMxV8i0?>;0u`&POGAi=?vv~Ci*C8 zfd6sGc04L49W|LAF4`c9{8P+#ku`F^A8&X0b0n@Z^AAuKbfzVmVrlS6?W)hIojT=0 zt5<~30kfhg&7cBDJY0usmtxryQ6D>SOM;_8P(+g}^f+b==jJ4T$v(nv8R=smTO-|} z=?xNT5VfdLn$wkvD;R0O|B|?r8)*g6R%NFSO~LRQAmq(|urAdanng*tV$UFt+bFSD zD7My{pHE3~2L_RL5Lsqfu|t>D_*5}zy-94Cc`Xi z3I#*``7eGRns~m3hD>T}6gF4zWFuq_#oRQ71tsxIYt~%xU%r%&vKOhCIdUk$uc?4+ z1`Z5h4S#AUpX3;yM3>htwL2IQ?ZbZwg^K4jb+({?vlT|>wC9i;xj1kB<9lY0IbQew zYU$d;q0HOx6zM=1G>WE}bWpOW&`?_%)NrL-OB$`P6(OY%)z?PGX)qxaC5E=_WVcCH z2bPL(No&@Y3aiLhQ=hXADtz~&_L^(|s<-$39iGGe+|T_anG-wU^5tnD=F|LI5Py&q z7>S35h9)t<4hM-S7CaZDGpntwR|e{yJbhXR6>=R%@mS^i-c`Re4*Dp4tg%+}@Vs)R zDecbvCwbjwF^0Q-n*(~n14g1#7*vYR=x7$h4-UZ`^)=3j8>k#HK23jcs)CqpTA9!<#n^~_Fo^q z70L#X5Ox(^%C!p(9bR*EE=Q7a1!y#iPaA;!5u^!ms7>sI%&DRo%+WQ$W%?Br6$r*L zG+F#nsuTYFKLM*=9$XEr=DaOS4AScjvKkE2>oF2%5(7#xryxn$ zL7kT17XB~X23TlyE9yEg4p=kVo(guCJoUz}RA9bp)fDNY899wAqMFGiPfb`lmzs^o zWnxvEz--w9Wh~qokd}UPurM^dw=z(o@RZxuI zne*pITg3XBng*=-7hR261{wS*$qoxxzX)$S6b#grSVbG;LAVLX7AWI2{rzUp(5e1g z(yS;<*~km+n4_jbyF5BNN~t&D#zhO{gFkIG^@4%GXpr#|@j&S3abU9>0DJ zHStF8nQJX)@72qi%wlDclN0JiDpRM*Fr$ka%dkFs_8LlN^}(aih|EKWR?z&M^7qdI z=Y>5?&17Sr<@}N@0_fcn=>_Wp$Pwt%{E?B7h}ZdX6Q9*W#U7e!41M^M?$R1?BpQ^4 zlqdepAS635jLdC_x1*tkG^k>KwlnA^Rh6L zK{b=5Jf2k8+H)U;oULf50Cl>cqJqu8s0D|(4mz8-xA|L^6sBzA#V8beX)rLU*uZ%o zKuVEopFdtLvEyAAU=}1H38aPs^(a)ssZRx%45U}ungj1@GTUdCwI6pbu{XNk zp7qO3<+36dv1V~3rQXMm&7~oPWZOZq$!Uy5kRBrPaCJ@lhne7?fhITD^Tkcio+ZA@ zJ*p1lx)uaAoU2?B&Xs?$thEQ3D{Gr479NF`C_s1^<4IzN2!dQ|jU;Y&cUio^W>G%| zy*q?A0q$>j$#qh0?dCQGp8z*Qy)P)_VJ9#f^E7u6Gm@%d$A;yO zRI10lB=3*NC3RWp@L?GakbJyR5h^Jzo{5DdBt%U`34O5|6<%Wa0OHakX#h#c)o~$5 z19x|T3K5v+a{>#FbTwt0SETTQ<;HOx94clzyX0vSkaqaGPN4WeZ`Sd6F*odkz&-?g zorc$0z9PLo1{4B*OBQKXPw{pU%xi-MjeZ%P0r5AA;19T0)VPj4GpS@ z^z4wjQK^d|VqFIr3-lGP{tv-3O=Oho!UNNviLS*q(8Dxji;6j=Gg7FG1@>lOhz?xC)V$ z2!U<$iOYc&|O`I0tP z9P~K;|z!Py|Winx@tsq;M3cS*)y0i_aXBOxm#F-AmX-j+EepLu@sVpbOB#=)_o`Ooo9 z9q&UU4rvr!4ZvSOiy))|D1govdbLo_HV)%SZr`D8m8L%patI8RS_3^YTfKTZ*e^jH zf3lI=@36r9hr$%+Vex~94<#r8LlK6!i=gsQuN73Ma9ulPwe!0E*eFOKBjdjh-$$s_ zzJ0zQHsLqno`(JY0gZ4Az;VCj1=)j0^?>;1ljrAv`Cw zHDDrn2$D&sbGez#c%Q7G6)R_}UG*?QA28G|PDiYL136`)_n;-iAZ!=Mm<%DaotkQW zqad+h=jXSWglYf%*9&wd#$k6KmTs1i=vk$L#uBq-rwYKq5 zf35Z<d~c=bQ?{Zj3WaMXmTaN8Bcn`8C9t160;rpD1Z~U&sFV+izA&D zmrlQuaNLwTyDUhB8=pv}G$K5;w=ZYO@y8K4tdl;nyVPt5)E^oJJB$r)*^t@ga(a2p3l zM{^5{Xc#7(j*zunZsLmx3)Qu%9_tGKj@5BahJ|QMBC|c z>@(y(Gi@EvvI_otp1U;PwMbJ-YjU!K65`rY!E%;%vcu5tUaFi=wRD~1AKxO;Llmn} z-014+qJTI%JNi+;j~1-TfwHgRvD01;_3`Z~-{StP1_WqG0MP4Zlvyln#d4zfobN7; z+#FLkG_eu#idA^s^_qxz&8@7aGT@-f-*)UQ=W0U@0R4xI=NT;tk25=N6s6%6dW{kM zhbeOeQzD9XK-vVh-+JN@@pvyoCSEc=zOL(wM*e$$xh!w(2L}GzWWSmBo1J&`{{b9F BRz&~+ literal 37359 zcmYIw2RxQ-`~TgN3W<`eGD4DwC>e#Mj7p+mOR_UFvQi|oBr8fGl2wT8og%VBW|2)O z5&qw^=ly;D&)fUHJ&pUmuJbz2{LC?Vg}737uT&gPeGQe z8-*esL`5l^4Hl_KC7VjmN%dsK^6&ikAwlr1V3>B9EoYsE_Oa`FM>SX7w``AGch%Ei zqG#vQU`57zE@98M#kr{MwULXD?P$xr>|z#+(#wqkMCj$HId9X;DOKC}rjJwq`#Cjd zy!a`N|NWGmZnpq_AO99YcS5Uk6}?=n&c-{d8T{7bhhZjjSXdbU-n~suIzr)g^>cG` z1{M~z(h>&_+=`3iR8dtmxO$c9+_`gsZ5!|{jm~GE$x)l0-@SXclCrYArY4iEo!$NL zaNmLgsc+xD?UI$<`RC6c{(bxIXb2oPHa1R~)H`uv{ne{iGcqzfv$GG5jg9G>n5@S? z?HwJ@F2A+<(VEJtt)pXLZr(Yv$VYBtwD)YwemNz(n(ww1)z#eM;!HI)HA2q{)ouP?CI0@$;zF+?#r-E?mgo+CMGUkUMdOeZUt>^wuzqN;H)e$4K1zm zh6egV*WW%VDSKV!MiOiqd!C=z%*?`KcI2<*%a<=h!ou$04+`a}dpvpkm?l|cBjv}J zH+%&J1>~z_Wo2nt#HfRUf-VKc;@h0Bp1~UbS#LzMnM0~1Y_AqQBcpdj1TzmGpLg{@ zU*1rxcX04#@|VAVU$=AT&e@TcO_Gw59d!9E!c|j0K66)8R&ojn(Pd_4-VF>qE+@D8 zWlBolw{L4bJUqx3EiW%m&CHCdPd>qy4Ikw{%lYl1)|oS&{@eHWcjej5&(zD_kBr>F z$jCUquuxWCzggagru6O>ZhcG3)bTUfiLYMK+?ePpO}+e7OKBe&T@>0{# zRNUUcIQ8@6{^ut3u^n__m@PF=FZ3~ON{rJhcJH7gPh+_D-$zd4 zPT8cK=nV`E$VXaOT8bEd*cz}?mcz@-OQZGo9t{nR-jSA+L(V^Wg4hn<3J9Rd&C4?w zX-;HY9mA%nwpv% z@9H#quis41#N@MPJ>9O|yVtq9yHnHARlU0ShM$uMJ2%P)d*t1-{(eJ?ukW@UxiO)D zy`}5^$9V1f^+sj4H#D>)bn<0yuVNO`&Mnz}Z?zX?X0Wb2>Y#b=w{P*|Q{;RG1(1(; zaO>9Un=^G1?KzgLvvYIPt(UD0ru!?!YVy~-udlDJmt6_4?94gF!(g!d_s{%P`R2a< z{%7M2|32U@`G8wLf7-s%E9Qunu=dH!6lr~R(cAdrqtc2Bh0eJ9mjw3i&9v^3`8(;c zB7gNNFO%Q}PkiLx$h4P=_=sk2e1c?tkH-r8WYMBfg`r4mnvQ>ha=6?s`fEo78>ti2 z{Tdq1+?@S@_2n~oegV(EMQQb}|3R#pkCikKreo~kGJ4h{MCxG$=98f?Kay>!BFBWFx) zj%BB?VFmU4pFe%Ash620?UQokxl*56_Y~c|cdu;h%Nr5P4zYrj<;AD>^YGkajUOh_ zH~Lw0WNnx%n4y#T^No@1h$9UX6H^CW^11nul=PB;Y5Zs?biHBe+q7%Vnl%=kIka(c zaqaIM_CDILr&hBO&(4HNHIXkqw7tMtR5M*SFui!01q+y#Q$GA~VQ$p7tW55Tv9alq zhl=d@aQv<$(|%p@98Hg~;nL#^KVNZ?ll0!JmDXGMySqGG&;5(RgRMA(G&>H&f1Eho z$*UT5Ti%z>VDZ;@d;X2|oN`k1JJ?a%?XMq*q}jAxsA7J4V7l9N5}()@#Ctk6;6{(z zcATUaX8Pnd50l#@$Cge#d27Doc%vh~!X}xOzjnFr-Z@6Ziv7C^av0WYs;l3^RrboR z{Ml?_VbO8ZW81cE7v5U2cYXcJNl&eBYU;Ovk+-+Pk1;7NjcTgCV)pgx0}98F+k`ed zI5^~u^M{>clex)!?AWo4w{Ly#-``SDSeR-2kts<-Anj|WQFZQ|vlPdJ2M>hZ7VIxv zylC@L=I>Ow;_o(XKR-Wa2`e@f@U^Q~DF!fmB_$=fbgD*)`I$>x?>+i=x_T$`L9?yf ztNSbcFSDIE$IQ&EU+zOoj(C56e}}yu?o-LPjGB|ZHCEa+0u7AOzjFL_ai#xu(YQ13 zQCZ4AeEb-x#A7&F_?u%*2~OhE&#mq45$o2jZO)IGnz~NqnW|SDyKVdSH#mej4x^e? zLo;7oCkwS+Ea7%{P{eP}Uw7;)Tc@e1S!HNw7!edix0j#)n!ShW_tDXhxNuex!?pea z0aw(MAAZfX@%}SEFYM;#<~ZGdJk9ddsg3AdCaCmdwUMGK0%5Ona}SA|HS=FJGAgH` zp@|iEGC|A9?LU3 zA3l0y`0>#`>Wv$3`T1>F_%-g2Quy@Mt5@fZ%xHqE%YS@+Zu95ocgL^$u{WQolKtkD zkeFCjUjF>+FNv$Q%-V%6+e%AI@A~=m`M3f@a0&}&)O7BUb(g}EIAfBRp1vRLa;mq~ z+og^5&~<(Pek(e8dHI6*et(va*hHUGFH3ag-`Jd&mq+nzdU1LUMKPEo^>b@g71hMV zge8ZGWo&FLu!809C9-lIa;OLF5!yn&M|WL&yM)6=i_;?yAfXtG+>7)70SJ3 zt0Sp%sh8JVTU%eaaN+vQ;ckx=H_B0GXXnDv9jvUfbFS9{T$YtBb8>Pho`3(|L~l7N z5LSk^kH+WFVK$%`z1r*PS65q9P!#(Gw>LhxaO*=zyly>p(tW|y_c+A%JfQG;}W~25jQ$|YG53AAB9eQ?mEBjr- zhlYmM%pWlj5f=7LPTsS1Mxj&1^Yqm;B~8m;;(x>iM%hk^6(;k+-4ZkK?H zZU+aS&YuiNlm5&E{5f1w6|lp*y}jL0TEu-c_3ibQWjCxGL5X_qe|9q)ISnU-i*-n^ z{GCx!Q&U7UdiMPJ{YQ^{;^ckhfE&@s>HQSQ&fd6b6KD8G8yq}~o}$7^n}b=!g9pUL ztA2j}Xfxh%@L^b3-(X#IgtV(T8YU?AWX3nJyqQP$C<0Z%#R;GM#t{VZ3CWckG3wV!7oI_#&W!Q z^TudqsNQ6!`#k5G5{fYJ9uqrzopk!2r25cbZ4rU261JU=_V!oW()CQZ6$A9KrE1^5 zze~GQ=J9bw#ny}a_w4Z_H83(VQpD?M$={`=FX+b?aE2MF*Q$1}G%WyfMWWbkqNBUY z$;oLk^eJWx4@giz;NEUNK2bk~MypIb6VCB(;=!3rG)#QAv!c<%YojGLZx_-o$uKC- zVy9cxJ263*ryUd!5Fp;8Wo9O@@Ov_NyROqJ)P8TQlCg@s5dM zf0deQU}Ut0p!xRpX6L$whKxER0^(!K*vaExn|S(3(2pP2x{5vY_HKI-dz|Y&H4Z2N z-&mKoe?LDzaox_Y9FJ$Z>9lONDb1+_29$GexwxY$Dl~YKd2P)nmEGNC6u7n!kj}#^ zE2}qfaBeC;iVzcP>&>jJtoZ5@Pym#`F1tB%98m4!|5_P4n2M6gnF|-Ffsu3b^9O=u zm#?)wsH_*buLMN4UhqN&Wo2p7!@$`14$5%B%%>wx^HZEwUAe2YwA#cwi%%t#mEGC2 z1%MByNoH=z{+sRLQ>RXyL38Zt(gxdU{-e7jB`V5z{``5C(aRSul$4gP`ufhn+uM5$ zYH#kO`F?J!#idoGMKb}(klo%pIOuckg<6ZNN3hA+b03&I(9`C>g`SQo9KU;)n&1jx z>tkwaboc~ebcVmbi&u`P>lR+3w`@uZeFy;1_qNBq9MG!6LT}C>z()E2iq!kMy5m?T za4}8>Ke0#Ohlb1_Ml(^bCG8hi18htxC&*H3er-v~Q=Oje04!>E2+|5|Z@&=0c9@~i zWv&dUWFsFRpQW$DrHiJfdk8W_ksSc6aQgE-Je$3daZlI>$qlNwyq`aRjtGV-@vw|u zx^$_myIYsr+2;E7h6`80?k->M)XWd2BVX|kOh)67sKiKa!_!CkE#I47@N7IpX>Xpx zC=)gaIGR_v?`EU-A)LTl=g(_sJVn>-ci7C9b`kG4?jE$ZBV3FfwcZ`oJgPhXP zQ35M3q8kE9uc4qB?H9K0mSW**s(CEwuyL>EtD`{r1UD8ePUJW5ZJ@EuD^N>U8BgD_ zV+W}9K8@%nXRe?Au;zCZTYyD020i>}-)>*?4${rxu)(7SFElhlkI zN7hM!uzs|9RI2q>Jw<_@o|KzYrRyVv4Q}PgbMHHI_N*7a(#q)~c38~hajq>|IbTXj zJfpdru$Kd$QAQ@&+1Qe{Kctrv`*+mIBJL6tq!9>vSt_28kkHi-cJJQCt=qRh&&t{= z5VkQbEsgZmsv!1FIyzr}ZCs43WG>G(Z=1qB_u)Fu2!ut*xJf<|(z&-?@8;RgzWd^d zXZS-xLuV?k@JdL8o9nrDtGLzBd{qgh{}~X69{;HyOosWe?G^u4b}1)eW+MO_FQ^;@ zdlTwdad*qZtSqj%`a`&^^_w|RL*;sk+(4KD00>Gz2|76+&?#`;--RlIo$>zDC(;Bt zWZeUB=b#HMdsge4nY|Q=SCMXh<@$T#x}&4dn>V5~w6xjtK_Ct1|J`^Tcu;RIV98(J zY^{3Li^cBJGV{G_C`SQw%F4>Fe|x{BtH7BNkd^s}qu{Mux9GQSMFsQ~_b*Nn*t6%D zg2GzDxxoP$FJ8QO{l*PnT<7`o+t5*c(P+v(epK#^d!VMI)AA~gBhq*CPB}p)yYZIa z(aB%A2S!FVN=Qh&yrM`wtU|W5=*2fzX^R#oj%a9V%4Ei%&I4!?{(bhjVoN8lv^0D1 z(l1uECo+MccmM+>ot>=vi~k<**}OC8m^angUX_tO^V`4Kis7fg* z@meDS0I3oywWGVco8G_j(9L<_9ILPQu&g*a!J`*N4jgFwnZHZt5#Uz zc2w3^@M!IK_8Uw-JtkMOW#`V#Rm`!j6NhOyq?q>?&2td|Jv-j{J5sVH?d3~3LqjgK z4=;2)g%c;ZIXS(eXGBrMiN=9l@{Q3t7X`r&46Gk}9*S5A;KY(FVL{FvlDelMn-1p*Ds&&@aqN8 z&w|-TWukuo$v%DlyoqWuAfzgpZ@DeJB0eFZ53OB$%jUWGm4ye)FLf2EP5+VXvz67= z1Nh8C*Lx|Ajg7s1eO{5GCeJc5=%F`A1N!IYK*`b%Fs(gds){4Kd8bUp(&F#D(0Ocr zPV620niNO+ry75p0(7-LRG!P1FQc)UjmHZ8r|ZdESqa^c4T$11#);C-&f<`EDf#&E z-x^F$_dUpL6_g~E4Ncf!v^Dka-MgeL-SYO%yxu3rVs%22El*q5Rm)BrFsM+zmlx`|nRci5>o4S`h@pd^6Tk{;=!5OBdW7f~SZfs}kh z+vjmuSXiLd-Qs9x=h{!7mJ{;$ae%bzFa5SO9dx4dl=Px|s1;O0Oj{a1Z*p1Ch4=5@3|%+`cwsZPC4PjRTIZ)w zHfPnWqDT>os7PXDcjz~Q(hQcqTak{Ajv&m!LO0(++(DRU>Auz)BJUc!) zdHJ}M6#pHfSM|Yeagtgcuieg)ysxBT&G0;S?{gW|%}&s*-xSn&xAoJf zPaSl@^r;ZskKMd^GbT0y$PL_T)qK8sB(!ZX{>vsxkvK0pW1R5=q$uT(ngRGWfC8;_ z@}yUO{t;b|C5PI_hZ#{xTOFO^89&z4lrAqVPJg}8!^y|zdy0)weKRhhI(VmD;r@d= zlarIr@NcD1K*yoa#@(3g{?YVeGo*|VpJjWMqHZ0HhwD5zzG8tdw+?GD;t$_devT(A6l z4k@P++I`BWPeY!6_wF4OkM$BaYnz%vSv6bHUIa2#ltu|O7*y9918FU{E?%MvJJs^) zp3d#8ZILQOJMQ(}>0Sc#G!5-^02)9A9zG8GD@ee@&G0B5goft%zT2L7)z28d3z$p! z*RSjx92}%Ezj(n1gW>r0G-^{B;9t{wt*_dTE4iKtKh%xJ09jqwsEUSAnW>*Y8?j66 z52I*n_dke;xH~`Z!~?|zH9tV)$MA4zT^%hxl&rOsv~=EWA0aOK#$EVK=LbFlj&<0e zIEB~^A&Oy8eI~mL+g}@RYHVuSLQn4rPr|S<{-oHa*bs5eix>SKKHO?GAK}7(@#ak_ zkR;Dy<-IAs!7XfxgTU?Ie{A?HiO)?QM^7MK``6eP?{D9(%#M9w{3nA7X)A5ZS9)`>cvrjg2dJ3+d|WVuRm) z`&MGx$6fUd_@Y(ILVsA8m^7^`(*P-7zB&EwA==egNA@P!zf)`QP^KG|_Ztx58M@O_ z8sjr#5CQGRXA@q2{{HedHX;%+8oVh_A`Q*vYn{IBy z?!xh+&lT=ST{hDidzqG|2yy287*~A(ZtpL41wq7+cz9JJ$Wn9d+I6%uZi`&(E^cn` zQ3vO#-c{ih55vP@29 zQwfKhrgo9mjY?DwssR~}$R)df-nM5zR~;T6*6x3%X2`(6aOUF04QM(}E-toy{`g#6 zFzmUM@rDhg88kX1?{ysSd`mPqjZFWLtLoT$Kn0L5rxr%jjlO>f!}Dmy-V0#@qEMMWxT zbbh_P`ha|qgSJ~5*Ch=VCozTt@Adu~?}YaNC*e5u<-x(CBxQca#wy%*j9%S69j$cT zJK%6w(0JNrL7Tself~tDpmsx_Dm$%Jk212dZsX** z%>Gz~BAQzGmgyBA%D;n-nQQb6{Wq%?FdT98kA&k ztUh#=<15Qc+3YN6^Cx*2(CO9_f7ae!4ElUXObjh9!O7X#0IIU~>ZX>V5J(i<0s*m-g`BlX_qDMk*R+PvG*n(DaLX+bIpDYK3k2)WZcl3>)a! zC2zwTA_lESe+2z%%KO?{1qfacKZ%%0b_)D?Lf7DoLG2TPz*sOd7GhoH+R0u!W&A zp1|T@kHP2<0OcYMr_=W^t*o=t_i?^$n$z09kBvbZzp7t)YaJRya85b*?W_{;mJBGX zY?AiW*b5OyZU||nX`kHYP0g9~>J{F4?rdz!R#1KC~TbYHH8C zJZJx3z#&q^p%xR{ivE-Omu~q!a3T@5V6ecTTTPi?T;%5Gr(U&c6*D}56Kt2x@2_G; zVWMYe4}{Iti>@JOV896qCt}rgs9r=<@`=5?+H7ll%W3p$^qcp>qe9W z^*s+yB`97-^98pFG(N~Dlib2)jgw`7gJfi66cH6=mOq+Whx;~$VBDtbN^3V*8+T7f zNBj73=rGjJ z>TuBd(dmzZ(CfMk?-sC8p-$$;zNYv_&qCo{?9DxU7WFZ`$Oq`6W2# zyq4AgoNMBUsXadAg=hX8r_5;}P(s-Cmj%v2c1zlmafLe~J|wX*I4H-%VAHaV#RdHY zhI_z~WV6#FJXP)O%&WaTQ~Dbk%25h`{rUwsK6;&RQ6n?OoWZ~H{rmUiwjjxdJb4m~ z58AqQ>r;?8Xll4d;HNp>$yC4kA$k6-Wwo_5xKOCG8QIx)0t0EG8Ta+}QXt&gkG9eR zaf-DivnMOFS$}!Gj_-8rZB*{T+DPy3AH&Z%?2=Pr2gSjTtwxKb2X}_^`}Fl|21q&B zPbDFo^1=`WAUr+)^bVzgr92Gr4?t&n7M2B#8N=rDn{SM@ugcS2uHE*_FF3fG@Ccyo zcz9B1fe`LW;KN;BWJ@#W6M&3LDjTo{)Ps=dXnzp2y9!)TVL12iKRs~eB*X!nN!r-I zDWPjoXW>0w_w)AhTGh7O^vV?t6K!L+k7(N8Y@NM75}6qC732dY9tLbfcty4~>UFjb z**mj)LZO_$yD>=zEpx)WK2{pBlZ{$hTJ2xHWRKJr%A9`LJ_jo45H-Iz~_kQ+-3&GP8v_e&U&s_6Lc%vmQNPWm~?aAC3S~ z*uY(m0|JLXP72P z6uU1W&amFZ#AIKwciMADzX2Jy1yX{cvP>M1=cDGdl;;iQMC&4J3QL2Qf%pX$jh%12nlGzb4IVC>O?(5$_Y}3xYz6xoPeOosk z_qLpyoqhV^g|BL~IOm~5xt~Tsz31DCm+$QK_`3!DbAIei{nUrO=~>3}F}jY9;&O6w z87B1{Btldj#Eyoeq%6@U!nh4EBtD*JA`!6WY_98XA#!ohF`$X!rk^2m(xnAp!6MLb z4@ChOWDP=Xc6yUv^L-I8`0?Y%)6-96SZ>5-XJ=PpD-$1#6Lz0~qk}%&kGnxZidU}i zJbU)+`k$Z1P*lrGOOFkRvWs17I5(H%a2~l2?78pDG62|{5S~DI;JSlDWzyN_#5)RL z7P-X`{sO+f&HR)B2_+#=)_aPrK2p^8-fnt1mTm0pv>qNSyxHHUok*_2ePNgn4|O#K zsxeU?C?yg-H@yJ_8l10+qG*z|4vB{m0UPq?YPq~qeg(i) zPoE80XK+nKA~vcHve_M^m3nC%o`j_@lB8Z+4rSnLwkg_1tz=yECBrS(<#r@b=(o9J74NF#m zKRWDm%?lpo7-$+1R&kd={^t@^;`$12)6FMnxD%I?8 zQzv(`P&)ego2zwYXvEVfzJ?&Ssyf+A3}Ij;3JMClq@>v3CZQFz@b8mw9J_Gn@Zsze z{PBVC9tRo|l)D<9fyT-2qSuEB0!wE#;!6sAHI_U`f`P(SKoEk_Pldks@3MJSNF?Qa zdgBA+Byaora>FqB{P{Ee#yb(>mMmZQC*8rhvKeXKLv}G%oWw?nm4prw?X#Cb?u@1; z6%df5@sxwT{VqT|`1I!!vj90jjL?-GC%QJEA`>GDIBWKI{5ArVU0$NH2WM{#Puj6+Q z+<4*)dNhxWQ>N>0D?r$^ifNqWEHE1dWniuAxP7FhN zx<4K)&5px28T6Iixj<|fSd~>#J*&)(k?|dajcf7qTaAjI-jm3Phz$5Nndpw9nHRb` z^0{MFVE7)f|3M#(J3*`fHnFE_jAYmyU*9)?&6d7x$&i=doR;}>`%6d10c7CBO+K+< zSr2RK=%kJ}C^72fI|ddm{pzf(tqq8bih7MBlL=+g(zj&~qP)LG(~A$FSW|iC<>$YF zmMfa9@scF|EPufMA4BQerz?B#U>MB4*Z7w=@Rv0M0>pU4#2ya+_>lpPxgay~?d8-H z2sErGt)DP%aOA#nUn8U-BMumERDH=i_W(MF?Kf&TB>s3lg!cP0?JGMnS+E8N1UF%T z?a7U>4R8xdMs~Z5?i182dhC8RCPIO$6s}1IB43d9jTUBysW)sm2IoDC{nCqgAsaRp zmW|j8M38k_9Dmz4Fwm4{lVq*1@vpS?$pm#EsJNFe@8y?T!#D$HU^ePfJ!hV#)iV=i zIqFL1xmTp8VFv9{v3lYyOt3=9q1CE7iRtNjL)|Lb;X8mOZ>-%;y|$}v5Ta)uXF7mL z@3a#sjg!W8OFLLtY_%F&B;t0n_kasu|IxrnqD#0%QW&DOa}|Wbxp(cNP>{%N5s==4 z4pok*&%s}37ZZEd&QIh|5@Gu>@We^!-qW(Pd%*+sJeCg9u^pl)D=QQ27g^Nj20HN< zfig31OzI+L!;Uf=A!Q5YS_>%Wl7K>NPVS(m(e2SkrClCfR zgrgb?A zx{r^~>6@zmMbs(Fe;%9ho>gcc!Y98ZRXjSM_C?CYIUMIWj&U`y7ov!AVU9{V>MSAtY&IdW?L`z>qDz&OSZ7x1t(o^_&BtJ66^*O39 zCvjJ#xsE1l*cRV7=$hT^*BGjMZY4-UtFC+3+~mWB-)UF55+=61{4x{2tYcBflfumm zQye0U*rR0E&%A1n@4oGJ*~EP!=+(?`Q5hKyxRM|hn+HF|bdK1Zeb#sg3>{J_IeG}u z+PSGlJxO}_@ZrBm0)$HvcbpK4(fPBJ-_!$r65VJ1$MaxfaBbVG+50?L#$}L~RXIVO zumQqLV6GzBMD;S_7tQ^eEC28uL|X$8fc8RH2{tyiEcQn*Ndq7;!6C-dtp#%h9zj$L zKgds%c)IM8HSRSws!zB&L{}x9et>`{Vo3&OW*=g%L&3VvcLiZB@D9&2{NK{#qG92c zlu!~A6Z!Y=CzO>~@87&M4j*SNc`oP_eqSLbD7C%NzK*9bw-zV(L-yX zMn<{o&T3K7H@vi^qfo1%l8?HnqLJg+{b)(q{z&lREl8+B2GLT^2XL}&u;aw@s<-VH zP<>-mP50_T=4znExv#7#&VC<=+=im<1ulUKiMshrb}U{MY6=O(!z%TLQ(1%pZ<3hy z2jYe>()QR_2aVrR&#E6f={7jz@1_V29A~VQ>vRBUEGds zQ6`PmM7KaC`LORzMn+PTqMO{gH(KRzL8+*yEE@-)MEe2!5T}axv{2c|4MAods>@I)bI3brcqjvdE#(O=&Xh!`@w z%dbBw>E^n4Km==JrCCX`1_jgxk16mEPpTh7SmIfMFan_qoM-467}OWPC+-%o&0|9> zy8dmsc2~VEw{2W^@7_H`*Y3hT6? z_sUW3nTEi@90Nnc)lg0d3h3%$BTxVNpxxZ>gW(Q2n=Zh$Bj*Xx#5<@41q&l7P!~fS zdBQ)!#-hVU$FEv3zt9l4qb)S#qT4bKLXtbVGC#06@XsmLgy+x8m-bnH6A|Q=*DiKvZ)s_9 zny%bF{l(;o&AQdiKrdL$Te$sQXy4F-2uFp|)sU_yOL3^b2*o1Par_c+<>gg}#z;He_RaIl~GQ?_cu}iyTMXq~KZ4{vT_|W}T`iNV#h&B%;E<^?q9@iBiA)(reircFioPLaq8~~aa{QkWX zp_NaGYB3++99MzNROeWHDI@Zn<6!L$$KMk=k0K+#y!1$sWX#k41WfV~V6&Q%G4#YE zcDy~)8`)aIMoc}6iriz0o#&*%3z#WaaWYVx+F__(gb5K{nIQ8!`j8D*zRR3IXw4Nt zOp+nkkO^9K?D;r0?~W(+Cdf&=dJ}^u^#-}x<3^FlQHzR;U&Xx2V2;~B6)hsc<)~X< zAR6T5K-{Q>Zv=@kGnDW2fn<#j;9KoeedQ!Vif=J1w87^+*2%r@g))+f$)iN<4Lrh2 z(|ohRnD;qA5#;86{!97olZYf9Ufu)H09uPZJm5sM@|H?(%8E&X>ILiX1EN-)W{Yi! z=x>AL<8=^hFCxv9Q*L@5(znVk`i$MmkL;5c8q2ha>=+A${6?nBGk4zcMyOy5r^B^T7y831Hj9dlzE0VpqCzb;IGE8yq4_3PgN zVM~~^!sD5p;ftq5s!%K<0(4s#A~}^OmLL;NBLf2ZYqUr5DDn8(5;K$Qmq4fvKB|9w zICNFRq2U!A0%i&x>w&-=2)DJLKi@~-z)bM>+?+9`9h$e;qn=`qC#&?aGrL|p&kTxC z93I4gdR^7mFM$v_61WI~zRsg*9}j~cGsSVZF`;As!N*AuOoc&4#gVPiymYA}XN#jv zhnmJa+!fd0&`>oA^2e#Ief{Rm<$^Q~0b?jc7SIkHmibLtcMEuoDph|4?21fC*sY^r zW`wq70r*zi(h{~xA6v!(u9&K%Vldm`G7yZB{8LKQrpU&uW2y0bccT)(9w#G+Y(Q!E zzqy}WP)01!7c5z#7rIXKFqjH@tStTBK|%acI`BKfR^&k;supNz(Vh|aDW^qRU~U-n zYfTCA;LJt=8}7(qMq;#Zcmb8Io|1tFAZgPU@H{nHLjbFAhA+}E;H%a3Gu`ONCCu_p zPV=H#rhfJ}v0dgtz;=ay%ygPIwDc{HgdIsD6sQ>pIAmWOb)WC^IR`WaFq1GMMP2(5gM1boekPH4#o}tO#@tS=|eYjl8UVwfc!!6MB9_>Om+)x z$*EvoWAtiyW7(XsIAFQaO_|~%p$NygGyLrceqO{Lk3eTa`_o6KGXXs`IhCOP+!ASC z*P)mj5x7napBW)@O}LxGyK;o2+Ce(o!3-lH`HG5(U4g7do&1aTT}yrt}L^Qzj6&RrJW1n6z|^>Jyn5JWG;rhv=d; z=~R(;M(>NNl9GOU!Ox8lyp0qU6$OQbLy$=qpLaL=H6k`=(%4lWFO6dmH0@O2IR5PB z1A}rO8@tt%4DkCmkk)Lp#94mMPa7q^cu|eU)S`A@d|dQvu8onm&WKs6vxOTPP2fa& za+J%>Rtsn#)sV2ifK0dlnHd_jc#~YM<9arDv=9?0`ICzmAH~GfiCeUnKt2Bi*@6|4 z)e)QiwdayGw4;$IK+VYQ(StSWxG-zFPf=B=4YAcqaqI3{NHEdavB(H40x@>*zbrl| z7Nxl4k@m~^rCgjo%!%68)<>ACU}9sd1=iP{%mF|^IgG37Vax82762Pl(b5V>HIY8w zWw{4o&{g{3`}NY6)R*LoSFc{(4zkrL^O{Ofw;%#*NXoyauI?wq)-gKt8|NPSPnwP$ zFE!}2D8&K>;VbmHZCkfyK)5_0^Q0bATZNJ$V&akf`0TvgTwydACJv5I@SPa_6a;yA zZm-gh7Pox)CN)td$xJ{`Pmg! r!UVz3GdMbfdE&k;W`8vIV!M`d|3z64I$hr&z zRBdstP;jO+B-OmvUVJ$HZ_55rB5SZmXAJ@}SC)Sl z(@++He}+itl?=1)Cqc}O8x9?w z`T3LiNqxHLpBK_(-5_U5eneO>)n;|e;|+2k3D~k3pTyzn)}g(Q1bj8OIPe+869o<~ zgk*Z)4$;JJg@iqfFQZN^DYLM%r)Cxu=`}`)G^=SuEr^4x4z|qfA8}-IX-sq0Z{}0b*igc(|6^{uWgY6u5dymT65+6-fNURr(l& zF@P3FOSjK#Vr|?)l5dWfMr>+aje=ZU$SQpXzxdC;o4PifSN;6hsyIqS?ECigq9-TM z$ue5L|M0<((vCpS7x->3TYT{a-h6!7p-ddX)rc^Efi4yq7^qsvIM|_XC)4zZhi;#A z&ZT$7QK$wo$zLBIwnOWM7GZVKE;#=e5k#qu8XaQ&4Ckt>`{s-)+77GKva=Kisg#l$ z#-Ak`@muH;`<^n8S97pH`xg0>WSU0-d*YWEu(Gw!?!flPk-?B2^=T z>61BI$c-K?+?H`@#Na)w1`9C7u5}NfFtr__$xBMQn+g`r+HZkbJu=nnh!6r_6|I=d z_RtpyL6K=6Qqf0U^Q@jdTYmAPPI5sY>>l(vp6Q>@{x1s<9LRR~0p^H!>8aa+-M*ly ze9At{zg1O|QzKY(|Gs_35KW9x&}+N83e@t^ZA>es;u&4%ifCzRqi5YOJvc8hc(TQ} z`8Ne-!vv$zwNLf&d!G2^0C7=y&J8!cI9beS`G+`RpIxLlf&)3E-|6_e9~=^OQ}1tx zs@6P*fulr@CxO}LHhV8Z#>q9Vlb-1CUg%f9-13&MZfIa&rb6MkTc%-ug^f#6Z_c%75e=_sdBL3Ucl=$F zf>q)5#_(R6Rm|IG2AH5GqCc?MEgXe4=!g)=fDJ{v-F|<)dfwwd7JmDJpbr>Ma`JFf z(RZdT{*`k=_4W1Zr|M00j>UImgGVOmci>Mm-zyCEy&J>e=sc`%n= z`$Tr<%5RSqUw~?PJ-y->F3WC{OV+!CR?-g!rzVc2i>OM=u$m*Xw4MShv+criR<({f zYBf?xmKm#QO!ZK`%+>FtkBpc-mU3pmEG+fr&DL}E!6H7;6Y#hYP{0o`XowGT@P|Jl zW`NN%)RVV#lr;4acq)(*5vN1t*q*Pzm3*Uk#IofCTxduRb47qw6byq&YxSf;a-~6R zT3B4%r>IBid>2J_xEW zFMqDKPA2SBf=%@TVCdhuHa(E;qX=2Py@?P4iHee0;C`iDOIR zdZ@2+Imvd8Jz#YB-T|EL3-9brFIQjKkMF9es3<{*(Z$6Dv!i9O?U8p30Qs=p*^8z^ zd^JoD8=xMubP{bb!?IHx-iKH90C77JKe&)-ct`OFh7zAKBO?$%LSiq}V#)x{)21)s zIc{NQz6-RGeXUb^G&S6F|x9}SI+kS$q*Zh7d?A+cO53@5*F=T%)&$?C)P!CW9& z-b*xregBkJ{Qv{pXN0VW4CXg#W7setK8b-2)3%7HnMl~}fwE&{ViHUw=h7@!&D~`0FWrdlbASr;X=}ML2fQHkSa_;+6gTP z8ho?B?KlD2&#oYTf)P{56Y^SG+wocf93BUT4Mb9Fw2k{-T@ zkxi>1?CJ-=@PGR59SejR#CH&s_QXlk7HU%}V|gcxB+{C=Gt_(c?j=$~W8;IxqCq<% zJYd!f_5+6O#$BfGW<5sQ7u_O#+T*zl6gjX(?EX7xX+jv)-NMSc$xq=~wt_LZ%%MLk z?fJiRV<%TU{=U^p|1`8l@`df)Pdr>a9}IX3v_!Aa*+tb2e?$0p9hOnGaEjC}D2q_9PzGK~mtiUcI9kOIJe1MEk;ME}* zR3W1kT;##!84>3Y#n_;`zAmg!jxhA{e$M8BL#T88;RvkL?VRYHC41 z!lvkdvqT+R+Kv=T)WL`(ur$qA7mi{6121|xfhi6#ydA+xW>SLvOiRR2TQ=qs<1sR2 z>)^2a{P`Be_3DMWj_W8<5Dece^VYekbu?+|xgEzmB>Po|i6N7v*O{4|dU`!=YCn!D zXk5H_5>`i+{cxk>-Jg+zQZFDDis2m@>SxaqcccTl;qUw$C+otO1&J`Hk1 z&)p=3T4WCaZUI``HRuLe`lYMM%HdwEEV(#>Pawf4<}!QjQtoxmeR@TI!;H$X_3zdX zHNm4{(cX`+!vo0O1na1tYE;Agn%qs9Cq(^jV`nMnHj`S17bR6f&a=TwW0Jc?Od`Vn z;2@HLCcHRIi#r7buO=v{)XG(4^(IHcYfM(E9Xp3?u|(fADSJv zNTePrYa`u@j~|c*g{Fal7BVl7+3B6Ig-IOg-GM#G6QgHipy!)7NgPH+$wBPLv0(W6 zG9n1#4U$_s5zHomQcOBP@h34=jKE>0>^TtZ5nWha(C&Ru?J!nLEETJB8X6A`RpXy0 zCi_GGZCX$}l2TI2 zvrLzXj%a8s zgsE;D#DfJX|1Z(+zOr<~)VIluEE=Nxq>I+ps&w=Tvp)oydH$Rm2EPq*X76iiP9BMW z%d&+G#vrXlayVp);#<7C;N3@$m@v2UQRi+{WTedozoMdLR#o~^do{2Z z*S&x5i5Eg)seq7j<16t-36fdN$?-v@&Pth5)C~)0Zrgdy<-kt?#2oR9xaBodg2$3m zQ!CLg$ndFBz@D3t$Vrir4VW_|&4|S7$G!DpyBPeYD!1$P=1&%3zMXXnKGebQ2K!HI zc$zG&V7kzcBzdcMdOYmtb@6fTJ}tX+8)b-$tzp`d>PdaLqf#mX7Ld&mTi=GD%~az> z;fS;pm{!mR@n!>p6(Oio!I(!d-0sHEXmJz*;_dCZ<$MQ*>Yw;xn96%cYkV9&1xZOV z^M^NBg+E644~EZkgrSil|0X0j$`~4Sz=*!-FlLJfKz%Sf^5DTOh$|$%0^AUC|Nb$^ ziv}3;hD3olN*MwwI82RnI&!YO-07I%^1&j~Z^1MInfNh47pjvyfm9_p{t8KoL!gW} zWMi08(b@umKp#AsnwGW_#-9ilB2i|x#c$)(m|7zozYM&M1R<(^|5Nh>)BMyr293O*LP+dbJ;)K&Eez{+Gzm!0-X-EqDbQ;+@zJPt(&kLsqhSLK55g zk^!s|ba>5z_6^TpW~#t2e0RRo4l$FQFjvU9vdGw zu(JB}&Rm4u2iJ(%sV#_t<83`wnhFR|o12@{uuEWF;%G~+dGJK1?+Xla61Zs#WHUXqB8qCtY5vcO}mXLWHn8EjmE}(x$M@PEK zHSD`6>GQ}KysO|slK3YORwW`lnKlE8Bx{u$guyu!O$E#j;p@+yKfex?4?IU7G>Qx` zpgh85xTH;0t|X5qMJDVqn2e?7M79d;k??DzTHm(pkHzGtKV*Bgn1Cz^?6!KDMH(`$ zk5nk(<9JiR$qoaDYnTo1V&H^!S&kkJIGph)Hq9A(49|fW$&iR5Fg}lFOzY~p_=u`h zn3fdmwzlnf;oNmwTQX0Fd0qbTjCkaPnIS=XUl6P!?==8y^~CaUtzNSx^J*OnQt2e+ zfIOe+Z)PkRW;m8UC=CAbXY2iP=aYY`NWF(OUbph~VoFkylj)6@3#&3oq7BU$M{v5s z$=~1K=GWLo67~!i(ji<5cng_TD)b|i=C>m{T0X=4Bu|D62j<|paW#03sBB(XGqhc7 zz}mV*V|{&31ZJKhVA<8SF!?1wixwMiGYM$`yR8AhBXJI39ru)~_eIIrLHfArd4N9b zd0&|633Vo02#OL` zT~Fa?_`^6^Pk@%Qt7|9>1f;z}@V>i>cU0I}S1c`6;bo?4uTUs8A3iWpUgzW_0ym%2 z(TP#`10{GCLsFmd9zRS=1|gJVigWU9bW|Co50S~?2k-%ZK=~2F$SA?Nlq(h%DyTao z?WcUCKyA-~0}o(2DJd%YAjnkH))pQs<0gP-uE@ji8KS-^bR`~cZoGKzY8Fy%7=E)# zaquz!aGd3DOXHQPXe_)M+?nT^idqns?T^a zK|H&r@59HBRS<0Cq!2VKiE$@C1uj$H%B%;VZuc=QiFIO;xp@epsnU^nO>QRtJve1J z3#udr2*aug+lJg?x~j&=+lAu{i?hS>MD8Wdkr<{+gbqhSmRiE8)wpy?V0`3&iZ2%) zCEUU+Y}Eq?5ALRbf2-l!63`?~ecOgXTJGYNdCC~dj(_^}D6$dHu)LvX_#Zl68|T7H z0gC?qR6Zaf!AJQvHl{+Jc1vq(j^kPD*d(RL`*?j&e^l^yfpSd&679#@MRFSXn(!*B z!?3*{!ctYnd_sI`sw!%G0?>x3Z|j~eylD)rLJ5i=(lhd|zsBM~rN}xQRaK=0A`Kuf zEpqE!zjm$Ckynm(-`?M+O35lVOVNvbx9p0dCh0iFtgObIxC3J;!!U++16}q{JAIp& zP(xQXZMalZa0vmyB}gLV*MNn^#Kf*48Ah&8Kv3{o(CAp|J4hP#aGMt(8b;_WGgykTCAZFUg(}CGlV-u6N$M37~A@!gH(8G(= zINtWjE%W3wIW34sJgUuUfT4Zu%WLEKSFcn?Mn=9349Fw)AG0iynk|bv}H6LY_n?$JLMOJ4~ zN>w9-y{=rjk_0^~3t$$n$GGo!Ek+Fi@Fmp!ZzCh8fOhe0YO1Pys5Wz`Ac8lkA#n~Z zAFnoOLhSwr-rmQ(Z{NPOZ4+=e6A^RHYI9Ij@uzB%IDxmspmcZ>um`~%1^o`1KY3XQ z^1gVD=_6VicV)jyLrh2YfoYO-FWMo}GoB(wRVC%+Yv7p}`^RTLfE!ENJ(fukgLtUi zgmYuTF!!u{L~3va+F2hOtQYtO2J`)at)M=wCp{c#pRDrGdx&wah2SW{&5Rl0*7y_~jO$2R*;~|=o zTTIbP=K@SY0X&NNNpMoHlcE2wtuv3xd5zxwV;+j5OdUj0W(gr>Rw^>j8fZX?M2U={ z5urjUNm0fmqKRlw<|su8QJIn{am>HZcFw!r-&*f$oj;DD=lKrzz3;uR>)O}un!zJO z`@LYm*2DyqhvElGXDw--5AF&CDP50`_MkICKMNd9fYvCL6>QwyoQw7^O2bnI9ig*0?=OUbC?5#I}pZh%Fy91X@o0EZHS3oq-rJUa0ZeY_xV*{k0WUceuIaINrn?3}`rxbPA zew7CqBZEv%PCjk_VaUc`U%QbZ_Orj*!a2;Aj8-(GoKVtej{Cras@*mvb7szLBZxm6 zKNA31czReo3H-<#<@}bH%6hf5ojPJh8ata__NVXOJt<2t@-^3LeeV zpeP8mjO*8%O0N~&wn0HttA0m4-u_8-PupP|2M;Ixr~L!t!3OVpKRh{Ut{85ih@o?8 z+I5vxeGrTbz(@Lk0BhKQqnuWPSe$AE?wG3)<6Dz7+lE~K-PmcGy~-$rdeBM7v$uO zEs1h(O42?%(Ltqor{UWX|BIw+7wm=mZ1c7UX+EZy;{)8UeVkVb`%msQ%F0Tet&sX7 z)dTAUeQ0*sy*DoMsBCS7R3 zx_oxecFHtK?*Bub!6lsU9>02X0C%#Qs3Q$bY-=$!;2m?3mSdgm?Mdhw>EMRtRx$J1 zM1mQ4Q-Fwh31PzcNIEW-Gj!eLpKMpN`P+vv(g@5G5u-OSN8+0_Fw-I?-M0;!cW8`^ z;@Fp{>c%~L_7t%x#=b5mM&(|)5)~WUko&}6wma@H(r|+Wny+gx^wWoE<6$ z>Dfn?dCqS4vX~taNA+SM0!TE=&;RFz!1GjS^y;NV442B^0?E2CJOBX*)yAb=inzxj zLxlR264!LnBomzdz|On`-eHXm7!d5_d=F5SiXdCG9|MP^Pox_Qd`<{gevT8i4w>v~ zgbcF_mN(iH7Ixjr^Zv%xjq588Xdk5<7{g-~r$v&pD4r!e-0BmLCmDI6f?{pc&D4bb zC&2@JnAC+u5k=X~f|Cl5CUb;0^+Xt#Q*<#3YcOf=m9CS1^qo8UX1VptpV;cK)rM)5 z)Z!I|nh$M{7ZSL{F^gxlAGAbzlxj-Kc~Yc4TUZe8R!V2B;ugVfzYxKs4hpz)r^cy#Ilqg!#`|PIwA=-XT+_@5s~R%pSL%l#3pSA zYn^i9%B&GzKGVpU(!)O@BEpeJ7qR$w`YF@iZ3@uKa?JVhzQkR%@5IwIUmx6CZyLm ze1IFeteiG;W-LFO4{-1b2c0Og3F3;X#iE1Fe$aGT|KOPTPSeP~&{S!0dH|pS-%ehJ3-n9=ojV%DfG|{z!`A(Cy14duTwIJ4F37)3(b*80C4Z#5 zq(?&NlJT!+&X{2=br*_jPy|)1Z|+Khr=fAs_}!t{p^YrgFM0aR&E5U^=lhp(6uWlp zn0Tl*Q>nl<>G0z?s|_a8aB(4U>(Qm;=$rqRlmV_V02XClTG2F9Q@_Nl^%jXgKV_@K z$qfXx7DPkn3-BXqUa<(=a2ta~Zgxv9L14wtAE0JYBu=0d=+j0|o7R%5NEG>DVeLYO zI5{|&mReQTd~KF`o2qO@y;GixZf&hy6<_uGwTA2VjQ^trpl8X4g$2-d2pA72MtHc9 z8ZfRgMjNlR=@@nn=0=c51)= zu3;dDA=}UFX2+?Q7A#yiNbSNAf;JK65W0cMvu62%yYL3(Hb~p)*Bj09v~|&Vf6{yU zgb4xPQ!ZSvj83uY-YOdYY2d(tDB4X(tq{49nRRZ}l%*r)3zCM8z}BBKZ1 zrJhn`k=w*x3=yj z-7+x9<2pRqpV|kXP_fMa@+B9I>=bMl&~(_5J<}hQfeNqW&5+e|l&%m*Xnoy5YVQ9t z-s!Hx#ETE3f9$B>FqMm=tonm}xRmUf+&!Cn|G;OB&5Z%Mwjpfc#~(b}ZCT9i`?V+f zkSpudS-l({_*uh#Y;9p_Av87x^SM+ut~o-q}d;o(o29$%M zaWd>2FNJK3?%QnbNqwEs1w=$EjpNOMu87Xv626k!iH=H7{NPBsBa)6APMO7n^rj(| z*C`YNVj`U8O~@~ieDkv9ebZsLX`~`*vLA|udqqX#v|8om+1K@4+v(nkOOBHx?>H=8 zjA`JWhtO=YsOa9+nR5Jh`UamKV^hHZwi1au)1m}i4aqS@+*s&}*(CCQ9zf|YK9%F^ z5QiHJAO5ky`!~^pXj8%tHBfDJ2QqR2Q;Ul`)6%TarT47E@O5gELN2@+nxP&Z9Hg+fQ4A_?vJR*oX}=G;FD0l}4kNxJw{E43nh0v^U(LV)BX= zXA;MF9Q*cmN5INiwI4ooR!GU~9Erjc>lVE=IvnS9N0Qr>rYx|xH`QpNr|%B3Uf)yv zt=7|%U*D6Rq`iYktz8EVeR7+^55~1tK^zi25-tr{X>rEBNu7h7u9MNytUge%GfrGL-h2S?v)y5{YVMh-kpN_{<0 z?-UCfMjQ@B^-+te!Y3Jdw{Gou5P$4g@yoB`xl&eaFunFgn{8J>{c*kven7K`LP77c zgUf*zX{L3{=DxgQyZr3yZY>(01`WjqVdVJn+j_0N6*f?zaW}ene`=k}vkj)R=Aqbsc~TB#VV*?<7=wCOev&G1Tlmueo6y^N_#<;%V@|Qly?ek^ zrfT^3NRrrY`^CDK*LS?AA8x~@IvRMOu(s`4q{^P`IW|mZIM9-kxh9U>IlTs1&{}%xolhCbWeH)#JZF6U+>$bZ9IZ8 zWFjc(GI#?7M4f7vb_2-*6qjiX()tY4R1TwzhjJ$5vepJ%D7~p&{MUmFNMX);zwf)A z#0qxJNq0v*Mf&ewYdKU*PfmRl2FOBfEKU2OIN}&9ZQ#0KLFM>P_wS`TE?%rTWXRu_ z&yLE$30`hy^-$|Amny6Dz6`l@|9&o4t+nIUty@uxatN0aPpOVs)4ciMZ2d9_@Jz}t-pR`qEtL}6AN!lG32b-4FAS&YU}^jiMoyK9E&QfRgA^ z*+#s2MTHYX&?hsAhVKZZ%O^sA`RHe*wRY2K1JwQ;Z^V+ovO>Gem4G13mk{7|F)YF)f~_-yZi%&EZbP;CWxLNurbw){4;xRsI4Xmbr zZuwpK-UlUf@(TwRO|f~rfmBr=b_>5KD%pnQViFL*S%H6&6SM5pVw*s1yuWSuZlso~ zKqhcRd)-a@*ph6Kw2%#(8Sb9>WzhZV8L5jeZd&Bz+uP#2GA}uYS~Yh<;`>3w>r8?v z<@{EboI0~ob|j8@{rc_cem!qmY`WDUM>k@gxEpuu*qrFe86+|Yb@xHKKChY*U^>Bs zimelPvWHV}t8bXR5IDp*DLW_Uj8@E+`Tk97ifjh>_A`G`h>i9B|ExNtfENT4B8ikY zzj=U7>UK&CF;`yw<-2b|d*Sd*}loHezh+#C@tl@=_5?sazvh0QT71vt6Up*AQYgEsK z3&~x_+dbr53tN|d{TbiAtQNIbYFs zN!J~h*{%AHT5JLA*Jb*SKR}0TbnWV6(Ue4YT1nKoeFOlJ&{9wJ5*ujIx^V*KH{a&? zvN_K}CypG8$qLdZuQM-QnHmpuuBkGGckJ3Ucs@H*9OTtLR zVkL!6+~OybpNOR8WU-jFOD2i%N4(=HP3N(M{RJV9I z9VVPAR~d1qI7rCdA__aK-lk@8?otF_5FSkz^Fl}82RdN$l7|Xi z30F0nb_Ce+Fdyxn$~6X%q2%;ggb+YB5UEBOv&|>Z6=9tm2|ps?VsUKfKzb|YHPT=t ziyVxT?qhKz+bvXM8&|NEh1_7^^;YN3Xb8N(S;#skgds})dKJmKfszYZ`}$QpP%-uYAD$t z?Zcx{9tsnQlsYoNuzDSNly09XwDdw;{aFX~RGNB|;(5;Qz4w9ST#7%EnkC~(zinK< zc=2Mf-=v{aY@?8EjHD}MeZ57YT~WA=UcFkM9f!i(#C7I)zA1>&7 z9UfSkN@YeEOEs6rD+b^~iYbquuJ(MwpKBEIw2ZLPIvJ%hY#~}`QmQ~kIy#Klm%nmcc`E))* z4`fhrsS3TreB;dJFuYR>qX7KqYuZR?)TUMzBFY4h5&`f z21G9i-7y}e9N8kQzc?&my-2<%1uj>?MQLhlSH|oNK{+0omrF|r07Yt>ZY`<41jPhD zWxdEm1$tjKX`E}{6KjdT=a2sArO_iG2a5nUD6$Wu5jO)iqzcIACAauOBs&WOfNh z^TJ-Q`!759?eO9BjT^-8lAEx$*6Fvm;wx;X0Qh}>Ih}a-e4y)dbI}Tj{WT(Tw>X3n+5@_33-}xz zZF}eiRrDD2l3&j9NZ9wMX*yyjCrt}zM~#_$Ser+LsuGsNKik-ayffQQP>vHPlt3|Q zDjo@E_G~Q#4VH!h509KWb^Gn@O9$C?SLLuM{_aiR@(HkAAvp$mI2}N1?w&Vqn5~&Q!rL@6HySBvvcM18d58h3R(XX$Oga&y>7_idfB%gs| zf$~hyQHo)9|~$X>`mn&`JCRpl?AGE}zXaRbWZw zZL0Omc`;@+7*LYWi7QIlhkr3FAM$4s6aV%awLms06%#Ei+M~h?HvNT22t&}T>lZjJ zFf(EAh)Zi<&S&wvi*j2llv;w=VQ55pIDS76wAOXqfdgfB<(G5B zS&8@i_RTqxl3ab17)@N&4BU+*>kxDw&ksw_a#dAYJit#bd@wKqYP~Ew?XF3ur*t^1 zsao`0Q`0{?2J3&8ajEgQns>h9gfiNiCz_l*TXPByzs-uGUYSZRi>(7__}iN3;a`vc z9(wHe9E8uBbEX02I|SHTy*5Zk=+YLJek`F-+;ku{&lY8#@|IW~$dIkJVp$~VEX~3a zcUav4Iu6~yMKFvhLrA)G&l&I0ok4;m_56?eB7}2k3I*RqP)W-$PAAF$lN)~8ZUU&y zN%2(mItncyUoZOne{8qFphcCM5Huuqsv?WEWUe*8g zyXfztn6pkzkhKE;s$hHp4QslvaS9B-_KM6#WaMH2 zOdpV#M&cFV;^yXsx AYTSm$Gg$MKS>t+}w&n4$`?w`k8nISEsy;T4JJJu4&80HD zZvkU*k_2xktlfX z|BN9f2&oItD_Izj0eFTjnkThkYxEcx*bF4O=wkFfScMk69QZDo6DxsQ+y$tR=QAXh zQ8_2G^}~0ZV*Mil#K;^%ObmdDRlNGWyLFWddJ6Rpv)B%{D4T33!J}p&ER309@7VG` zaiSrxmS{zb?5HIv`4cdbz+Fs^JVLzWHAvKubZsNcT$Q3TeyJJOIe^}xTcj)`$Bja@ zkKFCg*xLRJ#`%T~0sPe%K0N*IVWaWmm5BEzS5hs`G&T|w!VS#c5G9G|Az?fF{b9Uv z+=l*nSo4WMuxq~cXB5c%;=m_42p+s~MUvLVz+kTJ=Q!ex z!eQ5L8T-R#jI&bru?cNREE3Pr{aAHDz03w-7k47gPg6s^*qT4@e%k*`Biw7n% zhc3UU`M_eb#e9$A(Tj@OVDO*EEwd&7GGrOcOj?$q%%1zB$%O5cknR|BGbAjir=rclzk z|Gx>g$t-3cFfNeRb04cj_kOk4UV+)VU?(xC@gAE{cMbq)_~EY`(jD4YJk%F0YBXum zBvE%ceg0;xTwpG%eaA_n1{+AiGGZ_B9d-1*7b8hH%d?KmKG5vd>%=2F3JNbm0O%W@ zst~A$+;?uW2|d+?WsL&Sm>2*=vmI)HuxxjIcMs{2CMTnAFIagRK*iuV>hbOBhHWTL zBmh{nzR&=QR{TdPu^v|M&+&vB@J4vie5rKk4#_+(qZkBG`Sgb_RA4?R6 zAr$bABvHzJ@wLO1B6nf1{o%?T+Y`IYSBveenTZ*Xm}7GqxP@xbjB?KE{vwZpkAC9( zZJUtX`q1Ky)EUVTN2JKILZVYs=azKtr(EDqyGI$oqqND2LJ!xdr;It4foTE{5*`H+ zLr}TMtZA1SCl48BxsAlfZ7h-oBqGI%D}za?eaKDC47bmWii?wuJp6F$@I*6cX3(nM zrRMWW3#!MZs;CcmNjY0Oi^^HZm8V(0<>32SNO25qHtXl((NXf~Eh1i&@zx+Hf2v11 zi%ikaxRhP4BTWjAA3v@xNxhM4Oymvu&G**_I8`^v!MZfS4w3e3WR05gUA)1ugd*pRodo%c*tCku83WbWBnbD}}0(NrJLtj)9 zdU0%MtfYh?Hva{Ou0Khs+il{0Pr7ZT2qPHNic91eA6-?*8JC<(x1zL(ak(N^`@H^h z@uK)X?^as(K63o{xMyET9%r9h+?v5L4S2V7JW4M;?6t?MA;H1H{V>cW{g;YBY}J*N zoK~2}WuUY{Dw9c0TXN14;Qc&IS_gd>1u&r)XVQ_8;8@UA+FWR8l7n7+BRBruV7{-v zw{{Qn7bDrtYRCqtx{Y}hPD%ZRn?m?;knK4YU;WDG&+|ETn<%`!yi7Zsh>jjj7zjAf z4TBlQ9c*34qtJ>Wzom^LQq<)16+9|i%~q>B5=}uXRn^fvrB*5`qtNt{`dV99Pb&;Er$MS3!|Z?kDlAPPjy3-*O?;#f!sdnc#yge5mBY9yS069s9|Ig5R$-Svpd4Zj1^J$z%dfNHJT=e)3i+tA5yo}0djpQQ2Y=a(lT zIYl%5uM=JxE7V5-lsur3q`$5%)8@6WBSZQ8u(()7(e$ssLiB%eq}0b!;mzajInSY> zw0N@^usNjdy`j6^4Ss$qQ~j1cNsmAfX=Y;LtJuXyVZB;FJbuOcoo;2-r+V_=u>-H1 zJ~dCdU=ucL3Qiq{(m5pcr)1el9JYO$=}k*hS?bzHtS*+r;*TEK_}+6!5_faH;~AM} zz;I`>%|2VdPY?}Cdx+<%U*nV{(Njyld1*R-hQF#^UgzI6DC0+3n;@MAvTB2BpQ8&m+ z>^Ug(fQ^B?5&ZXX4&bQ^JVaI|-g>e@6*3fFETY7E=A#GQ-tKQ6Yy#Qs0`|xgR=pH| z^r*w9c|i9S`fDYGG|_sZWTv6xqJ%=^Gx_f7BQ_ zfub(S!UM7|040M!4CIodfnq_4>Ah}BVvaam9>N2p0K%A3h?p5#`Z&EV>Ia3d07 zDX$TirtQ8~{y{|DoJI1S*(eVnBxItGw3ikhpP`zYz8&d`Gfvtxj6Y@VnDB)RqmY#3 zq-Bcgz0KzqJk+$A_Adp^&4Qb@)9S@CQQVndjdjGGmtO&H>I@w}yJQ+CEsDvKbLJES z=G4A0W-2XPZULK=Dg?y^JDJduAp;i}Bf&`e`lOhf$UYOtEL1yYYjsI2wq8O~QX>@% z8;(D^PFyeof>cP07y=s@ziul-@obfNY9p8({+_L#k4Fp&wOd^SMIxq;#d41^`gO4=T@1~t;W$WD{* zAOqS?c0C4E0ajHUL#2R3Tqp3QK&6ntoPU2Bc7TaFd$K&wkv96;;^I!CFCaZ7=#r!r zn0IfdR%^n=Q~5qfTh-UEF(BPuedqsQr#)zu9ot+iOGFtCtqxJ5fl!zb;I!PbZ?AU~ zp!8bP3YLZ7NF=x<32X|<(y1q9Vp@vM{8}LU_c%g}-iT(@vE<*+^CJJ^Epp#g0d|Xb zaLr6eKV&tpi*nWlOAt6bKvSx@Bf%<3d_wk1Zyb>O6$}LtCI4PHA?dcR)!yAEH3O7u zHls-nmIXIq7-!0OG5zF=gQ1Wdot!uvTK>e>Nvq8^?HvsLjm1DfHW=ft3Sjq>pKzex zi8kJLl~^c&jAEZ&!!eQTKmT1vLs3yyHWH4YG%s&ifX57vcvt9;h70KS4gS8KXC5~@ zVKbSGbQw|bIl#}lQGLEugU@-J0P5K4@4wABj}CK!{Y%=8CR9(@acm$@WEY}?J{IRs zVo0Fob2Kq=H=FDwx0gtH)s7mE5h?-S_rtYlqko)>k)(9MP`(wn%1|y?(QjyU8@CRN z)M9V3Or%u-crqKZP@_xM8i)SBFM-;Z0mPHLS*N)VwhKK)8;=6`CYkb-4+&)G5v@M_ z^7NC}_Ai2Rna@GLPn|vr097K=g$Zx}WM~J;VgBrU!dFrJyuQ^gq0KXy(Sx40lR! ze!1myRy@6Qbxq*JBP>!oOlI!ho#JF!9CS@fLEuR|e;)1sC|pv0{`^V6yHi}u*_eCg zi>qqjjYF+l2ewAdwO$6B4*gQ<3isJI{MO9sAp3nxEhF*ihz-Oo{GTTto37a|0Cu0@ z8$}GjeGNTKfaOXU$2wBO^L~|fi^lWS8v5Aa->p+pG1We1UwC+zjyqchjNKO&X1iiV z(Cpp|tEnKC(K+3wcJRm5LrL0CUHm-P$!XcLBWKSBb7L!)ww<&7Q(@o0!ZI?~a=ap9 z$a!gm$bb0IqB^Kz-XQW%yPd;w{u=S)G7jfisBw}Vw0lHM958U8@z9Qo_!{h^D=H2T zOxZ?Lk!`=Kw9kU#*dq*{>2hp)v#iTiwYF+LD{ofS_Iz-4#tfa1wheYEec7X-x-)lQ zWMtB9*R)667r^Rnfh)~M8QTx!aofq81XWA6)WIOis^f-#Y<_;b+3o`g;8~7Hs>C%OlY;cl>Z7Gj$Ts`~Iy7%I@kq-f1!gFHq05Uf zt4yc}EqIk(x#FpN+H!-4dxeF4s@9&i#;s}V70p=3pj!)Z6U4YF|COHi=!|bqtb2Qr z;_r1cxL7e{>7ek|(+4jtu@3-t+y3NT?u!}~<%V$=>-zawZr;50>f%;AbBn=J=ze&? zg0H|VppruMezylF?xA(hlA?@Gt-saO=F^Ff?0II|$&Ff(dGqvSdZc+l;(4s27!(t7 zqtB4@?@tU@qQCH&o58X8_#+7kDrt{)+*+9R)I}JQ8jL>ubbY$^(UeEdtYG0ojvaHnR^Iaqe-M|y z_3F5T2Q|;MA;%3vD<6p+jvo@vk9WrW^?{UP#lxjZGt%@@F8}k7zQDPi*1fM&US9s` z%a^lG_jR08D)%l9Iqy6@bNrRe89Tgtwf1biS<$P$)iI@6OLd0253y4TTse|;+Xq#T zc>Ymnd7}2)(%w0FQ@6-R?)0T1jP`2Upf%$Onr;-LEo!vsT2{wG2yaij##UUxL zD{DODdXN>-ys`lAF?PzvjU$k(mSWA8Y?38VsSKINXj*U( zTk6MBw}J^?SvJlWYiDw`@4x>n)LD52CwHK=%^-n+rdpX4_}OpX;3qeXfA}`#MaRsi zr&DdV`T70WNFXXY`|6dW3>~M2X7n18p$I127L=5XgO8>BEC)Y+&H<9EyM1wp#lTl? zVPFRzY5yIduYa|yOam~!?tD6;0B9my^v(4<%)HuCMJ2_6kFMCrVn!p&wPc3vnly1@ zT_QWr8%VscWR<%+^=0z&FsI{|Ukh=YlLo=8tSsff{;J_w^cy-9&*=IJoI6D#GSjcB zsw%7e!}OHJDk^~kiBv{9OBtiUwajh#<$A@%{5`4Z=?7>{!$aZRx%203)f#TN)M;n# z78|u0)jGpRjF`lz_0(a`M`ahDcJ)tQ%=I=-cJ;qPT?8t3hzaKhuh);0JpJyWn)t!6yyt1BNu68eXGa#Pn zcW8cMM0Vyv*F8Grtr=Z~T6<1X!7DwCM2<}EK0BZ;|6jahFS~6}xb{BtAXBY2D~H9$ z#}mrtWE{t8 Date: Tue, 12 Dec 2023 16:21:49 +0000 Subject: [PATCH 18/60] Add: initial commit for the dockerised ctf --- dafni/.dockerignore | 5 + dafni/Dockerfile | 32 +++++ dafni/docker-compose.yaml | 12 ++ dafni/inputs/causal_tests.json | 136 +++++++++++++++++++++ dafni/inputs/dag.dot | 7 ++ dafni/inputs/simulated_data.csv | 31 +++++ dafni/inputs/variables.json | 47 +++++++ dafni/main_dafni.py | 210 ++++++++++++++++++++++++++++++++ dafni/model_definition.yaml | 58 +++++++++ 9 files changed, 538 insertions(+) create mode 100644 dafni/.dockerignore create mode 100644 dafni/Dockerfile create mode 100644 dafni/docker-compose.yaml create mode 100644 dafni/inputs/causal_tests.json create mode 100644 dafni/inputs/dag.dot create mode 100644 dafni/inputs/simulated_data.csv create mode 100644 dafni/inputs/variables.json create mode 100644 dafni/main_dafni.py create mode 100644 dafni/model_definition.yaml diff --git a/dafni/.dockerignore b/dafni/.dockerignore new file mode 100644 index 00000000..0af95714 --- /dev/null +++ b/dafni/.dockerignore @@ -0,0 +1,5 @@ +../* +../!causal_testing +../!LICENSE +./!inputs +./!main_dafni.py \ No newline at end of file diff --git a/dafni/Dockerfile b/dafni/Dockerfile new file mode 100644 index 00000000..0d83569a --- /dev/null +++ b/dafni/Dockerfile @@ -0,0 +1,32 @@ +# Define the Python version neded for CTF +FROM python:3.10-slim + +## Prevents Python from writing pyc files +ENV PYTHONDONTWRITEBYTECODE=1 +# +## Keeps Python from buffering stdout and stderr to avoid the framework +## from crashing without emitting any logs due to buffering +ENV PYTHONUNBUFFERED=1 + +#Label maintainer +LABEL maintainer="Dr. Farhad Allian - The University of Sheffield" + +# Copy the source code and test files from build into the container +COPY --chown=nobody ./causal_testing /usr/src/app/ +COPY --chown=nobody ./dafni/inputs /usr/src/app/inputs/ +COPY --chown=nobody ./dafni/main_dafni.py /usr/src/app/ + +# Change the working directory +WORKDIR /usr/src/app/ + +# Install core dependencies using PyPi +RUN pip install causal-testing-framework --no-cache-dir + +# Use the necessaary environment variables for the script's inputs +ENV VARIABLES=./inputs/variables.json \ + CAUSAL_TESTS=./inputs/causal_tests.json \ + DATA_PATH=./inputs/simulated_data.csv \ + DAG_PATH=./inputs/dag.dot + +# Define the entrypoint/commands +CMD python main_dafni.py --variables_path $VARIABLES_PATH --dag_path $DAG_PATH --data_path $DATA_PATH --tests_path $CAUSAL_TESTS diff --git a/dafni/docker-compose.yaml b/dafni/docker-compose.yaml new file mode 100644 index 00000000..a1228bad --- /dev/null +++ b/dafni/docker-compose.yaml @@ -0,0 +1,12 @@ +version: '3' +services: + causal-testing-framework: + build: + context: ../ + dockerfile: ./dafni/Dockerfile + env_file: + - .env + volumes: + - .:/usr/src/app + - ./inputs:/usr/src/app/inputs/ + - ./outputs:/usr/src/app/outputs/ \ No newline at end of file diff --git a/dafni/inputs/causal_tests.json b/dafni/inputs/causal_tests.json new file mode 100644 index 00000000..6c78c57f --- /dev/null +++ b/dafni/inputs/causal_tests.json @@ -0,0 +1,136 @@ +{ + "tests": [ + { + "name": "max_doses _||_ cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinations": "NoEffect" + }, + "formula": "cum_vaccinations ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "max_doses _||_ cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "max_doses _||_ cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "vaccine --> cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinated" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false + } + ] +} \ No newline at end of file diff --git a/dafni/inputs/dag.dot b/dafni/inputs/dag.dot new file mode 100644 index 00000000..43628817 --- /dev/null +++ b/dafni/inputs/dag.dot @@ -0,0 +1,7 @@ +digraph CausalDAG { + rankdir=LR; + "vaccine" -> "cum_vaccinations"; + "vaccine" -> "cum_vaccinated"; + "vaccine" -> "cum_infections"; + "max_doses"; +} \ No newline at end of file diff --git a/dafni/inputs/simulated_data.csv b/dafni/inputs/simulated_data.csv new file mode 100644 index 00000000..3ac0a3ac --- /dev/null +++ b/dafni/inputs/simulated_data.csv @@ -0,0 +1,31 @@ +pop_size,pop_type,pop_infected,n_days,vaccine_type,use_waning,rand_seed,cum_infections,cum_deaths,cum_recoveries,cum_vaccinations,cum_vaccinated,target_elderly,vaccine,max_doses +50000,hybrid,1000,50,pfizer,True,1169,6277.0,15.0,6175.0,629466.0,530715.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,8888,6381.0,18.0,6274.0,630796.0,532010.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,370,6738.0,15.0,6621.0,631705.0,532864.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,9981,6784.0,18.0,6682.0,634582.0,535795.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6305,6757.0,20.0,6659.0,631292.0,532464.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1993,5844.0,17.0,5755.0,633314.0,534478.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1938,6465.0,19.0,6353.0,627724.0,528993.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4797,7044.0,15.0,6919.0,631246.0,532433.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2308,6878.0,6.0,6801.0,628865.0,530038.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4420,6429.0,11.0,6348.0,633803.0,535030.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2314,6566.0,15.0,6477.0,629288.0,530550.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7813,6913.0,17.0,6818.0,629290.0,530512.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1050,6963.0,14.0,6860.0,627981.0,529212.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3215,6671.0,17.0,6577.0,628802.0,530038.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2286,6597.0,13.0,6505.0,628986.0,530195.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3080,6926.0,16.0,6834.0,633636.0,534904.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7405,6438.0,15.0,6347.0,630353.0,531540.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,9668,6577.0,15.0,6485.0,631257.0,532409.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,8211,6197.0,13.0,6103.0,633827.0,535056.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4686,6761.0,16.0,6653.0,630557.0,531737.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3591,7328.0,24.0,7214.0,629949.0,531124.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4834,6617.0,22.0,6512.0,632609.0,533705.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6142,7017.0,17.0,6902.0,635965.0,537252.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6877,6845.0,15.0,6753.0,635678.0,536925.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1878,6480.0,20.0,6390.0,630807.0,531999.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3761,6972.0,16.0,6890.0,631100.0,532329.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1741,6581.0,20.0,6491.0,632835.0,534088.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,5592,6561.0,19.0,6461.0,636799.0,537959.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7979,7075.0,17.0,6966.0,632902.0,534140.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,71,6291.0,13.0,6203.0,631694.0,532901.0,True,1,2 diff --git a/dafni/inputs/variables.json b/dafni/inputs/variables.json new file mode 100644 index 00000000..8e4a1add --- /dev/null +++ b/dafni/inputs/variables.json @@ -0,0 +1,47 @@ +{ + "variables": [ + { + "name": "pop_size", + "datatype": "int", + "typestring": "Input", + "constraint": 50000 + }, + { + "name": "pop_infected", + "datatype": "int", + "typestring": "Input", + "constraint": 1000 + }, + { + "name": "n_days", + "datatype": "int", + "typestring": "Input", + "constraint": 50 + }, + { + "name": "vaccine", + "datatype": "int", + "typestring": "Input" + }, + { + "name": "cum_infections", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "cum_vaccinations", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "cum_vaccinated", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "max_doses", + "datatype": "int", + "typestring": "Output" + } + ] +} \ No newline at end of file diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py new file mode 100644 index 00000000..5ab40c8d --- /dev/null +++ b/dafni/main_dafni.py @@ -0,0 +1,210 @@ +import warnings +warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*") +import os +from pathlib import Path +import argparse +import json +import pandas as pd +from causal_testing.specification.scenario import Scenario +from causal_testing.specification.variable import Input, Output +from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect, SomeEffect +from causal_testing.testing.estimators import LinearRegressionEstimator +from causal_testing.json_front.json_class import JsonUtility + + +class ValidationError(Exception): + """ + Custom class to capture validation errors in this script + """ + pass + + +def get_args(test_args=None) -> argparse.Namespace: + """ + Function to parse arguments from the user using the CLI + :param test_args: None + :returns: + - argparse.Namespace - A Namsespace consisting of the arguments to this script + """ + parser = argparse.ArgumentParser(description="A script for running the causal testing famework on DAFNI.") + + parser.add_argument( + "--data_path", required=True, + help="Path to the input runtime data (.csv)", nargs="+") + + parser.add_argument('--tests_path', required=True, + help='Path to the input configuration file containing the causal tests (.json)') + + parser.add_argument('--variables_path', required=True, + help='Path to the input configuration file containing the predefined variables (.json)') + + parser.add_argument("--dag_path", required=True, + help="Path to the input file containing a valid DAG (.dot). " + "Note: this must be supplied if the --tests argument isn't provided.") + + parser.add_argument('--output_path', required=False, help='Path to the output directory.') + + parser.add_argument( + "-f", + default=False, + help="(Optional) Failure flag to step the framework from running if a test has failed.") + + parser.add_argument( + "-w", + default=False, + help="(Optional) Specify to overwrite any existing output files. " + "This can lead to the loss of existing outputs if not " + "careful") + + args = parser.parse_args(test_args) + + # Convert these to Path objects for main() + + args.variables_path = Path(args.variables_path) + + args.tests_path = Path(args.tests_path) + + if args.dag_path is not None: + + args.dag_path = Path(args.dag_path) + + if args.output_path is None: + + args.output_path = "./outputs/results/"+"_".join([os.path.splitext(os.path.basename(x))[0] + for x in args.data_path]) + "_results.json" + + Path(args.output_path).parent.mkdir(exist_ok=True, parents=True) + + else: + + args.output_path = Path(args.output_path) + + args.output_path.parent.mkdir(exist_ok=True, parents=True) + + return args + + +def read_variables(variables_path: Path) -> dict: + """ + Function to read the variables.json file specified by the user + :param variables_path: A Path object of the user-specified file path + :returns: + - dict - A valid dictionary consisting of the causal tests + """ + if not variables_path.exists() or variables_path.is_dir(): + + raise ValidationError(f"Cannot find a valid settings file at {variables_path.absolute()}.") + + else: + + with variables_path.open('r') as file: + + inputs = json.load(file) + + return inputs + + +def validate_variables(data_dict: dict) -> tuple: + """ + Function to validate the variables defined in the causal tests + :param data_dict: A dictionary consisting of the pre-defined variables for the causal tests + :returns: + - tuple - Tuple consisting of the inputs, outputs and constraints to pass into the modelling scenario + """ + if data_dict["variables"]: + + variables = data_dict["variables"] + + inputs = [Input(variable["name"], eval(variable["datatype"])) for variable in variables if + variable["typestring"] == "Input"] + + outputs = [Output(variable["name"], eval(variable["datatype"])) for variable in variables if + variable["typestring"] == "Output"] + + constraints = set() + + for variable, _inputs in zip(variables, inputs): + + if "constraint" in variable: + constraints.add(_inputs.z3 == variable["constraint"]) + + return inputs, outputs, constraints + + +def main(): + """ + Main entrypoint of the script: + """ + args = get_args() + + # Step 0: Read in the runtime dataset(s) + + try: + + data_frame = pd.concat([pd.read_csv(d) for d in args.data_path]) + + # Step 1: Read in the JSON input/output variables and parse io arguments + + variables_dict = read_variables(args.variables_path) + + inputs, outputs, constraints = validate_variables(variables_dict) + + # Step 2: Set up the modeling scenario and estimator + + modelling_scenario = Scenario(variables=inputs + outputs, constraints=constraints) + + modelling_scenario.setup_treatment_variables() + + estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} + + # Step 3: Define the expected variables + + expected_outcome_effects = { + "Positive": Positive(), + "Negative": Negative(), + "NoEffect": NoEffect(), + "SomeEffect": SomeEffect()} + + # Step 4: Call the JSONUtility class to perform the causal tests + + json_utility = JsonUtility(args.output_path, output_overwrite=True) + + # Step 5: Set the path to the data.csv, dag.dot and causal_tests.json file + json_utility.set_paths(args.tests_path, args.dag_path, args.data_path) + + # Step 6: Sets up all the necessary parts of the json_class needed to execute tests + json_utility.setup(scenario=modelling_scenario, data=data_frame) + + # Step 7: Run the causal tests + test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, mutates={}, estimators=estimators, + f_flag=args.f) + + # Step 8: Update, print and save the final outputs + + for test in test_outcomes: + + test.pop("estimator") + + test["result"] = test["result"].to_dict(json=True) + + test["result"].pop("treatment_value") + + test["result"].pop("control_value") + + with open(f"{args.output_path}", "w") as f: + + print(json.dumps(test_outcomes, indent=2), file=f) + + print(json.dumps(test_outcomes, indent=2)) + + except ValidationError as ve: + + print(f"Cannot validate the specified input configurations: {ve}") + + else: + + print(f"Execution successful. Output file saved at {Path(args.output_path).parent.resolve()}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml new file mode 100644 index 00000000..a5e6d602 --- /dev/null +++ b/dafni/model_definition.yaml @@ -0,0 +1,58 @@ +# Model definition file to run the causal testing framework on DAFNI +# https://docs.secure.dafni.rl.ac.uk/docs/how-to/how-to-write-a-model-definition-file + +kind: M +api_version: v1beta3 +metadata: + display_name: Causal Testing Framework + name: causal_testing + publisher: The CITCOM Team, The University of Sheffield + type: model + summary: A Causal Inference-Driven Software Testing Framework + description: > + Causal Testing is a causal inference-driven framework for functional black-box testing. + This framework utilises graphical causal inference (CI) techniques for the specification and functional testing of + software from a black-box perspective. In this framework, we use causal directed acyclic graphs (DAGs) to express + the anticipated cause-effect relationships amongst the inputs and outputs of the system-under-test and the + supporting mathematical framework to design statistical procedures capable of making causal inferences. + Each causal test case focuses on the causal effect of an intervention made to the system-under test. + contact_point_name: Dr. Farhad Allian + contact_point_email: farhad.allian@sheffield.ac.uk + source_code: https://github.com/CITCOM-project/CausalTestingFramework + license: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme + + +spec: + inputs: + parameters: + - name: causal_tests + title: Causal tests filename + description: A .json file containing the causal tests to be used + type: string + required: true + - name: variables + title: Variables filename + description: A .json file containing the input and output variables to be used + type: string + required: true + - name: dag_file + title: DAG filename + description: A .dot file containing the input DAG to be used + type: string + required: true + + dataslots: + - name: runtime_data + description: > + A .csv file containing the input runtime data to be used + default: + - #TODO + path: /inputs/dataslots + required: false + - name: dag_file + description: > + A .dot file containing the input DAG to be used + default: + - #TODO + path: /data/tests/inputs + required: false \ No newline at end of file From 7500e12dce3bfb23dba44887782f98fcaf43174d Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Thu, 14 Dec 2023 11:07:20 +0000 Subject: [PATCH 19/60] Remove examples --- .../surrogate_assisted/apsdigitaltwin/.env | 1 - .../apsdigitaltwin/.gitignore | 3 - .../apsdigitaltwin/README.md | 0 .../apsdigitaltwin/apsdigitaltwin.py | 148 ----------- .../surrogate_assisted/apsdigitaltwin/dag.dot | 22 -- .../surrogate_assisted/apsdigitaltwin/dag.png | Bin 55421 -> 0 bytes .../apsdigitaltwin/util/basal_profile.json | 8 - .../apsdigitaltwin/util/model.py | 235 ------------------ .../apsdigitaltwin/util/profile.json | 85 ------- 9 files changed, 502 deletions(-) delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/.env delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/.gitignore delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/README.md delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/dag.dot delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/dag.png delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/util/basal_profile.json delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/util/model.py delete mode 100644 examples/surrogate_assisted/apsdigitaltwin/util/profile.json diff --git a/examples/surrogate_assisted/apsdigitaltwin/.env b/examples/surrogate_assisted/apsdigitaltwin/.env deleted file mode 100644 index 329072bf..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/.env +++ /dev/null @@ -1 +0,0 @@ -basal_profile_path = "./util/basal_profile.json" \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/.gitignore b/examples/surrogate_assisted/apsdigitaltwin/.gitignore deleted file mode 100644 index 76313cd3..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.csv -*.txt -openaps_temp \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/README.md b/examples/surrogate_assisted/apsdigitaltwin/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py b/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py deleted file mode 100644 index 881373e5..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/apsdigitaltwin.py +++ /dev/null @@ -1,148 +0,0 @@ -from causal_testing.data_collection.data_collector import ObservationalDataCollector -from causal_testing.specification.causal_dag import CausalDAG -from causal_testing.specification.causal_specification import CausalSpecification -from causal_testing.specification.scenario import Scenario -from causal_testing.specification.variable import Input, Output -from causal_testing.surrogate.causal_surrogate_assisted import CausalSurrogateAssistedTestCase, SimulationResult, Simulator -from causal_testing.surrogate.surrogate_search_algorithms import GeneticSearchAlgorithm -from examples.surrogate_assisted.apsdigitaltwin.util.model import Model, OpenAPS, i_label, g_label, s_label - -import pandas as pd -import numpy as np -import os -import multiprocessing as mp - -import random -from dotenv import load_dotenv - - -class APSDigitalTwinSimulator(Simulator): - def __init__(self, constants, profile_path, output_file = "./openaps_temp") -> None: - super().__init__() - - self.constants = constants - self.profile_path = profile_path - self.output_file = output_file - - def run_with_config(self, configuration) -> SimulationResult: - min_bg = 200 - max_bg = 0 - end_bg = 0 - end_cob = 0 - end_iob = 0 - open_aps_output = 0 - violation = False - - open_aps = OpenAPS(profile_path=self.profile_path) - model_openaps = Model([configuration["start_cob"], 0, 0, configuration["start_bg"], configuration["start_iob"]], self.constants) - for t in range(1, 121): - if t % 5 == 1: - rate = open_aps.run(model_openaps.history, output_file=self.output_file) - open_aps_output += rate - for j in range(5): - model_openaps.add_intervention(t + j, i_label, rate / 5.0) - model_openaps.update(t) - - min_bg = min(min_bg, model_openaps.history[-1][g_label]) - max_bg = max(max_bg, model_openaps.history[-1][g_label]) - - end_bg = model_openaps.history[-1][g_label] - end_cob = model_openaps.history[-1][s_label] - end_iob = model_openaps.history[-1][i_label] - - data = { - "start_bg": configuration["start_bg"], - "start_cob": configuration["start_cob"], - "start_iob": configuration["start_iob"], - "end_bg": end_bg, - "end_cob": end_cob, - "end_iob": end_iob, - "hypo": min_bg, - "hyper": max_bg, - "open_aps_output": open_aps_output, - } - - violation = max_bg > 200 or min_bg < 50 - - return SimulationResult(data, violation, None) - -def main(file): - random.seed(123) - np.random.seed(123) - - search_bias = Input("search_bias", float, hidden=True) - - start_bg = Input("start_bg", float) - start_cob = Input("start_cob", float) - start_iob = Input("start_iob", float) - open_aps_output = Output("open_aps_output", float) - hyper = Output("hyper", float) - - constraints = { - start_bg >= 70, start_bg <= 180, - start_cob >= 100, start_cob <= 300, - start_iob >= 0, start_iob <= 150 - } - - scenario = Scenario( - variables={ - search_bias, - start_bg, - start_cob, - start_iob, - open_aps_output, - hyper, - }, - constraints = constraints - ) - - dag = CausalDAG("./dag.dot") - specification = CausalSpecification(scenario, dag) - - ga_config = { - "parent_selection_type": "tournament", - "K_tournament": 4, - "mutation_type": "random", - "mutation_percent_genes": 50, - "mutation_by_replacement": True, - "suppress_warnings": True, - } - - ga_search = GeneticSearchAlgorithm(config=ga_config) - - constants = [] - const_file_name = file.replace("datasets", "constants").replace("_np_random_random_faulty_scenarios", ".txt") - with open(const_file_name, "r") as const_file: - constants = const_file.read().replace("[", "").replace("]", "").split(",") - constants = [np.float64(const) for const in constants] - constants[7] = int(constants[7]) - - simulator = APSDigitalTwinSimulator(constants, "./util/profile.json", f"./{file}_openaps_temp") - data_collector = ObservationalDataCollector(scenario, pd.read_csv(f"./{file}.csv")) - test_case = CausalSurrogateAssistedTestCase(specification, ga_search, simulator) - - res, iter, df = test_case.execute(data_collector) - with open(f"./outputs/{file.replace('./datasets/', '')}.txt", "w") as out: - out.write(str(res) + "\n" + str(iter)) - df.to_csv(f"./outputs/{file.replace('./datasets/', '')}_full.csv") - - print(f"finished {file}") - -if __name__ == "__main__": - load_dotenv() - - all_traces = os.listdir("./datasets") - while len(all_traces) > 0: - num = 1 - if num > len(all_traces): - num = len(all_traces) - - with mp.Pool(processes=num) as pool: - pool_vals = [] - while len(pool_vals) < num and len(all_traces) > 0: - data_trace = all_traces.pop() - if data_trace.endswith(".csv"): - if len(pd.read_csv(os.path.join("./datasets", data_trace))) >= 300: - pool_vals.append(f"./datasets/{data_trace[:-4]}") - - pool.map(main, pool_vals) \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/dag.dot b/examples/surrogate_assisted/apsdigitaltwin/dag.dot deleted file mode 100644 index 660a504a..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/dag.dot +++ /dev/null @@ -1,22 +0,0 @@ -digraph APS_DAG { - rankdir=LR; - - "search_bias" -> "start_bg"; - "search_bias" -> "start_cob"; - "search_bias" -> "start_iob"; - - "start_bg" -> "hyper"; - "start_cob" -> "hyper"; - "start_iob" -> "hyper"; - - "start_bg" -> "hypo"; - "start_cob" -> "hypo"; - "start_iob" -> "hypo"; - - "start_bg" -> "open_aps_output" [included=1, expected=positive]; - "start_cob" -> "open_aps_output" [included=1, expected=positive]; - "start_iob" -> "open_aps_output" [included=1, expected=negative]; - - "open_aps_output" -> "hyper"; - "open_aps_output" -> "hypo"; -} \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/dag.png b/examples/surrogate_assisted/apsdigitaltwin/dag.png deleted file mode 100644 index 2c6525f4e6aa9418cfa0121037f8f62f22c81cac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55421 zcmZ_0cRZJE{|Br|AtD(SQB<-eB_lhdWJkydA!KD$Dm$dI8Zxu@s;o#tlueV&jEpE5 zh39>A-M`oK{PSG*>%Q+x@aa>^w42#BQ&3RQs+>^JqM%sEOF^+l zjA|o(g{pg70{(BKxtg*9#VYwvLSN-$e|Xv09N4g7L-^m=fPjrWJUop* z>%w+)JW+{Y%h06K6jwRD6(6^r!qe09{Q2{D9z9ZXa1g)`jf2_Qe8Y>CdHNgfD)+qK zwr9_tLtt=LZYIt z@Wa)s%WPSiR5KS3o$nUJbtRcKHZ<(!=Jp>NI^W*W@us1nZ>p!9eDU48Gu3t}DJgW> z^4i*r_%`*EC!>T+&!2xH+1=e;S6xlP&(BXoOB*%3L(JB%prF8cRVXzzH9Whrq9V4U zLUP#C%#4$po4fHRuDfsklJ1!^CuC*U96fq;8xxZze$dm~uBfPpAMQSWeA2{({lS9= zI{4@FslmE1VR7+#T&<|6C@wv}px|X)-9~S3?}^Dte9_Ccwyjc9mHU=f{OMbU?)_W# zq_{V4-f#*C++JAlXm4*napJ^oF)`gz4R!UsVqztqZd67`Z^u7fhD?0?{81zGFHD$} zG7a~$-NVIk$LjWoh>AwlkByBzdGh4W{re}hw6sd5TR(hYAzwN%kzZ2sx~=W*w|9>^ z3N6(POKkmfa&p+_)q^%F-@JvVuNjlB!;8NY6c)xUAdvg%Y8U^$eaZ$V8X9!RjvdR+ z$tfr)(Veotcrnu3&d%=b+qa$1_2={r4cTzfwe|H8k&y~oS`0HYGmUn_!ou6OZ(m!r zdRE6jI-38ip7+G)Xb=|m;k;6im*Rh4_gPg{_4(C5ho3q040Pw5{Zio>asNJxLvO{K z$H&|wixwU}eE8whr}k_E(T8Go0m;c0CAbgo(9qD%3UMxBVa-MNiq#jqI^*Nx0}~UW zAtBVcxw+5F$_54oZdO)SKFnfHYQC4iMYg!#?5wl7x%u;J%MzNJnqQtf$#3G>v}u!$ zp59th8ykbkiSmdD#&?gz>+wu*tHs6VdmkcR1@ti9aa zYpTLKGB)!&I5<3i{=9E$>h_HrHy$2jmYdw0PL4|pm$-OLdiws{b5*CX&1M&HYmI%b z%F4=y&m3~r-}8112new4U6;58TSfK4COW!(l9I7OK~&BzE^VEiwI4q|aOf#(tMqm+ zbNj9oxMkPg;#{!QB`?C+l9R++-Uc2aPYQ8M7 zdw;}drGmkPi;HW&pkRGnUBHX0f7+{je3s_EE90U%jdST+BK}>sU~Wn{QWg`bo{WNa|}zg>gaT6Z(eqGzIU9fGczIKdI+;1!_J*=X9nw!UR^r0 zxVZS?Y9bfg+ zC1KNQijA~veyBWUYD@zvCrBY{`+*}(&&sND;llpJ#KhR6E&*yW0-6Q}!5J9`nwy(F zJUsS4`#t%&BVBWkTMU1#i>8T5bJRh7DjJ%-LPB>s(loS=Zc&K}&)-@pwcBt}e*S%acoKZM`F4T-J&`U`bJEBxO`_VKX)LQLkV7|NNe` znC`9I%fn-CVeu;8>=ldjl_Mr5CZpfKXO*jSPVFNlTVeTLM8qq9YUU%}zxhU5;z#~2 z{zOGcNJ&w|b{cI<4ipi+R~$*D`B*h&0MBCL^RsX%@86gDYc{yLx{kij;~bQSQ8{_i^6QJM ze9P zadAs^m3Zl^7kVp7%bJ*yWDguT(Ep?V)y2lWH5zF7;z>?sZFxhFnOxJ%bJo zce)1ZF()Tytfc2X#(k$QVV~a$330k~DFGkK*ZA024YvY6FCX9emN==u zvrU3Kk6fa-{?eCH^x9t!rQ16XSQskU-y!89>G#mk(50ncHmBkw-@J*~S9+;`?eOq$ zv2B;?CZ4NHzY-NgwvP+wQq8Om4-Mtpy%)H)ILy3u-MY56HnX?)h1KJwA4^|dO29su zVo-dw_umtFr=p=T(wU*HDugFQbX?Z`};hy)xPxfVEO29>{HyQd(WIXla8ZN zD_VwQmZhI()scDYRwajIS6)$V^E*m01bpL_3KKRl*OLk(Ny`pFU@J!-DhEC%_!C4R5*#fc>a5L&ieK1FD=em;$YW%PgmAs+dVmZ-(&l~<(GVb z(#3s0_K4HXn>W{zrVQa?#0H2K6t>oZX-?g=g&JF9UWQ!3ize-aBQA(W!kPYW=@cha!@ckr7;8ehdf1!_%|v znd7;m^1OZv+D=YRd}p7A;((gl+rPuSEo1}r_I$VRdL~|D+|dHY zs_%1+yNsjLACXI`4dD_Ki(39OU*z~z`u5JFa==i#BwS5DD#{HQl~_dt5-Rlb@_jQleIdj!}h&rj6JpT3SVA<-nEYKcW{u z2nEpYQ2P1pU6*kx{lJE&7WG@Cmqzz6Ffu-n@sXacT9qy&691d|g+)aK>;Wu%`C>FX zJKNCG;xj%eX4AeND0SG%-2CRaq`O)CT~LKP==tbfPYlRep74c{nLVE@&e^0Dw(L>egMm-dV?Att+2? z@uwy>mVrD8jf^Kx{OsQ6;`dIftJ46rksXPf#-mM7Pd^_R z{QP0X#l^PYfgxYNc@qpwI50Hi2iRr*srco`kF4v~`LZo_uNWyRZm7S#%VMPY@!zHI zd1!R)z^Y2&?C-GTV{OTlVQgYrXRiXRer{}RTtIDG`1`YE45bS-vHr~)l^)CBholS6 zjNl%gqjc-%nQ#MO{rGXo;mdPbPtT)(l}C;o(K*)V__2_C%a$#`nC1Y1`wRu~iILrp z{ruK*>*#LC{_x=po?csPYXZ*t^U~6O;L5D5gV=GYx>9%XfRcgLQn|4?9I^RaN)<;yWnzY8n?-3vnz8fVH} z54;MXwU`_4!lue7=hGA=C8rMX=!}t zm`)wG|5OWDZ_`_G6pgO-{rebzfh;Et6vUOrrl!H1XT9dAexy9$I3Et$ek~CYpf)54 z0I|CIIIh1g*SNf)xjEH)ehNiyt)t^~y~9>|x=h=w7a;74)7San7wgyn|S38xq^LM{8SvwbWo3;gCoyX z?BNk7^RO-SKAM(RR)uIj{X7c6 zzskCIW&%&IjXpz%_uBHD{)^QW@7wHBx5C0`u~96>J2RY9Gq$Isr4^xN>E)T6Jb!*~ zPr17_>cqn0qHg(x6u?kx(mu~RR=!40SorgGwHb}kQZIk$Bq}AX3N|}hw?$i$qEY$f zFkITe^mGf5z|dL)+b3EXM?7cuX6faa1IUmXFJVPRdy_dFy}gD&iU>iD%8jp8jjJIk@leAL_d666E7pzD0u2V9)@V(C`=wWUoi zE-u^o&Rnmms#>qY>F06#RJ?RpUY;Q8^u(u<5cKM-^1U-d0&f}&ILAwYPb(@bS@CJc zWv-k+4p@J+6B)@nckMEGapfooXASm3smlmCIXHHm#t)%wuvB`_e*>_%|L|cup!eme zFBwz!YPGQ=6r^;~(Qa;_zU1yc0%d^^0NNdgTY=TwVtCknQJh!*j>wcc4{t^p{59Sc z0^kr46O%U8j;$t}UsN<$8_XEGL;N<`pyAR>L5+>7ZZX_(*IkzxlGs&GoG|^nFk|%M z$^&k-=pWr@9aY{ZdN}}jVa;}EYinaS1Y~7p;fP=@lWo7;diG2lC8YcO?NX3 z7$pEY*4EW!V)e z1Iz|_^+C}H-F0lcjEqcowt;-I()end7!)Pk80}^6@xwkVSEN_}y7XTAV}C04NaB<; zp2< zo>Q?W&YWR^mN71%LdD6p9s-hHu2C>rnPb)R;k$S5{{0!h8d+7~7W&tYO0#|gz%%|D zN|7HBSy|K8l8HalRcT2{H?a>+^rV2V`HJGaKqjcCllfbXlYKxyK+j!efE~vp{S{Eq zBIwG4W9|+)M!L_?ur;I?{Vq>_LXisuP9|IpHv)o6vAW~F$?4P8fWVQmi=P$Xraqr%p@M-7o^2?K2(s_xAvN}53K!jfdg?LUTnDGWZ8mOJPK?vd{?V~%@J#&HUB zP@_T}&#PLIpxm;v4-^hX>dxcGIaBu(n4;*DnwO@1Roqq%*E)w@sk|W_oiw}g?OPobw5%6@rjF?X ze(M0OuB>aS0=45k-UX=NRbWxC1BEiP$hsvH8lx_BF`u%^`r)ame_u8?{q>w1Se~wl zN!&O#vUX`{X$G*LPJK9sesrbNU_c{|m!-9Jn#7fbd|p984Y#>*UC`i|OrR-VW8)s{ z%wd(t=;%y{dphT?En4Q-YvZ;LL4?rz`0-Qpc+@56uOq$IMyhib%Ubk)?`g9F?{{epf;m7X1n$eGd z=W$WM(0!AWVK;89Av^-DZ`-zQlJ6g`gg$+G7}y(NynFnzKmCB>z|_>K6DO$9ib0{3 zb4#=+fw_~mlwVu+$~l#B?0f?~RFj$c`4K!3Rrfr=8?eLq58nxhM_2Uja|8GQo&s9& z7#)F<^;^eHCKy-LfvsbkS6WtPZfDm726p1qsf~SoFMTO!*d?ETD#RM3fa{Byj4Xfo z@&$sOSB|4TR&_6c8kjEwZf)t;n7pfNahaM|B6O>yEluykWk%u2aifG^sS=+y5U#?aBqc>tOG^$~E;`b+(K*{r;Xl#Pd7$|!C@S8)bLTjKc+UIXAbM*7jBoEc7V!A- zE*ymZ#;Aki0%QK%9gNhPfvjw7(uI$7PLdYh{zQdb*fqZoQ9g(x@)WJ zQnIo_5)(Ngs1J>d$k6L$M(3Qw<2_{4K6Imdr<9Zw;GpoKL$!eGU^Kse{SrES7#hoI zPX}AuTLlF(>%_s$Ee#-7R*4D=_cz?#6Urub3-VWi&+0WvDXA{wZfYV5yPZKpFLfQ$ z%CT>$t@XoEYKN3U@ZRs=si?2eoxn|B*vcwuXoQ3H4toM5curT&-#={6X5Ae7^3$h> zDBC(`eKa-OwBP?upIrR&N2bkd_UBJDOVttqxBmWqXsZ1jP**S6^|8qiBH3`205u&iL zumQ|!-@ZW5D$4G!P)YzeIH(Tk%GSMoO9LbU?K)HHX?}ism!1r#UphY&QFN3T=#7Ju zlkWiee$38-&kJq?R_io_5@9;r7zGK<)ZX6s;OAIji;Z8e&J)R%8(+e4ZD9?F*!4B* zGWp+^x>%zC28D<3b)yN??<;9TQqinvc1;YI+phngQByYk~D<(ac* zBktXs?ff(Q1ZV975UJ1d_j3?F2SxMp5)zJY;>V^X1UGz|_7M5k^XD&p@8*Y1Q0hG| z0HhEZ6{UMNJ~j2*9$)n?3+!h23Iy=Cw==l7{Iu?+!|K>T1Gv~z78T66PvpwNrKcA^ z)-(xLWlt5Fu&`yOjrrn_Fu?)=rxJ4croOT=U-f?Bcdk)s4e4e;2f<-sDNciRxSs$N zPGRsp+n&6nq$H2QF!AReGh9|yR#$$tDT5m^UJnhW*)Cwz0C57G@g+nP08OIN+~ZcG z`0?Y1>Ei4#oxQ5hhrAS-PrdO5Wv&?_(sj+vw?WE8&_l2NnH1;cUd5%!W#@ zxy;|CqoWr95R{>wLB%6r+~@Bg$CahAG$@%bx~*0}fui8>4WoG-k&x(aJiL0s!6BEy zD6>YbYdM+S`uusFvw^6Oq%`3Mu;vHD(en!m3$@}tfA~O@hKGl2l2R*~2yRx>nywG{b4{>p)R8?=y9)W}aHccs)@ZyE^<;$104zsbb>5@}7H|N6N*t>VHPI=X{ zgAe24OiKFjsKOpUHZJ*N_3ohxbr_^F9sejXyY77Z&xfJ00#37ukRwG27X}u@18k{( zXW$uYer<9J&KE@Ob1SpV(tjpI-@bcS>NUrUWAY7;@9*!=j?n(QF3>`Fx9qL(+$nwK z{*85%-J?&(i{O5NC8CpD8@hVs%9WDi7N~OIK0vAXJG$NzP#Thx_ZS+AglbdG?8aez z`#{v7{4)RNXO7mjL0jpJVNk3+dGeiMZm#TI!2PQQ-t!Juu3m*M$oti2wbCzYb=v7g zAf8NQ)$*jHM#U6SG|_I*(23^-FbIKh=A(6dTcKrBTKeyqt3Tg~R{+AG;R<>Go?z?g z4mqBpbvHUX)h`;t!oc8QFlq$&&GY9+APYsYwxOsJZRF+48(6&lzP{_ezUu%O8(mPT z#ipa^<&5Fyh+4hv`?|U!P9!n#!mi#ti2|l=+xGh8#m9j=k4I$Yi$Kr5_IE~k>_s~T zSj#LfYG!dH{^#VguhK%Vi_)>t3_u*{85xs{>%RPv%&D+0yb~ne8hE>9%`GGOUhg{`Qk_DxI$(%>%p)- zmbo3oA=dMp}DOSSm)^Jmd=+z}x@PsLCDESvJ@gk>cqR%}U==ir$ zbhWfXdjaZs3<>mh4!Pzl2< zfQ`NuMzjJk@H$dyCqI=WqoXvsjGf{!&5RBDzbt@4Pce`+xbKhZ?>Ml8MO!vSXC!8N zX`n&R8H0-yS-sp_n+_i7 zIV~+MD^`}iU;J38I5s{f1QCz#+>2P?s7`Y?+M9U%Y{d3E?AZI3`P`M4AZ{Lu!!e#X zg)_6W_k{Uuv4M_)roCyASq%ki;<)l-&C1G3dz7+d1ToKx0quwis-Uo5+_CpbMFlqU zE{HmT(DVQrT}NBjkgF-az*)uU=b=Oj2lh4gA@5)dyOO-TFS-XKE9(JmCK*;D@F*ZB zaq;Vm$6zIK5&JZtb=K~1taurDKs*2GQ$2T2!O+;)o#bkP8HsE#H8oW+*L{}U;vt7G zXL^Q%H$rg@#rgn^q`Gp41O@HJdOC~cGt!eo1L0j;5>N11sfYF`ZH)%EF$NrPzN=&{pU#3fvxQATXijMZD?3S(C2p^ zyBrFE78nLnUm6hmmANi`NUm=lbf&rf{C08+hxA)uf}_quK_c$g3=CM)PCZU4EWAh5 z6euknpnHpPijN~6KYoWN56-b0>hva2^d}0aQGIZ}zLZ^_6fG?+#CvYWUP)=vo!sG)azSZ@A@x7;T3GllHYZ?H*7CjetzBIycr;e6o}9)z4-kQ}|5acqk`7R-Xm%dC0~JIRD*Z5x zD@X>4)_DlAz;^>E`tk7`pxW@FHQ#x2)pDqvIMD~V0O#f|`kvERJ2%n|6#37drJ*IG zgBp~^eBB}UR{$n6U^YDylfP<;T#8m=PR=`&)mvdYK^ThCH!*{Nlcg^5b^XW*`Dob|eq|IA?}Knm-YIASXT#V2NDsKf{uM_w8k8w-w(?&iq-4vGks zY}E{?L$Kd5Bh!clzKDJbJVPaS5pRhtpp^!?_^GBtKt>WFLTK zaBiR%m|wi84j((SM_pY#&h{82kcr;PM|0obyYyAB0sej<=_!K5KpISj7SBAtbWiYp zNP`A8o~1y`(3cu-7-0X8c4ja_-Ash6hw`q~!%eMeck9+IbRtuTwBYrm6+&c-EHH;7 zQ#~!kzH{eJa)E08t*E033dB zw$z;sUrSvmqf`~Kmu%XS{GDe?GtXkhVH(ypHfA=*o;<0hwg<)#)FH}T8g@oTd8AV) zfM-dH=i4_(p8wnyq6ZXm{7*3l+qwY&B`GQSV}9Oxpf*^CmB(O_xK4gzf@|d#!&mDPb^m??lp!4dZvcS`3JRw+HE#gjgTKS(WkQkv z^vqEM$_OG_8BvTrpxCvqUmxgr0L;M1&fW}7fW%!I8#ilcXn4&}oy;?-9Q{<1RVJ|i zAnAv@`T1$Rz5mG2@Dd8Y&>zTWP&ib2QB+Vc9QBf6*RJL2H9o-bg|VrrJdhj6MzppL zOiAHJEC|NC*@X*1s9UpV^!1q+md8KqBDB)h(UJW8`F(_FhUYR@N`Xm$9qu7-20s{^ zV`zBzE&_H4Vqw$o-M23yI$Ce?55i|YyNyb0yEY?O0m2*CcH?ZEI(QHn&9OuD15P9l z!@y93qWlpkM?cp{7T1er1npp~D~nmm>sLZ{HjTD+hXix8nyM-lEiEYv*w&2jxxi;| z1R?mapv7#YV-NNBSG12pQX*(c0L%tRr3i!pfh~4cJwgSf7l$GYGcv&Lu`fw3PF0l{ z{4>pQWCBWNvARfEz#7m{_Rzp|bMo<2VN~#02@z zKFxvX|*7RBpw`i7X)Dth(JN5fxm~yl{uctqeqVzj8MRo0V}%@ zZR$c+4yd>Zc;YS0%oA#A$vCjs$$f)^^2oq^pIc~#B}tt7Ejy1Syfq&i9`;9d2XV+9 z%jYKdI*wj^1qcPI{8rR56Yoi}73e)qtKr4SRDQkklNOy&S34y(xcG4?u{-HIj!6x3aM{X6j~rD7Ik)_#keTkB`sBEjvG$)@|J6l$W`Q zFSD)z1^YHY1;r<@)|{sM<>lpNWlw^bjj(x$ebZCs25!)YI2|q(p&gKRrJmugAc(`b zi)s+DqaHtY<2x&$vsNts{`?DW2+!dIxp%mzRv}8qLOqEYpHPz?fy78QqUbN`)mBwA4|-6{*^+kK*2aVCZ?VMxo?jgkrAz~uMb2FU~p_~v7rT} z6Gi47R7sY@7tiqP9ke~$b*LWCb^r!CvF2>r_x@d8c5-&sa`%+$%!Eb-EA1V=_sEq6 zA$ZJ*i2V_64z@KoJgkWFcA@=AFtTuS4S?ka`KD{mKE0R-$KcA!;t0(y37xWDX$3gf zL~xN={>8{|SWFsFi&voz5fC-O7Y^#@4k31sjEVq{1}T-w`tkWPBuEg(9s644!>_|T zev71TKqR_zjnyD65X}v$>B7=dV z7zhNE{PS-ZU`|c$GbX9V&XXxxPQOMqaWf=5gqSHgXX?ep(OZX;pXcQX5!EzX&ZOmD4bkJN^x-{Yz=?n$06ed1Lifn2Nvlo&kym3 zjHu!&@k9<^{7{V^qM4zCMAA+aoQjHyPUARwH&W#HX~dEEBMwN!ojcb{OOKLFS5p%; zLNs7fiBlqqwD^l}ABl&~=0hP306ZycvK7Lk_M9wv54MCtB6Y~I_pg&QfvcaQ zk!`G6U3Pr2vSbH%PSShw7VGBoz^d8C<NV6ROvP$i>)Vzq_kOv0a8>+V_3M6qB zVYWDLmExQtPQ#^3La63DC0v6@WGySJiI}KhMXX3gxNgm?Cy}L<a*YR*W8=ESw1EJ+QgjwBtJ(a(iFuSQbISCW$m3s7PQ4vvNkfj=Q zdNktikqAlA1)CRZ{jt(p9Owb-97}R)!^5v3!do6L@EZ8+ST(#n|J51@0X=2l!TV>3 zJR*>jBPqzsdv7Lno@g>iXMhVMaWkvTc!oGdJUYe+P6$S=r=&cne{t^T&j{+-U4m}bgA|fn8rW7~Gnyd~ zVFx@LTUxAv?nvK-khBf}q}Ov~7i&J>_+$7NTXsp*!ub4vL>1r|st?(fF!H*{me#|T zrley(4}MG@Rz-y*N)R!;kS%!vdL|O_!G!_r10Ubk-EGyGu8Eo*h2@m$oFm2E7DN#A?DfvF{T>ZOuVevjI5A@Gnom+XvolZ4#i1am-^*7icL@^=m zJhq#&qQy*2Lt~>kE&dA-%f!g@}yu4gE=f0G5h0y&F^T&aS~NIE9q(t(2qEpT)f{`?WW)OURG zBfTw2F(XI>Ckj_YhJwI0h&mA+t%a;O>J-7Zn5!`|QvYx4rc3U4SK;+p>A!-KIypN| zck2)Lw%k${6%tCneA>a>Jn8RLYkPZUw^S}biz>Au&=}U(Biml5q_Frx+u9vFb`Y-% zt=Q#D$av<>Mi^JbK)>YTGKl6K$}aT~(Hdt&#w|a>D<(;3$dL%kS(eNZv<95eKQfY5 z(OqDnjJcFx(0jO$udvmy!$s{s9fH(+6tD?JYQt^>?m!6<*C5UUFdQfJ2yD?u0P$P5 zZSzY@8(HTz4`4tX1XpAzAb6ptDk6^;2FQ3&e-^;#Zd}|oAC_oj(BhGxv;-U{L1p9+ zU~8O#@_`4Y?`|X4c>uI@;!9}+LP+#1EJ3geAWslzf-w_7lko%y>gYBHu#?5b^*v)_ zVw52@y?*_=aS$vLzk^RtNly<#X2>=?>n4G*;ocXLaU21#odZ^T_1580vtc9()`m#e zkIHHNlG7J1Trf2^*RsE=JLRa7o|KbgoO4hs3iT)%DH1d>bRv>*ha){WKHiAb2S$M~ z@}y+2(yJHZ-Q%4=S9OjuH9!AH!HbB%jLlaxxDFmv^E6_j&-$`sp>Mvb2uGW)r+c|% zWMo8JPjBOKI4j8AU7dJ%2t9Ol2W)rZ(!z6*Y`Z3vo4IZUL1ZEL5yNw6&-pkiwmFP} zbR1Tw_*jbwsPX%o;tx#5IzWGkBs*lJYj?2lr7fS>C}RqiF|uFzIkual-7zxp>2 zU|W~^n69>M`Sa&G)ZV&o(_gFNQAed$1;25^w8W@Q&$>Upz(?$pD71eU2E((BO78)D zuFS?%{lFu9vnGZQM(%XcWYj{x9$gn35$`H6I-nHP`xnb)x!x zb@g?U1Od~8um3N2Fi3tjHBE?(%9HTO+suY1p?W`k^RD^$l)NlR=!D24B1YnGc-mgS zC*Gj;ctAMK@{eY2Ls9uLGqd>0GV`VpjvO>>vfDApBnlTfWryy`bcsF#FbsN#h>1Hb+*ss;=;%>5DZ0XI=y zaut#cv2)4%*Xy*b@#cnx)8pwj(4HWx$|{D9x%(D#)g@%3PDgc|bsTZ9a?#!yRhwJ|Tu(Z}sNG zk3Kx+E&wTD5OiV$Ha?Gp#KVMy-GJEht#f)M13fM6?f0HM*$cG-GmF}%s3fy3^BVD7 zBNMbz4qLl+c67*ttpaRP_F`yY{c8E~LHDm$m66Fq2_7PR6}Ud4s7g+9ow2boM0$`< zZe$9uK}lHc$8gL!-j1Cbu}5Y~X|k`j!34Yma~KGPm6a8+?qEyDzd6X=d-lk0+Nrzl z{B#OtA&_E$+um3S4HS7Ax(cg{7qk7gz+g<-Am2%I^5jXNcSIkOAT|M$O4jjlZTsVPpXqgaqfg(x}O6GU;IWjNp6{ z@&Slg>0sm_r-p~8TPZmFq`rO($o4Ee7%~zA-S|nUsONoHa3s6{T_Ck$J5+F#K9nZa zXVtn1HuKCCepnFf9xwtuI7zw}e-O0sIkFv)TrB`_ z;9JS|q-<*I6SM6+aa1(+-n|+Ch%-f2w7`5&rG3#ig`o(LVNPS)Ft{205Oj!!7#aEG zi?yYtt~WoR5S+DP7i``-NN%McF_Q^gVB3O8&`c?2+IT2DsH>JB$B3E~;n)&V2!R9v zw&5=QbN2b~C7hj{`oU9=?uaCAQc&vDZ|K4j*g8TfPNE0 zlgca@2_`g)36c;*5E5XCpu{zkxK8UfLBDik-Hx4(h+w01D}}OXPNyC{g)s$bX_Hc2 z|J#{oA=yA>%K81^e@Y?%6?O=j6+p%T|2P*teVW@PJ(#qDolD#jaE!aqFj`++^(Ns7 zOhQA4IPLCfi=$avePdlI`qR3OH8NR9H0V8=Vm5twa|54Y@#u}O7O;Q3O}jCClV!$B20oFhBn)#gMrCm`VfiD&YsKMkFuUwHtF(wO3 zC%r%+D43_*O)L4E#;e7=;SMJY3rjvA%Ac?I4615F`iV|A-{X$wWV+U6Fc{~bpY z6MLp0r?8B5dWLn1KoRw_&&&kA2U6gL_7B7P9r$3~>(~AOJwlk}`12Jwi7hvC%`9m1 zH{=lir|$8n-)~L$GqJht&i|iICo?~_8Dtkq=cPZt?Qu&G(bBi8PgM%|Vde&v@sM3N z4J9QdGRK&4BxaoF3Ku)Yt+E_!Ic)U;+b&-C$-#(G;-}YNzj8qN#%*Lp3!ZBx>K6F^ zKY{J&Eu1Ed08HzYe|xjcrghpA?YZ>wBpW1lx<8KvWEBrd0?849!Dr=XJcg(T2M1rR zE`MEZ!>k1um7j;|^ZvHh|H}dZz~MdvQR0X~bn)UvGDZhZ6MprMrlFxKR5;99(0#$; zBpMkTDJxT=7+!1hMBiY$*(|LSA}+Y*W{#&dPBe zLqnp912JHS$lLd*z0WpyMMl|>lfnicB5E1LM;{Na)Wr2s5ST=f21P->x;m$w3GPp; zjI%@Z7m|^KY>oWH4CW7|T`xsr?4rzRa3c~A=FnA85sBaenB#{eKkki=7GvGQ2&41r zR~<%TSFU0C8A%dlWqFuEZYO*WU;Oc@{SG2D*jIR=jSKUIK@kxV z?PUgC?H>^~piH&L!%T&UN=gjct3JP2oNrA%z#3eT3KOoLiE0<^(o)S!Y6{5$tVvZ zO+@uY!DraE4R~GhN7apZ1U9gjCT`G^DJJAv*q_T4(GSeC|7&MRR2eZ;GXjlS7j$Ec_pzuml*PPO^ww}5h;)C&*JT0V?6TwF}O z_Q|NRu+T_VTlFL_laTf{P1#7rZEMzSP`G=Q zTR;z-;HP$nZo5v_m>T!AF!%l(!Nj0f7MMx=@Z4z+K0S&*4Br|CQ;0mUppcMP$X8z) zY2F20IUzH1^Uf`lu-fqMi5qBOsAmsacJ+*;KY6kq4ss3hY<^G)s+*g);4w>}@NYy~ z_~^)O|CAAzmA%X7>`eE`3=z@s(Jlsp+Py$dY+x(@GE2gw`9DH2ZyK_cFK zP%oQes~J%B0wy}hRL3Ariq4W!kW?@ zWMgL_Ui?~mmwDz)nFLlMC^S?7iUi_*?rE~ z!Bz*`3WIJxe*TmP-9gpZ07yz^u^)eXS<`+B63Gw1baOa!$Y@q0Q-SH-fWsQ)_m~-i z*W+;q1qaI_+EPFy-4jZCco5yHA^qJw%H!h4gjAX<$7dRz9{uwqeO;VCKeX z2z2d1kQ^11mG?kx5k3_seRZ=y3^f84(6>a)P;OP-CO?08R{+!w<;D-(?G^sKO&L~a z4+b+qJ^#@FE^h7(Fjnhy%$iy!iD>@pssAY)Qsgfo3F*O9m^E=b-xZLIz# zzMqOA*bhjvp{py3q7~`!F_=p_`ub@XZ7_|n%8{X?ik9sP71t1@;N$1dj4@5z6v$(c zS>t37oYRPkz$QCzcFIKsxow>P2sBftQX?d386G1oh_gzzcFI}Ktg8C39m*7-&cPUI zQ#CJG@Na$uhKAfj3RIJO&_0}w>|A@iuC8u~Sg+Uq@MFF?LV?Tdnl{u(N%n9GA=%yZ z^x^f%_!@0DH@8Raxj8v0ME~-W)wMhVCv2FD9$07dLj{JOsgHPNp|+ZWak#F#_W$D z36YVJTZZ?+>_(AO-nk{v$jE5_-n~!cN5*K89%#owQrW-l9&!h8mmCF`q|ehbZU2`e zKz1huH*q&JlNT4F1(mJPvLOGLEeTe`S1JZ8dTXO29cXu1atcO z`gXr}Et3GyA|H;hRw&A?86>bXkRdTC6$;C_;UMc}VX2UT^eWlpk`gf(*Z-obKW|u> znb~YW+k;2GXYbziB*%d~DV{Vg<2Hy$4KjiV&=3jZX{UfZ6k2%41;{iY6N7BT8ntM? zm%xJ`pKLvZDLXQwutSUe*ku|tgv@)9Mfv*!5w5JG)`#j3$e^jOPYL9Pr8e)YzVV^Z zk{V(E59bBA8Iy(EFq^<4Y`&h70^Np3Ch)0n0uR|3gp*L9|LUK1P(tongIv(Bq z`FuBc!@g6o$B}u0m_^32pbMCT0;T8X)}f$d&_C~D=_HSo)GjPFdA$wBF~jkFsN%5v zDf23S8(L2%zCwL^?0aG_$rzHHGODaOMx@B9)sIQl1;`8}ObAWmCBOmRArp9p&w)OsY1n+_@8Xfm z6D)vt_qG+~H{khZZ~PODCYGF|_jY^^K2z8fc!=%Gl=fcZL!f9>(*Q zmi=%fYT!_`PBtmq+w&t(8i2g;Ae^1>E$Q9omGrJP|I`9~~wm&e~5xJkfCP3StQA$fZ6C_+ZopXF`G3(dB8SF0(}X zr)1$wtXscfLqF675>d^}$`W=RJxzin_eCr>BNXy=a>p9uD0~%}-8g~~Jq50<53}?Z zuSf&BLudbxa(Twe%#$4(L;aN)QV7rp<&~5+E?kiY7i00Q1<68Tm}*_b9V=g%eX|el zL=Ci}+t4&!ZE5KyDp`eDvXmfy=&X5XdWd!Hy zx7%-!Z?h?6EIPjXdO(0un#}|sSo;i$M<1SZYe&Z`M27NF?gT#DRkzOp8y&}E`RU{4 zCYG6*`LNc`$iYL>#QSpTq;P}R6v6ngGy3t`9Hd|^X(D1wyi_uF9Q=J4T2g#@1N*{P zD;I(d@nW%?k#Ax~x@=IF?wr`06NlHY6}^7<6F}sH%}doMGZzs3rNJXWRFxL5X0pm0 zh0r@x_Ttd7V>^g}4Yw`;1;`xe3EAPGFU@&WK~xy15JaE#2d<}DzZS?JjU8`DV?x3} z8E*k177Yq*Kb8rMp4bQ4^EMYQpn9xDJGu^T552V&A$Zi!B(TEMC>1yvE|X|9ySHtI z?L^b}G3W%kZLPU5a1?n%73>n6d@dOo4hXAwRnKu)B^`<80P`&B!6fNq(`Sf6FIG| zP2OKZ{#at-8qAiC0EsW3`l=WeMBcmu+k+Y7W0xRCz@5&&)Gudg(95T)#C3P~zI~L) zqv90>N>~6GJ3uahh*hEcAK%N62%wMls}5byv%k2cxHus~Lu9;`r=Fui0UU_*e}?Y^sUR>#C-%iR-xxP6orVatzSNO$-d)uy7Q5}a(<`iXm7uH>z1nTtN=BHD}O{Ke^&9G0tP5oU8g=4 z^x7N}rKpk$06R!T_27&tNPy|)7++ssxzt-qii-UhtW?AXBQxLRZ5aMfpI%N&`@==L zJj&eyCZ{wR))P+vgT&?Aea>M^AnIKgl4S5byGkQQ!f+0}I{()&fCmaeWpWNb;VUsR|*9!tLtfW3jvzP{4@?w!95ulv8S_Zfh69CS7oMMRy_b8`Go>^0{* z|8boW<|g|jjRctSK=A$*vzs94@HosaUZg?EfQMg_KIz%n+1UrTwi?ItE=oJzNO7^Z zf)z+{B6i;7I7VL)ll%dMzL7-L(OGG~jb!0k@LoSS>OvwSH?j4=@e-4hU$HP1Dk%;Y zwzrR(4?6GpzJEVIKU982-?}r<`r&QfjgJq92M!V{&XUsaT&nJQr?SE_9#Sw6L+&qY z33MwAoIp8H#wZ0`-ah1bAu^jm!0vf!{bC>9jJ5?^n`ioGbf0r6cpr(K0j7L0uK?fz z{M&u-AT@UMe!O~0ZgADoO%5wpG{Ugi|756#0fbRH9FhldhSKqL_>D_Pc8-pYQcyr> ztnRlLUiWTu1-W>)xSy}Q}U<=7LEfod9uLmx>Uu$5qpYzV# zvFd1QN&)kTWycP@$z#6Mb6kL219B>?!gw{GvaT)@;ZE??H-_|D&Kt9W4U+D#(BiYQ zjy*gOT*ee{pHcN;7AIK+aKD$3HG@$AC@!=|>u74qBBFx_cJ7Fj0p-nXK8n?Y`wC|d zWQki^KBsIIrHMj)A?gT<7{=<<5oT3Jfnn6Z%PQg2(6UVwDplOQtGO<#=-6E0+7{m8 zrDPfGGFqAiW>8}Is08o}9H2REW`bd<`@?)#}OtY`>O%plZe^sGRb1SG`) zyR8QAX>l9td3^*)QW$(2ZXB=v)!oF##}NEPWiUk=@Bg@tqzs055fBIgd?e2URSvt5 zcHlSO=Vo)dDb?@L7EGm%etsrs;nyHRDdFu`d5~*|7^d``sjBuR(SO~j~M3wqLyN_chGLEo{0KR}i1j3|0c*ZO|$(xlI zTkr%{l{9PyhWookmB2VVJ~-?|%)fm2Q+P9KG$*nc!r$0-xEHb?g%f90ppSw}LMJE@ zBwjyaeVmvS(J7@u0qxt<`ho)LlzsfsynJF zWO)#W>e^k=z+)*gAB_$|C<5-YwaUkbASaZF`xtRNmXbxyh!=pH`W6Ps}A((a( zyEsUJiv*6LB!R9&hNESx5%Gnlc7GEV5CsXrwKcdDG>;Yu`P1|Yewe=o3KkAX&i<)! zxF!mZ5L$|e)aU9V&9tS*hv$@Sj(I1#;^nb0(^-;-KtoYtNP$OVt79QPb&rGJC@R?* zxI92PWVnnBec4REf>1jE&0}MDy`9^`G*k6c0gvRB1$u?` z=yF&X3%DP!VN4}?U_Di-{od@lFNi9fL&l1xW+}i7cM84?=tct17!@S%p#rNMZQUjU zDhxpqID^1uyh2J5F;-Dc-(@}h1P`2 ztcitf_m)C>cZ5ebbjKjBm!zEiW^2gn1*CL{vIF~S0C1G3SKD$iQozEG*IGs|yi)aXO)lUFE&74mJTI<+rc~Un)EqVb9|6129^!y~gV{Y6wcTv3c|20~5_w zY~fuxO8_13){$LpHOr+6Xd+2qT%#5njTWO(5JRU0vwOK}5!WT6OgqLaB1CdDh@(403y zihR(~+R9$8+95!j>V$Y2qIgGT_{?1+C=pZB_wk`f}W;>(bx@k0a- zPYcIokF+#91$m_{BG9Jj*o5~Nnr~+{FJQs*C%&@9eKNUau-^{)0H_s+>5VbkXV`@( z1oxtfT$Kr1L3DIuvU8ynKYmDnWd=I|f+bV6sCXgNl(&Dp@h2+VqeQn7PnZbVcv}T2 zteBxbpz{h%Ll{~mmP1Tf57Y`T93VdL0Ud2R#ex7;^i$j$NmOBH(r+bicF7c1lho^v|f&%*?N{E|V$5nG**n(^qC|JaaXpH{w z9eO|w=1<7n9A+jl<60%p6jA%(l zh3u?CWF=8XB@#&kksT4TDkSs&I9=c0e;mjEd)&wUz3=OyKA-pdHO}+7PUtDgoH^C# z^7x&FzLQpgkpSWAsHw2)bfz`y3sNU^RCl}a&DV@K$-*WbiQt(}fyPm%OP3~`$BLNV z8S!ZaqQbwhr?!q?#Euv`G|^}S~9)+b$hE<^VSQAZi7400cM=FFnQhqiICBF752{?h&LpWovF3FTS0 z49_U5I=$lOlI3Kf?K!$%qo8-lO=Jk7(1&3x0sYXl^feR1DyltqL>c`&+%+73Mmofu z_p34j;#E^q3xJl%2C1g9UjYGCvP78`8MVI>hKAz9}zgo3c!m za$IOIc_PG+u;k6P4}HseT5x;czI_U`XrkT*3$Eae(N(##VKtF`fV_nBSC(`*muO-# zXApfFjJf&6c{HeSuPVXT(@w@wxp|TW-0$*}t(!Lof}l1Ncm}b}ChZ8|l|_==Y|_fBYf5dY!uF8BN`VrVSrWmWe5v}We{*f^PA3h< zFVnX=j=Ddo+QXd=?WbE29r-2;4 zE*E{xX@9!*=y8bKcO;E#qL&Z;439UWdBWxMVquCv*(f4O2^dDw>AqC3NXMz}Z!IVk z-y#A`qFe21W~QTvJ$Z6CxL`;5ptgFEiH1wGM#gW!Ry2?V?@}!103i$e{RLe+LT_G5 z01?%b)8#t^FtRyrba!f*`uj@AIy-{{U<1*dhv4)Dh&hc>#X=Xn(v67IaCUWdBz9&$ z>scrVBnUq+uOB&Zz`D3FwKWBh1wdC*l{!giXi``Gy_{%xIy;g|8aq@okODSkt2U$` z8Z-^#d-{nM6vATaoHewxntv1r zCqVW1gNFEqL5}R4oHhVQ>uy{6hNWePS)59^*rR0kj_0D|z8>3n~b}0pI`9gTXh`s&i-h0#N|7ZbL$WuyULs0aNOE!sH^!Lw-oxK+5 zfvE2su&O_48pf_o9ovgl0j-Z-|GmRO!VYrOdpW*aNVB@O|DW%kRM{TuH=`*7>ewQD z?P7Z{gtr{vpkiIj2j>8hb}%zLb=_wzt#cs^;o7{iJ1ojK*sk03zfWS0y^dN0=VM>J zW4;gP9icty0}tCuTYD7U`r9*k{SQK^-B!C^7%q`Loa2$0E$KlEc!`4vr7ccuzHxNN zIPuAI;$=(vIqHSk_9m(ZJUv1L9bj-Il=f(R{D@?OIK8b7Z($)U{J+h9QQ|DH8VWqN&2cU`>$fGIAgJ?d8YT0&HqJh|jQt~jdsU>c!2RcB%R zlG`8N#%J>8sES1fD}EbxxR>KdGFrMPfs`j=B38(1hH>@Csna05H9;kZ5G=`qfJBkR zkY#s|te|hfI<=O9^tjuF$@tJ|9BKm{^I_s@le~i;LT-|%GZ0u#ei}!?o(@N^ zOHQ>?dJz9mqrW5*?x@&g3p+4kh^wlecohV(A!s&K3H`kpr zXU-4cpZL^N7e4(JphIs67|NaxQ3)h4QC-xjr*g)7(-H=2lL zNShDfi2CL``EfsWy)~mdg7Az<-lDPQ`pc3#!v|br0S4lKo^o?b2jfniMkSl1M9~A$ zac6+`0`8fSOpt7#$XoKzXVIea({Ft=0P6!GETrDrj;ItRz*bZHkARzkm5ZaCeE}F> z7wa4`d&aI>j!gy^e`@V@cyc=c2#OVb2w?3Vv(p7=MlQ%zV^v%@NfRPjI3r>9Ct(dh z0#k=_MiDb7yPTSb>RDGuM}Zq(ukb0prTZ$#INK9bd}O1|xzR zDxgb^UDr*NLXvc%p$kF>40uq@(9}PV|nH zS4l~ui=#9rC3UkkXqrK+H!$rr%sWy{V1pPr>>Xe2VIX&h@C4|f@YFXzYzmyXFVX9g z89=H04Ni%TJgW5E+(y{5Mf`+MSG{%X_0$W-u0}b34-c~O0t`RD^i$hEzpE!pryjX; z{CEs}|5%J65JGa6c;&c@$^!OjE%Xd0TXOk10#B@|>)^M5q@UQUPFSV{q=|(s2yTVu zbc<=IJ7KxP2+^Hi`!X_F7WVlI7xWp#AXGWp%|7;q4H`K5lU_5B4o7~x$cuRcBa;nM zJ`;SY3u!>@O${^{lOv8mRhAY6Wc=_&&8`4n+fVGg?bokg%`IDeXhjG3W!2y6Ra0lp za-@*H3mH;L>KgLF9>r%Z@smGYQSF9qMQ8QEfusRipz9{|XpY*+6DKx`vkj?^9}IHAM_#G2^YLCD*6*fjh(;_A!_gFy?ck-<~{y-d~#Y0Ope{4LBW7kL}7=MT2VN~ zMyFUyvZ`hK+5=tUWAR4$^o47^v73JJ#2V_UI{-W!E#Y!{XJ#JEh-r}Ln*Xe5R^e0s z19sx>Ta*fJ~MmH(~BwhzC3hr&6)n&S7~*CTu}Is#5(c-C>h=SoS@oVZ>kj8sJi#U8)x8nDvSP6$ zicW&~oG^ZT1l2rM)<#ZRvW+(rTwg!t3YXv2=KwSq$ascR_E?K-wuMNLK{aEzA4|n|*b{dF&m<-K8 z4YzQ~l8^b_7cO3GzzHdZ1lop3iY*`&EtEF##DQY~Ia3oq??_VF&ZMs~|85GP06p0F zIPW8Q^zK@td%zvn380_*r%gZ&G_5*N@@DMZ-1Y#=wK#OPU-f7=w@cTSuDO^qcGlao z!O79XCs?vhRU0;pg(^THaEJqg3mn%)@E5}c2?b{z#4zZq9(KqVTZM(E%I+1oWfO0H6uCAT?Y~OJ5 zMUXaV^eAsJ`< zT~ZQf%sz6rN#fvLVPW-UFIjc@PRBKaI!N{2!x;aEJNN75CEjycHfR6wSEE{ff)#A^ z0x8dC{y2Af_sj5B9rXp z?7}I~u|o&#&B8Px&EN!&SOMtFGX>l5~_pTfFQt9y0)hzc++gsU`Z29Iue|GkQQj3aMQ z6dVA`xI<9D{`~SHo?WCv(m}6d2jAX1eEe0FUj;6W`ieH>BKQ9H%>MkU8qQe<6&L%Z zY!wrCOtm`Zw)%jK9AUMQ(m=oy)k||f?AOl=OHEyNrflY~-AU)#&Wi7kxpGBl`R3O% z;(sN#pI4dC^Pzu}KC^7&dhhJ?vtO~Z?y@EQP*We7-KxzC;;QA)0m#i=GXHg1nM~l4 znvZ+^K%HQdujg}z=wm!mD0s>n(ne}>#0G+{qh-43tF28h2?8I?C8gy1dh5{FA71-4 z*woI_SHV(8$oK-md=`a%Bv=KVzj6hq$gba)_^_Kj6O)3i#g_PJ_sOh2UMFJuzUy$9 zhUEGDUQQdeH8q9Bp$;c@bZ$lUhZLhZDeZ4wNl#ZO#2SrNZbG~ZJkF!$!%bS&R#5VY z@=u`n?tbx6FyQoH46rwrR|*?u&;F8y5Joi_6E-4G1XLssJ;nI#&68 z_PsdvWwzy4pdAm`ctrs9oG0kuhh>L5uG!eXJ@jgB%*j8CaF{$gu`HqghL^{V98t_z zwW2Vx@=^3f`jngxZSEBK>&$z&^V8D+)*dTji|N59gy7c^2#+#y0V(te=`f!mS@*zq`R*B;6JA>1&z9~_Zj@^NDl zOj8VVb8>2bPE%g)C zHRH~mQSO6P^Dp~9oR~Q=#4=~BYvHU7$DnnG;;iS6}_32Cihnjd+55!N4ZUh zv^h7lb<&m{)UOW6dDWxb@APS1ql2cOb|=5SaHjB+0~lx>F5{cg1=H55AVj)O7>5Le zqdtF`Qi1s5@{f0&1X<#=v7u@bZiI`uZii{^Gc2=4qI;(*KoWE!J#tS7mgAlk9wPkX z9!3?k!;`+I^{4^XFk{ZZ=RxJ&nAk5}8a66b5|(W4IBx(@fOzWnOm zF>lww0VJRX_pLJ{uO9*AGfp|*XDk*7M}7yda=*7{xAJ^9p6EAYR?4V4?NTiEatYpAz zl*Me&cGD0lwt3Z`P*pe*j3dYfQs~^Q_@hS;@QEr0#U)zA`JJ51d18Tcdr-K0eo>JT zY=0`)@0fApMgi*4!*5XJOUm3;qL&SU7^S6L^`|BjobCwbt#6;64Mz&#eJY}e40uI8 zA|K-c1NL%wbRYwWT+hCDw@3Num;)+NeM@CRm7UMzfkEKtTOuP1-la^s>x@{eJ$kHE zR>HBic02Fn$NYD*$!rt$(5R!ztu{2;nl)(?(B95HhsDDffx=~1k>!o>Z;$WbTZ!Th z=(qCMb^rw{4spn&FuOlq$IgCvXiYKcIouj;N+ZkqZ7h9HW%xaE{n$sF)|XJr12pFy zES^MjaRQ5jaLH>1fjb-!pg4N`_?*fMbotHp=6&iqH%t3YWSK3O5TLEgSG9XF%hucd z`D*W<^MAi3(ZkmXkbw-exfISeQcK@J*f_yv|JkL({41Zddb_lRLDTw9STJZaicBxA zBq{O^c(T+?=%xECk6~XgT)MP9G-DqYSm3AFksWHje^;YMV!xyvnHn8pG4Dta0ac-y z-pM^js)O2uap&x!C=$BU#KY9YBvvdmEAM^%`W4)K(#*<*xnMK6 zN=ATU2%bYBdI_E37j|~ntw)h_!M<2L(tgky2Kfsf*8=Q-cKkWS2$u+I-VzmMZF~29 z3P?TwUHYBR{&4Y}omFr9lf;vCn2@2Li>gP|l9a-k_q%(4>Nmmw2d0#;m?j+VIPd-O z^9Y!z3kGL?LH9?!jtz&(^qx7hke2p@PTYqJhrtGAK_c!uELSjglZqfu6s zRI!biznHT&vm~F>bjM5FFnmx)%eE&cbXjWB#kML$RlR^QEb(B0TXyeGu(JX}HtOF$ z2F!N`h}F&LD`qq_(hCwklkDH|A#PCzjy3Y&FAW=cy>K&3y&+R0&|Zfn#+h9N&9IP2 zBq9fEZ`(U(KV1Y~SZnsbTVnf^e%V@Q`$fj=K37)YT4icu(((>=T$_oV$HhC=et46W z?#)~mNHoLhCGRPF1^=TEa=_r@88}P{fQp@ec;cP%SC3z-bx$iii_EEQ+qNPWpan(% znLp8@Wy?6x-Z81h0~8e{V^pDrdJZa=52sq59^Vk?6yu;zkUfRA7UoK3M0Y`k{ z7VlaguGcJQ&RcD)4anUj+6>+b9ye7l`1GlyY-Gk}EJlO`ddyPD(d?j@WY=JC-gOXt z-KJr}C}EiF-O=-H2k{O}zT6tk;&-~y3+p45Ecy2A+JCf|L1q0U&it{PbUOcIu27 zc91D^t2^K6x991k!PF!(C~xHO_`x3BVtRN~3g8#P4eIFV!JIlR*U#PPyvlU5Np?}B zv6kPqt+#Kwj_Y#A?QKj))i7IY;s~bBojVM^hdkZnvG;fQ)J6E{1ibNdS^070n95Qo zjDW4Qw7SML+|>YhW6PdBPrS93shgf05x)sQ7|>)VdJ5UoZs)EXn9u1gump;tH?+T_ zl6Ma)1J{%;3xM@Q#JFD3F+-4I5{EZ}D(qFFSyEr;w2XgP`laeg&8DmHa4Yv8Jxa*^ z*KvX*0_cM44m$TnC*t^C?~B=&x=EI&tE+3)!Re0Z3iELS^c~~aW&fOzbqrlnktP@_ z#D#K3({Kmg#nsL`340r0Ddb43tZI^i$qXnT|wZI4?jjcFlyCVb|l zt(r_;$Ca{9>YBu)Bm?#kGTi+E0ReMM+HOUtN`T*BITa`ZM>3KV#qJXA!=3da*De_F z_`#g{(?4UU>w4n8t9Jd;g(-_+-iO`4o9VrGzx9_}3!f0j2bjAVQQrq*CF!~Q+CU;4 z0%DQzXIuq_G}~mxI-i`qh2{B>;2&=PC}?SH6P;X}hAa7f|E!x!%rNG#IS)>ir1naqUXs7X6or4__P6!tay4aipE#8Z;@!SO`&Yx;#dXs zK0PHviFiNJ0^E$gex>l0NaILbk}x$kvzog27)5Xl;`6+!1~Pe?Gzir7))+D26C~66 zdS7rK3J&|i_X@{Bp`<^+BG!mhtzIhHh$(bzOsHmn)-cg7dlico8 zdn`e2Ubw-#HQ8p&KcH+^Ik>Jss$MYx1h!GOv>x? zR5Ha~Cyd`B@Vnr*Am-26Z|fJ(3I;;n9pvug3C@)%Pe5jw_wKcTlW@CI<&LP6f?C0P z6Zvn)t!vh%zgmbYB!oqx{Bd0v29EmYeac(HQ4(<;2si?mQCk5bCC0{DwUr0Lqe28R z|0e*R6nf}Xic(Z2UvSjjftRbVs2^;N zo9%$Os3`wW?zX;J=+AW<2j_T)z(p%QMx6kGQ(1nCNwnb{8ku}m!}@jcEb!|+?~0Ed zMACJK`HaJ9qU!|sTDGmVPmWu@>l-|;Uv73fDHL)o9ngLG)y zZTl~`<8G0Cz6Arn4Hm;&qq|eB{~&(v4YiA#m)q>?hgbI71Q$lQb3?8J5$%&+7RKB4 zp~6{5-s>6PQ@9n5|Ei;yyLCNO>jHE=kvQ6@9K+SH0@#(9jWVK79!h+W|G$OsUk ze*S_5tHDA>k&$K>JDb1ztjBVap?Sy^ zGHGl+^IAH&jy#6M;klqCd^;;!(cU3c#Zk}%6PT1e@cc+!Z$?uownh_TCW%D57OeXy2c4QYlBPK=+2_j6Y~TQP0!3s zjU6UxMlz@?;I<^>?R|5r-Xazetp_jH8dX1NWe5bc2W0o`R^`?l{?rn6c{Q@_N4UC% zW2cr08{(er)p}#1;kWT?Rp1%VGf$QtYBi6AIuL`4CxqGhlCBgZ;0$M~*Tm)EK3~U@ zg?iCmzd#TGDks>VD}=aHN^p=7bEJx{Y16xRGr=_^^p=WnDAK$D(mP!-ZZkK(i-u3l!ax!xM-4Ks z)gYG=nJ48H3v=C;!jupPPxO~+Xoi5od9}4I+cp6)6#|JTCsIu1T;!i~^XC#&b%vjFsYX6aAz2*_8{`A)&XHG5d zkvP2@^b%)-_)`(*()rPPl$4g%dDi?# zg**+-&WULvZ0ve)z3^sGU(`|a$%|uXi?Qp-@a8RBO3vRksuS@%$T|=g4&tCXN~;rJ z`D`au=>_MUXtg0AnBig}AQ-N-F}9WKx{rQWg%(8?lNiVll_7Ae0}CTV**Gp?yF|gp z=l-W)vt03`Ikbs5Q{mghX~&`v_41P^ZAb+nD62gj)K=58_eFw9&yDTiIeUVB;hN}- zm`Fj=gG_{N)bhpY?0WyRuUS|$7n>ASDz!?$^ze4mZ*5;K1_gfR`=YJmhR0G_%W7U! z))+_ee3HR2vF%5+jk=!u6LUq-9z4?T-!p}zwzl@JxdV=e4C;;7K$QKW(sXdJmR0=Z zLMU9Ml|j?hD3qug_={7oz=%yg-+!x}0Tz)NQ!a|6oA)MstCp4-(4{6I!<8XZNHAF~ zN*{EW>2wPh61E+sw}*_WC%`N+WwGW_URR%AG8|2hR0UM_gwZ!-hOuOuGGim?_Rz6@ ztY{vxNZ^7(LS9hqpdqcLo^ki?7IPO4v|w8O#H{crZm}!;0L@H;jvZAw2Su*MkP&om z5|YbfG+s6%BCZ8jgiQYya}>9*@2~IXuzBQ}D}nsTvqnKUd((n?VoZaenvXSHWU-{~ ziQ1Zs@qa&z8FQR0QWfC=k*errRwLjzPm-HHfBTjNCP5^8{QHDc7PUn&dTDh@6e&`y ztONK!6sSWVq|aw{uEwA!^5~&?7U2uA4oOsB!=@AD=_CECY6xVT36+jYbK3eSU7HZ& zyJoQaeC7q=$_@ACg{GwV1r}%&Pxd=;Y_pw#*x)3pgF8$lUTz+3hInG`;VKKIJ+#(Q zdoI-?z^nhD>w{R1@@E01WMCfM^qyC%lgc$aZeWa>xJj@u?igG_)2&cYv<)qhp$4PI zqb%Q=kUBjA5f>+y5wE`MrFDua?(P|yn~=3i;qXLdX*Y~2^RWmXS|sHvm%%Md7xYtd znX%hHwOq8l#URdT&jctCeU1cRIYDf>hb3e(GcO}!|FsSqr3eF%|DXC1bc(TJ^0Am0DogpZAn)95OaGnVU|cUL~AHB zPJomj+NUftht6I{;GH(IKoNDqfDnz^_?4<6- z#(Y^|6eR@;I%>*4P;+DKN!tGOvR>PHqJ}8yeVZ zGbuTWRg>$93Xee^9H}ARK^iSutj>9Lg;7?(U~<#2yQ|I}Yn+)QiO({0jDAAY`5deK zCt~MLRlo=;Q_)SY9?=s)b>rigm>4f&N-1S2k~t0JP{Y7YxSIR@7CyC6HxW)y51$i- zd-E18%ztX~ztKC0{p3hOf-T4i3$DtwacFJoDUn2Dycgv>b~NfY*1ZaBi$wIaS)h7| z6Ps;{+1*NPAXBe%n2Qa9L*jK$ip}-v4|_3Lec$!1ulRMM9uBM3tX;cJM3yblHshL* zvmIBRMDyZgy)kGy(+wZ@S;Z%ixi{ig;j-P+J1T#Uq!wVU4y8R8X$s!=P-2ZP4tsW~ z;nwJg4T`(-d~^EWe|BwDa^J(q;vfWYEvq8K0rn6k2X5CU4M^mbvYo>D|1rw-?5V+g z&Ow-tWDTa#k5ij^t&|ZRS=-4>n)athPiwIS$WT{iEA`{-`Ap675*_^iOzOt zD=@{?+@!Q@?gZ;|rAfKMsrRAzJfFniYWj#?B)ju#&B#V&qra4(wX&SOgEo?SuLZOm zI8bLwOancSiBUI;jJscl6JR{u?b$b@882cHiv_Y<3~;ZJK%Fq{e)9fZ#KPFf6bB6%@^o(PIpT> zpx5iv%(};Yl2-gk8b0gzg?0D(gg4M|X}#~Pu}8San8&ld=Be4K>?-~9yQE8vmQ|;9 zLD^+KZYfc%huyBJ6Rk3`I~{hwj$K81&eSG+Io7Vp*N&qv4!L-zs% zjvWW?+_8g!Y7IgCnj8I1)2MwbytWb#*vh#1n#>X`lGYv5*=kdGt#6)O=n2UZ%puv+ zwI2=25QOt9$?KE&uH)yxG6Qj&X*Oxs|0)2c`Kmv^rX8!WK~}FsA;j_L0U|+vIdfy~ zN=>U*KHyR8t#yocgYmYzy>{(9+AFS!t;;rGeu7hrol8F^UQBU7jvl=)F0L-pd+ttb zn&SQ<&GOZ$%#!&AzyH=%S{>VA{QGNKRF%$HD^cKZQ=u_e!k~n;vew?bA&68Bwp{!k z{IhaE9IhkTV>EL0r@jc7#vFAzjI=Sv@L}Tb<%n9(@rR!1xZ#ihov!FUBj~mPfzgB6 zw|P$ND?Rt{nJf4dX!+XWd5J3aNtxJL+;V(OXs8>DD#~p5tVb{G_=M$KcASw&z4<`Q-J8z;LUjSP(l%*#CX zCxnB+aZmSEHJ=i;fB*C(Tnw9&CvW0wp_6jxU(O+>rcwv(ICogohjjzWac@ARpavYm zdky@s4mqBM*5|7^SF2&z|2ya+Z(Xj}-+;eKsXjOVM--*9yV z!H{+mJ*Nl|Su{#jr@3`WdJthb4>}k;m{F2Hj+fU&+#@}J{_6oD-%>1r2ETLbSyNJP zYtIopK!B3?F2A=04&?E{ETI|4foobbG9f-ba=Ckto*1UFVLCK^_aB{VS`#(LWN290 zRF1x9Kb`ZCsMBkIN3cSHUb|ksde}-bfgLa3eRSkMdm^&$frVcUL1KWYqRL*VRd^gx zo9z6fv(WsFFJJA6!_3~U>E0GC22ho63!gO~NI|^xC~skCUUGZq-@kty8vws(HEs2% zaYI|VlDEbND^qU2f6^g5y9n>8GCb&1a$RbBHsnRt-kVo%({)UYDY1ssoit&NF%9Y{ zLDD-6+Z|RLKxIR$zY88mP0vPGRyeAMwqfhfPnmdQdh73L0kO=%sL`hS3d23!U1ch- zJj#ci=i=o!&lSdPHEfyGtCBB4tq_|U85Zg}7V{!94Xugke-G6f%yQvHr&n#x;u$S^ zeJcY6=+h&P5700(U#S*iLrRdrbN!zr#Zn?(A=>~<$9Bq;=NE?BTUo`WU_?29{UZHu z_nv8QuANEive#QFW>xXJ0rR_@yPYlB)X1D(aHY#A6skva`t2T%UIkd>h95WVvNBp> zcl+pN-v}kUb0crQoc`L4b4HJ#Xw2s6+#-!4?dniXp~e&PU*;pFtaBMpJJ0qi*yr_g z5Tm7tZf!%T_=DNMKe-aGLS#aerN+7W79l)KAHP`%-IXK$vpkEfEd`OG<+*xg*~ARl;jYo#4A|X zXPihg`VKfDT~orOOIibdea?I|H)M^f_awO~DV8~C>+#A38KLcJ_t^JEP*O`;RC0Wv zjIINKjc|5WaP>-)#~0GRN8Z`DSP5P9qL>ZM6GCZ2MygZE>RoE{|p1Bzwj!Ft1j zlIlY7iB&Za!fy(w%L7uNHxKU$!KkmRTZ?LPFd&+NTc-WMZm_XK8-*i<`RlP!aPde4dbC(W3BuCF{wcWgLd&$Yp~TxKWbQ*fDIvzna!n6)*(-zrybtNRNIY6YFTe{Uvtc!01RdY~Y4xs=WhpXbwbrd0f>=o!8?OPR#MdkQ zGd5mv*KupHJJO##xfFcNDAtG5AOq2rNO9;Bd7?>!s`E5!Z_C-bv)PcLk5fC$^=4dC zAiV}qj+neS26Z}gs889N>oUr9_6T)X98J;`F;DnVQs+_kyXf`h;$&y`L>;fZ-&qAU z<8m$ovIe&Jcn2lfGe}Xu5|Yr^EMOva+3VtM73=|X8s7#wN2h$`vt6_ayL&BA0!zpQ z4R!~|fbe7_XI5EPDS$-({z>@l;O1Tj)qovm82jFcWq3_}R1HyUrO7`?)AMC_6MFLX8~bTRekw zIESe&60)+3Ht5kszit;u&B?iOqvrm7r10Ujrckpk@6tW&!b-mUX^A$a9yoOP@O66Z zVs2bb&-GiQqkFuG`i7dsEKBp)v16NE`pXDZrgz*}#0Ojg0R0+0DW%j7IxtQM&G3yI z=Q@+(WzI3&>I{Y<(yp6D7G9G^o1rX)Dj!aOk(LUq9TTONygaaFP}p@b~l;w-6)N zngU25|4+i@%aib9S!;T-YfMlT{3bo@R%T`(EUCi}A50K?h~C@7e1I@F#>(-Z8>x?1 z@n0g!y3W9(Y=s#WOEMXOMrtZk-L!NU8sa+Jz?6Xw#>S^`@?~EuOEZ^qyc@H@=C#+p zYUL$JF`(^G9ko+%>SjoP<$c8aU_#Di*BaJ(O)EFdq*t%;Gk1X0LXzLQal<~5NkH?? zuqw>?sAPB;PUwjq-G}s9(3vxI4Ba>uPz*lcYwe&Ud&F2tf4{%nOkUK4PBp(rI_n<4 z!zamwZrjcuC@+j_@QAU);gf@gsMIXwv@x-;*z%j)BzJL);y14~g;FGQ%xT!o%^pPj%7ItrUilpMeh zDlvB}IlLW|>#0?N0w}0aug4Z0KXGCjo&8Y6%>^0OPsd<2sGZJde)`3*`DnZH=-s<- zQP#+!>1JUum3za3-Z-&yxLTza9shd{>GzDa=^~oU^2aG>rdmh^H*Na#>%jI|q90m< zX|H9wc6qe-&t{2iYc^S#w+N9V<-aS~^b)cywO*}{IOohvT5-^riK2|10poqY$${!j zZUl1GSA2V#+@*2HE};RCv1W?*G{_@!KD>Xw9S?J-&YhzlW?^K|n(g}dK%pjGgYv+I zzhc>U3bX!lsKZg3E8g>yoK~*-Yox2Yzo5ECYpJ*M(!!%YOxYuHTEvYTC;?w!!p@NX z5Uq-g1tY{g+9xb1VjIH;Gl8Jk{@Y9>hQ!)>%~ZW)=(5(9zG}ccv=)LVaN!Wb+K}Gp z#-;D4FJE4{xmW7?@ci`CnJykzAf+ z2dp<2!a+>jTwYl334-`>@+3cbR;0R}&*H_gF`Yo41_s{npx#}_&FVqD_nom&5xaM< zhtvlb?Vd`n_hj%k?N+Tw@YjGi#tL7XDw3`MU5-`)vDFQq*QD#^I$6#@By>YNjS4Ta z0Ue`Iy^{R6+tRNo>sMydAsfPts^Fzf*k-5V*N8iYkx}O|?Buc&NQ#CP?q?l3WE_Re z##LoUM3va^S+#;RDqa#9Yu9LLhqhzC(?K zHlTnDrwmypGj(vVjtN(xqnu2!73}9 z`c&9j1!uQ_77%ciT2x9~u3>XzNql(?NsWQlEZ$o`QlFMd2sEIoJI|+6!ZNLPi>jGa z%cKBMZZsl6R@AiuZ?d7FB#5g}Vzo-B1yQifa2t29o?%eLxMpFt{GK5c*>dGlFL)wx z+unc0vQg7Dw$+C|q@WM@{dYnyga@Y=m5JBFnvYJjf+lW1)`~ga8BUM%2M@kqN;0@; z$_x6J21=(S=mD4@=_!A=mh&!OQsPkJZ87ox@BcW`Btsp*y_dDT|Eomx5=k;fx~i#?(?8$N#H7;%@mLX{8L<<)=$rH zSSDlo?17Ipt7N`rXeQkEv|E_2Xk6mmExA~B0DfurTV^45;7?=`#eC~xim?amxVL=+ zBmSPmlK+9>A#B*KoE?djJ*VkhZl)#}En2wn$Ne2|uGyU6>f@H^*#l;KnukZ_)0xZe z-TK0*HS>$LZ&pl`@$lfckzPH3lbn`7W8fAEcD$3;>HVmn$ z8>M=uVD>UG89H+Z0Rkknn zKM=Jta&mV+|CTs&^7Hf26x0?e%lY%?Eiy06ogpA4!RZ{w&%PL1bn931`LnO3C^z_N zwj)NUld2^r_C7@RiJdp7RYp_t3*t_%#lm3PAI5
$F`mjiUx@odO%9*^-S+GK(fVxC`Uk;b!O5MoT~y>6^y`LndN(ms#J zd4I6-Vx*!RnAA8s7j9@)w?$zOG8bit3I8;dZi5tuh7|i?pf^%2tg{`31i-5x>mE{e zwM_oN+lgSKIgfpdkKf;;d_LMt;zA;Vwu2o@Rvh9Y0cKOXbm$QIZjBM;Owo*e1!S>d z0>!OhunFjs|7MngsUI1-58*P^K0BkSI%4df3ByMUgv$?KF^pr{~31t%gpx^#L={DelM;# z8wOXMT3%DB99WxuvH1N1bsDRy7`%6VYcSV{j>m>f6sZir5T>B{Cv$4moVBxatzOZT zNJIr2li<87hc0dbd2wxP=^MahCa+dP5K+*o(u>@9@H9xRm?YV2@TW2jijOP|9ddXf z!?`P_x;^zrz*~J&t*WwXMXPtbb4IV(=%|5NyxaPSGBlKDMMbdh*s%tn*Z{{2VkGuW z8yK9-#9$zA5LtQH)Y?cPVoN9s|TR`ym`1*}Q^^2ALdQ#!@R#yX8t)FrK zLE?Wv>;dLqb1{PVfMZ$57B~Cyf3yHFvYZU$rXZXUDg-E+Zy{$V8HSXUT$}t4nS&>1 z&r!sjn*vD48f}c=xAO1cU#qZATZ8rCn~5qTyCLv+=7oBq?Ao5Qs zLmH)?zO=E|c}1RUATUiW%TIs1v~BCMe5~91YZ3a52X<-N&LnMPFXv09{r7h1y?whVf2!7U_QQoIQQ%`>PLNCSM*B~Z)lFA{vX!nWZB^* zQXHM8VUqrXu%;k({qczJir=rg)x4a(yuWIH>2tv!A^PUEgDko?#0}|d94z%lr23U-tk{b``FuH5PXD)AV>%rDf zqj=i%TrvZK1R)!A`e3)$$a*!OlCGGrt$jC&Y%Z#mhnuyNlqXsksXY1p=iVsVR(3{W zfe6VIdgo|-=L4gOgN)8jrd(pT2A5n6Lzt6#u}PLlDg|pQC{U*vJw)uR(CGbt ze;LKZNny`EHEr49h+1XG;W3sOyzJ9_Tehhf?qM%6mu(eXXc-Wf=-SYUaeIm+NWh+* z-5m?J2CEfXfFES^XV%Wzbzb8WO&v= z;NnlK{)Qs}Eg{!A1A@5vgqxpy7{TUFF|!w2R&9*&gzidoY(8^JSY~#|sGkGT%G-yu z=z)qJ!QAt*vU;>w6SfUmI}z8+HBo4Z7>!RL4JwrunyL-z`h)X7-!U~n4=0ZweT`Vg z=@4>KW5*hz8R$0cRyrLWS7&1$PLJ!;J{A0(YyBsfqJ_lHzEm|L)DTe*${jP_=w85u2M^_lWbAP&$>hT3 zBVC-4d8wn-VxwkKvC3y0J2pBYC#pAiA=7c|0ynRwsHA(_)hSk?I5H?W!l>zF-*zk; zdQ>S4i5i~}Xn#D@b1^}h@}hA zH_q-0l3F~SPzd4P>E7gNd|c}~Or#as+6}1e zWh6KjL4;(7^1BS7wPJXKvXnMlQy{(pWI5HlpJ>utRU*&q=QSnBMLw%0W9WZAHkFTq zT}uR&(y{@PN?e+J9C|@7K8}-K%q}a-gi|0OqLIYCybt|xV&_e2DT{cFA`1{5ICPRM zCPq0gN5ahj5;eDZdSxQ2#?C8Zac)1lt}jzJfj(603&PzX2gS`fAW7oo7HgZ^h|v4id0(prFb3X$X9LOZlA`G0T+>MB6$|}7zHYRtihd6c;()@C5dkUF7uIQ zArhYglqQdF!v-hEXltwBh-%Y5eY8kA7(y+Ty7=uHkzVKJopk!*X;^9XEA%V)faI^E zc#;B7GSxdjH;iT0i>}D6iN!~cHr2~31DOWc{l2+wbJ*%lMN7b0EOHwl0?D^FM|pwD zMv30VmgKTsfxy&Zn750yg(J;;HMH3XMT)4W1#6NH#A>TWPuW; z%49n>kqGwLKY<)?>1fwE_vSQORnPc{T45pUMO;Z9>=nTkxI8Ktk|vc5ZCz3=IwzF| z4K^=!L9I|27n}QFg?`Nnpxfr*>h6|p{}Em zdjJm4-%tWgK*`DD7*iA%*(Abud~$M7ZEu| zKxMVIQygh;N~VWZrVHz1^E4zZw1wO*VvmMQW+Ey_>MoaZ6~7JC(F~$_E)mi4K?w?z zQLX?EQS&{JrDg-L%%fAsyQ2)E)47>yY3&jR z*DQ&ETxq-@3iCIB|2!)uX!L|n+cJ2D!8}ua{o33F&|XTPe|dbkvoHpOf3Hz zI(${k-9rAc76G> z7@5Lz)aD^}1~V56XCS73D%KSZ#3NrsMI+*f%B{zZyNHcE_jJ`K`CKQ(9wj|EM<6vV!<&}`p6SWT zl5QOhG!Ccs2OXY)U^k+zGR$M^v0e(M^rLr+AA4Zi-c>Vjj>vRuskG_oaB8gi&to)n z`fB@KxZoPylr1r4L@Qq90|IOblk~dPDhyBR<-Ni|b3!6C|9#G2>1Uqr93G z)G}e{T+k=mr=l+qluxNB zb>_>8`7f^4XU=Z}ez^_J3l7uTTt$AZe#S&pGvHZ+5)yZ+*kxsKFWoFET2*L8sZnHS z@t+CX93CFwE|AiP_H;w?QEnfJE+=2&Qp2O0`?AF&ml6R`kH7Hqp$|E9FscDSoX9WU z@2Lj!6k`UuI9;l8enQlBj0}*>i4dr<-;UO{l(6>De1I>3%TPGP|Py@qwsr<`ikeDU#in>@9H({e7s1tx#e%`L_Ex+UF&Gpe<$&A!Fbh(?(EV zBZ1sq8Za5_9w0&y6Op{%eC&(6ve_F*4wh}Hkk)-7F zY~uH#oRbM6=U4pfA+8I81*LvmZ_a#XKH}Jg&*%O#K4_zKyqxkZ>K4^mLnOsf&PI)A z02}`sQo?Am1FQc2d2|0{J6edRzShG|YjE!%s0Gy^t#KWPNRPMAHmmtX^Em1ovNEIQ zPK5N0DvH9+wI2?r-A$86u^U8~&w$U>66c125nOXHk5SWeJRtxa>t1dLE&>ItwG_1k zjFbLZ;z8^4ji#*M&&818(}_CjpP~y|2h@zJQICjOAO~?q0k05Gq=J+lwRjUi(!j7V zd&kSi$Hs%ZaS}w=Nuk`-|h8t!_&SoGB2yihl0BgE_ z_}VqdB9uduF1DWRI{MDJ&n|Dk*2Rr2)q{+HLHC7^Za{W3o!Ad+*RGXXfI(m3p!W*j zKh?zAwUR4FM1epVH_tnySXl+D`Hh<}p@UDxGFeQOl?Izo63D^A?_zP=|H$cRY#d;& z499iDDjCT~>XHu*PKERE%r*mZyAB-0u_SZrtYMX?fP;t+V-Bqfe}N$5VO)(8Ejb_Xpe0=pv*L?-HI>j67xi8o*IvA)|e+Y_5C#XQ-WZ?)62ZO z_`T*?pJf6jSBQnj%UuUSNkThwAaE7*rJoH1qn8)JBjXg8vq~ClRij)3E_95nQrn@y zleu>?gN0g&Q#g};Ql``W&>z%a6KH|zeoIgk#WcPHzA_XV57ut3L|m=hprL!bE-nu; zDLCdt9X6tqUll7QI}}}~D&=F;neUuEa$}>*l86Tl?{hMvkgGs^s?elNoD75`!z|~$ z-Cb`R$F#V83 z4p_M!ErHB`SZ^*8V}x#Uxf@h3Zlmp;BWXPdYhKc03$y6cyIs?NCs&rH@e>K9W4nG?0`0_lS12Gzj^6S0&c=KhWza$9Z>_}`E5ucY4)$HmucV9YCFpLli4;q+# zJnqEKZ*7KvR1^(0XzEViFVa(Cc6=*5W)AYU5{D5_U!i~$XC~6z3N33J8`&E?MPQtr zM(Ufp2{)-$>7l7U-~>@%3eIh5sm6{SD}4yPA=zll_HjAF#>Reh3Ch;=YzY7J4>4f$ z9`!TOWU`zE$H#Ega*BxAMkIonnKng+H`-9|F>-Go^aPE(fB6Ri7%1<%*6*m0_!2N2 zlADxerWy-w=sTT8(1oHYY1OLL$Xd3`yR;qCK;pOZDS~+M_8+fpLX$16LgrFy6~8?w zOL0`c9|%N)Ji*ItnCpT~eNHtkbXbAN292zxoD7^UIjOA`0dP^vsy}W(U7hlvoo7Ru zOufh>J37spz%e6n=lldgVS#OCPMD<&RU%ehKt^_?mgQ|I3}@pypMNPW!Gz3b*$<#h z1-9E@{bHm8N=CX1XCjmlB|fOToo-w(^>S>BR1hM`>}UTc?~KsfBvIweznrkGz(klJ zV$k5Y#X(CKo7ma@1YdvusZ(k!LYZO(=7sE3B2Bnc#rrj2z~rXZFmEzg9U0FR#K3e4 z^Or8g>DQYuDW*`^pE=X4vX->4AoOzP2nt^GDyg@$tzJ?F30g&>#pUK+5|P$U zW=^u?Mkm%m%{d+ir*V0GLg#(e;K63+Isg9d+L$!E}kucgw_df-lY`N;&`&KInY7q4S&lwk}-03`l;(aX~mv6X9 zw{@EA0h!_l9wWUV>VAfz{2JW7yI9*eSoecuGhx{-{KL4haLeK_lEE;5LTqqfem)yn z>XnNZEu|4cr@^r&aK^=G>_t>~)PN`p`r!hWIXnPR;v++@!A_<7i&MX+qB~MFIuIiz zsAMvDPu>F?6z>avTi@FWu%1zBR5UVql%G%mAbmRaz4X$_L%c92zIfAcA3Gz z+owLc@7Ap;iT(UIcdizR`OpOeKq<_G`R0%jGLYW*>>psi1)S9)jh2Xv)@|AdAl<8P zU#SiO$8r}kcWOWv<*n4%QW=2>YVWru^*{DcMo;#G$E6H`;8#k$hWc{_pYOA>FM&?YXG{h;r@BI1k?>syf1e~8=qC_Y7 zXvUmRosLl9yr80$cm>I4zL81`4gzY8$%eZYwL$Pzgp=A zUF-tfb(OI-{}>(y2&68**>b~=L3Xp4xKfXm&mnu^+PB_Rk& zw|`Xe`SXm$Dqbj+1TNrWN8K~?^P(X`uAe*cdrojjh>`*oa9lK!L?U0+#p+HHfN*Lz z^6;4D!{Ww&MJFj>BBAMw^*DL0^PwUreNq)E<4jtNW>4ca2!kjFl@v_?n~8dsXcY9$ zFz)Q*|2aZgu>23O1_hej&cgVk#{k3t3IxNU9-Esa!||w+dz@#>c2AwSyjf_Icf+hy zyiH|53}6{Z%X9P#uvE2vXz0YYD5nao)@9b@@Mw|Y4bsqEuCuj+Lrwmk8 zOhMknOpVodySM!tf`J{*g#`@w(t~p{cIBiA6N=xIpP%^gk;dPad>@!$0klj_k1G$* zICp>$8b{On_wUbJY&V51eB)ZNY43NN!Rcw)PBR#onw;pQixsmVTle$rQ9yuBg8ZSr z7Xd!71WE@!rGOvU{b(8y+Uz@cu(lW|ImwmkISsT|sZq0Jf(}5L^qE9EHbx3j9;G71 z$5DWUgFGsjO4F8O!K}?@6`9D46Mmxz{vmLLp%MKfn>^)i>PfOIdP$ov9P^>!dmFQ` z5WP&nHY`JcpB~!w_UHCawlpn}NxUIMrt{Rs(p^%LK03F}<@U=8ttXREIVzIAe^q0P zQixyWvH$-`I3ULYc>6#)01BvNoK_sKuUl#;JrsgG4g)C_;#ZGbJA1^gzPiV8A2g!A zB2DFi$4?&JAWrjuvV#UlwwuB&3yA<;99MKNKcA_b$4me1TOc{I^4Q01_;{gdX#-$S z40V~%_3rpA|9gL)Zap9O=2i8$|Bu8l`o|79N-pHsXs~G&ih3+k^9qWZGxdq0jLx-a44CCyyAmeLiW0chj9k(Hf{_)s~Z))Y!G@k3Ppy5{OR>0H{XW?EVgr?i9bj`($vv@TP()GeZ@ z|DW#xcTXqlsFLkMyliz>JuTg!<=M10 z(rgPQ#I65dFI4(f`1YciJQB_BGnor0te;r z53*L(4YDYp#r&VURyqUgb-VYqWCfekpJSDf>A2|n>xtv7Jj$-Y1&Ue`(DAxoLLyjn zY$QK|JVX(MNP7>R=R7*L>dV44F5JDjtYwk>Le#frXQ5@Z_4baZ3p6E2K?}wp7~y=))6pSy7)}Da`zA^apx3Y_*D*K z9GwEPpZWX~H1-p=6~AV5%s=@K$UeL-3q)EW^2zvcp-QaCi*>U!#nCKp2kuyic5qBD zx0c>U7Vk84+mRL)O6BxiNO-`0iCsaQE$LD~OBMxV8i0?>;0u`&POGAi=?vv~Ci*C8 zfd6sGc04L49W|LAF4`c9{8P+#ku`F^A8&X0b0n@Z^AAuKbfzVmVrlS6?W)hIojT=0 zt5<~30kfhg&7cBDJY0usmtxryQ6D>SOM;_8P(+g}^f+b==jJ4T$v(nv8R=smTO-|} z=?xNT5VfdLn$wkvD;R0O|B|?r8)*g6R%NFSO~LRQAmq(|urAdanng*tV$UFt+bFSD zD7My{pHE3~2L_RL5Lsqfu|t>D_*5}zy-94Cc`Xi z3I#*``7eGRns~m3hD>T}6gF4zWFuq_#oRQ71tsxIYt~%xU%r%&vKOhCIdUk$uc?4+ z1`Z5h4S#AUpX3;yM3>htwL2IQ?ZbZwg^K4jb+({?vlT|>wC9i;xj1kB<9lY0IbQew zYU$d;q0HOx6zM=1G>WE}bWpOW&`?_%)NrL-OB$`P6(OY%)z?PGX)qxaC5E=_WVcCH z2bPL(No&@Y3aiLhQ=hXADtz~&_L^(|s<-$39iGGe+|T_anG-wU^5tnD=F|LI5Py&q z7>S35h9)t<4hM-S7CaZDGpntwR|e{yJbhXR6>=R%@mS^i-c`Re4*Dp4tg%+}@Vs)R zDecbvCwbjwF^0Q-n*(~n14g1#7*vYR=x7$h4-UZ`^)=3j8>k#HK23jcs)CqpTA9!<#n^~_Fo^q z70L#X5Ox(^%C!p(9bR*EE=Q7a1!y#iPaA;!5u^!ms7>sI%&DRo%+WQ$W%?Br6$r*L zG+F#nsuTYFKLM*=9$XEr=DaOS4AScjvKkE2>oF2%5(7#xryxn$ zL7kT17XB~X23TlyE9yEg4p=kVo(guCJoUz}RA9bp)fDNY899wAqMFGiPfb`lmzs^o zWnxvEz--w9Wh~qokd}UPurM^dw=z(o@RZxuI zne*pITg3XBng*=-7hR261{wS*$qoxxzX)$S6b#grSVbG;LAVLX7AWI2{rzUp(5e1g z(yS;<*~km+n4_jbyF5BNN~t&D#zhO{gFkIG^@4%GXpr#|@j&S3abU9>0DJ zHStF8nQJX)@72qi%wlDclN0JiDpRM*Fr$ka%dkFs_8LlN^}(aih|EKWR?z&M^7qdI z=Y>5?&17Sr<@}N@0_fcn=>_Wp$Pwt%{E?B7h}ZdX6Q9*W#U7e!41M^M?$R1?BpQ^4 zlqdepAS635jLdC_x1*tkG^k>KwlnA^Rh6L zK{b=5Jf2k8+H)U;oULf50Cl>cqJqu8s0D|(4mz8-xA|L^6sBzA#V8beX)rLU*uZ%o zKuVEopFdtLvEyAAU=}1H38aPs^(a)ssZRx%45U}ungj1@GTUdCwI6pbu{XNk zp7qO3<+36dv1V~3rQXMm&7~oPWZOZq$!Uy5kRBrPaCJ@lhne7?fhITD^Tkcio+ZA@ zJ*p1lx)uaAoU2?B&Xs?$thEQ3D{Gr479NF`C_s1^<4IzN2!dQ|jU;Y&cUio^W>G%| zy*q?A0q$>j$#qh0?dCQGp8z*Qy)P)_VJ9#f^E7u6Gm@%d$A;yO zRI10lB=3*NC3RWp@L?GakbJyR5h^Jzo{5DdBt%U`34O5|6<%Wa0OHakX#h#c)o~$5 z19x|T3K5v+a{>#FbTwt0SETTQ<;HOx94clzyX0vSkaqaGPN4WeZ`Sd6F*odkz&-?g zorc$0z9PLo1{4B*OBQKXPw{pU%xi-MjeZ%P0r5AA;19T0)VPj4GpS@ z^z4wjQK^d|VqFIr3-lGP{tv-3O=Oho!UNNviLS*q(8Dxji;6j=Gg7FG1@>lOhz?xC)V$ z2!U<$iOYc&|O`I0tP z9P~K;|z!Py|Winx@tsq;M3cS*)y0i_aXBOxm#F-AmX-j+EepLu@sVpbOB#=)_o`Ooo9 z9q&UU4rvr!4ZvSOiy))|D1govdbLo_HV)%SZr`D8m8L%patI8RS_3^YTfKTZ*e^jH zf3lI=@36r9hr$%+Vex~94<#r8LlK6!i=gsQuN73Ma9ulPwe!0E*eFOKBjdjh-$$s_ zzJ0zQHsLqno`(JY0gZ4Az;VCj1=)j0^?>;1ljrAv`Cw zHDDrn2$D&sbGez#c%Q7G6)R_}UG*?QA28G|PDiYL136`)_n;-iAZ!=Mm<%DaotkQW zqad+h=jXSWglYf%*9&wd#$k6KmTs1i=vk$L#uBq-rwYKq5 zf35Z<d~c=bQ?{Zj3WaMXmTaN8Bcn`8C9t160;rpD1Z~U&sFV+izA&D zmrlQuaNLwTyDUhB8=pv}G$K5;w=ZYO@y8K4tdl;nyVPt5)E^oJJB$r)*^t@ga(a2p3l zM{^5{Xc#7(j*zunZsLmx3)Qu%9_tGKj@5BahJ|QMBC|c z>@(y(Gi@EvvI_otp1U;PwMbJ-YjU!K65`rY!E%;%vcu5tUaFi=wRD~1AKxO;Llmn} z-014+qJTI%JNi+;j~1-TfwHgRvD01;_3`Z~-{StP1_WqG0MP4Zlvyln#d4zfobN7; z+#FLkG_eu#idA^s^_qxz&8@7aGT@-f-*)UQ=W0U@0R4xI=NT;tk25=N6s6%6dW{kM zhbeOeQzD9XK-vVh-+JN@@pvyoCSEc=zOL(wM*e$$xh!w(2L}GzWWSmBo1J&`{{b9F BRz&~+ diff --git a/examples/surrogate_assisted/apsdigitaltwin/util/basal_profile.json b/examples/surrogate_assisted/apsdigitaltwin/util/basal_profile.json deleted file mode 100644 index b4333d8a..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/util/basal_profile.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "minutes": 0, - "rate": 1, - "start": "00:00:00", - "i": 0 - } -] diff --git a/examples/surrogate_assisted/apsdigitaltwin/util/model.py b/examples/surrogate_assisted/apsdigitaltwin/util/model.py deleted file mode 100644 index 52b16dc6..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/util/model.py +++ /dev/null @@ -1,235 +0,0 @@ -import pandas as pd -import matplotlib.pyplot as plt -import os -import platform -import subprocess -import shutil -import json -from datetime import datetime - -s_label = 'Stomach' -j_label = 'Jejunum' -l_label = 'Ileum' -g_label = 'Blood Glucose' -i_label = 'Blood Insulin' - -class OpenAPS: - - def __init__(self, recorded_carbs = None, autosense_ratio = 1.0, test_timestamp = "2023-01-01T18:00:00-00:00", profile_path = None) -> None: - self.shell = "Windows" in platform.system() - oref_help = subprocess.check_output(["oref0", "--help"], shell=self.shell) - - if "oref0 help - this message" not in str(oref_help): - print("ERROR - oref0 not installed") - exit(1) - - if profile_path == None: - self.profile_path = os.environ["profile_path"] - else: - self.profile_path = profile_path - self.basal_profile_path = os.environ["basal_profile_path"] - self.autosense_ratio = autosense_ratio - self.test_timestamp = test_timestamp - self.epoch_time = int(datetime.strptime(test_timestamp, "%Y-%m-%dT%H:%M:%S%z").timestamp() * 1000) - self.pump_history = [] - self.recorded_carbs = recorded_carbs - - def run(self, model_history, output_file = None, faulty = False): - if output_file == None: - output_file = './openaps_temp' - - if not os.path.exists(output_file): - os.mkdir(output_file) - - time_since_start = len(model_history) - 1 - current_epoch = self.epoch_time + 60000 * time_since_start - current_timestamp = datetime.fromtimestamp(current_epoch / 1000).strftime("%Y-%m-%dT%H:%M:%S%z") - - basal_history = [] - temp_basal = '{}' - if model_history[0][i_label] > 0: - basal_history.append(f'{{"timestamp":"{datetime.fromtimestamp(self.epoch_time/1000).strftime("%Y-%m-%dT%H:%M:%S%z")}"' + - f',"_type":"Bolus","amount":{model_history[0][i_label] / 1000},"duration":0}}') - - for idx, (rate, duration, timestamp) in enumerate(self.pump_history): - basal_history.append(f'{{"timestamp":"{timestamp}","_type":"TempBasal","temp":"absolute","rate":{str(rate)}}}') - basal_history.append(f'{{"timestamp":"{timestamp}","_type":"TempBasalDuration","duration (min)":{str(duration)}}}') - if idx == len(self.pump_history) - 1: - if faulty: - temp_basal = f'{{"duration": {duration}, "temp": "absolute", "rate": {str(rate)}}}' - else: - temp_basal_epoch = int(datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ").timestamp() * 1000) - if (current_epoch - temp_basal_epoch) / 60 <= duration: - temp_basal = f'{{"duration": {duration}, "temp": "absolute", "rate": {str(rate)}}}' - basal_history.reverse() - - glucose_history = [] - carb_history = [] - for idx, time_step in enumerate(model_history): - if idx % 5 == 0: - bg_level = int(time_step[g_label]) - new_time_epoch = self.epoch_time + idx * 60000 - new_time_stamp = datetime.fromtimestamp(new_time_epoch/1000).strftime("%Y-%m-%dT%H:%M:%S%z") - glucose_history.append(f'{{"date":{new_time_epoch},"dateString":"{new_time_stamp}","sgv":{bg_level},' + - f'"device":"fakecgm","type":"sgv","glucose":{bg_level}}}') - - if idx == 0: - if time_step[s_label] > 0: - if self.recorded_carbs == None: - carb_history.append(f'{{"enteredBy":"fakecarbs","carbs":{time_step[s_label]},"created_at":"{self.test_timestamp}","insulin": null}}') - else: - carb_history.append(f'{{"enteredBy":"fakecarbs","carbs":{self.recorded_carbs},"created_at":"{self.test_timestamp}","insulin": null}}') - - else: - carb_diff = time_step[s_label] - model_history[idx - 1][s_label] - if carb_diff > 0: - new_time_epoch = self.epoch_time + idx * 60000 - new_time_stamp = datetime.fromtimestamp(new_time_epoch/1000).strftime("%Y-%m-%dT%H:%M:%S%z") - carb_history.append(f'{{"enteredBy":"fakecarbs","carbs":{time_step[s_label]},"created_at":"{new_time_stamp}","insulin":null}}') - glucose_history.reverse() - carb_history.reverse() - - self.__make_file_and_write_to(f"{output_file}/clock.json", f'"{current_timestamp}-00:00"') - self.__make_file_and_write_to(f"{output_file}/autosens.json", '{"ratio":' + str(self.autosense_ratio) + '}') - self.__make_file_and_write_to(f"{output_file}/pumphistory.json", "[" + ','.join(basal_history) + "]") - self.__make_file_and_write_to(f"{output_file}/glucose.json", "[" + ','.join(glucose_history) + "]") - self.__make_file_and_write_to(f"{output_file}/carbhistory.json", "[" + ','.join(carb_history) + "]") - self.__make_file_and_write_to(f"{output_file}/temp_basal.json", temp_basal) - - iob_output = subprocess.check_output([ - "oref0-calculate-iob", - f"{output_file}/pumphistory.json", - self.profile_path, - f"{output_file}/clock.json", - f"{output_file}/autosens.json" - ], shell=self.shell, stderr=subprocess.DEVNULL).decode("utf-8") - self.__make_file_and_write_to(f"{output_file}/iob.json", iob_output) - - meal_output = subprocess.check_output([ - "oref0-meal", - f"{output_file}/pumphistory.json", - self.profile_path, - f"{output_file}/clock.json", - f"{output_file}/glucose.json", - self.basal_profile_path, - f"{output_file}/carbhistory.json" - ], shell=self.shell, stderr=subprocess.DEVNULL).decode("utf-8") - self.__make_file_and_write_to(f"{output_file}/meal.json", meal_output) - - basal_res = subprocess.run([ - "oref0-determine-basal", - f"{output_file}/iob.json", - f"{output_file}/temp_basal.json", - f"{output_file}/glucose.json", - self.profile_path, - "--auto-sens", - f"{output_file}/autosens.json", - "--meal", - f"{output_file}/meal.json", - "--microbolus", - "--currentTime", - str(current_epoch) - ], shell=self.shell, capture_output=True, text=True) - - if "Warning: currenttemp running but lastTemp from pumphistory ended" in basal_res.stdout: - shutil.rmtree(output_file, ignore_errors=True) - return -1 - - self.__make_file_and_write_to(f"{output_file}/suggested.json", basal_res.stdout) - - json_output = open(f"{output_file}/suggested.json") - data = json.load(json_output) - - rate = data["rate"] if "rate" in data else 0 - if rate != 0: - duration = data["duration"] - timestamp = data["deliverAt"] - self.pump_history.append((rate, duration, timestamp)) - - shutil.rmtree(output_file, ignore_errors=True) - - return 1000 * rate / 60.0 - - def __make_file_and_write_to(self, file_path, contents): - file = open(file_path, "w") - file.write(contents) - -class Model: - def __init__(self, starting_vals, constants): - self.interventions = dict() - - self.history = [] - self.history.append({'step': 0, - s_label: starting_vals[0], - j_label: starting_vals[1], - l_label: starting_vals[2], - g_label: starting_vals[3], - i_label: starting_vals[4]}) - - self.kjs = constants[0] - self.kgj = constants[1] - self.kjl = constants[2] - self.kgl = constants[3] - self.kxg = constants[4] - self.kxgi = constants[5] - self.kxi = constants[6] - - self.tau = constants[7] - self.klambda = constants[8] - self.eta = constants[9] - - self.gprod0 = constants[10] - self.kmu = constants[11] - self.gb = starting_vals[3] - - self.gprod_limit = (self.klambda * self.gb + self.gprod0 * (self.kmu + self.gb)) / (self.klambda + self.gprod0) - - - def update(self, t): - old_s = self.history[t-1][s_label] - old_j = self.history[t-1][j_label] - old_l = self.history[t-1][l_label] - old_g = self.history[t-1][g_label] - old_i = self.history[t-1][i_label] - - new_s = old_s - (old_s * self.kjs) - - new_j = old_j + (old_s * self.kjs) - (old_j * self.kgj) - (old_j * self.kjl) - - phi = 0 if t < self.tau else self.history[t - self.tau][j_label] - new_l = old_l + (phi * self.kjl) - (old_l * self.kgl) - - g_prod = (self.klambda * (self.gb - old_g))/(self.kmu + (self.gb - old_g)) + self.gprod0 if old_g <= self.gprod_limit else 0 - new_g = old_g - (self.kxg + self.kxgi * old_i) * old_g + g_prod + self.eta * (self.kgj * old_j + self.kgl * old_l) - - new_i = old_i - (old_i * self.kxi) - - if t in self.interventions: - for intervention in self.interventions[t]: - if intervention[0] == s_label: - new_s += intervention[1] - elif intervention[0] == i_label: - new_i += intervention[1] - - timestep = {'step': t, s_label: new_s, j_label: new_j, l_label: new_l, g_label: new_g, i_label: new_i} - self.history.append(timestep) - - return [old_s, new_s, old_j, new_j, old_l, new_l, old_g, new_g, old_i, new_i, g_prod, - self.kjs, self.kgj, self.kjl, self.kgl, self.kxg, self.kxgi, self.kxi, self.tau, - self.klambda, self.eta, self.gprod0, self.kmu, self.gb] - - def add_intervention(self, timestep: int, variable: str, intervention: float): - if timestep not in self.interventions: - self.interventions[timestep] = list() - - self.interventions[timestep].append((variable, intervention)) - - def plot(self, timesteps = -1): - if timesteps == -1: - df = pd.DataFrame(self.history) - df.plot('step', [s_label, j_label, l_label, g_label, i_label]) - plt.show() - else: - df = pd.DataFrame(self.history[:timesteps]) - df.plot('step', [s_label, j_label, l_label, g_label, i_label]) - plt.show() \ No newline at end of file diff --git a/examples/surrogate_assisted/apsdigitaltwin/util/profile.json b/examples/surrogate_assisted/apsdigitaltwin/util/profile.json deleted file mode 100644 index f471a10b..00000000 --- a/examples/surrogate_assisted/apsdigitaltwin/util/profile.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "carb_ratios": { - "schedule": [ - { - "x": 0, - "i": 0, - "offset": 0, - "ratio": 10, - "r": 10, - "start": "00:00:00" - } - ], - "units": "grams" - }, - "carb_ratio": 10, - "isfProfile": { - "first": 1, - "sensitivities": [ - { - "endOffset": 1440, - "offset": 0, - "x": 0, - "sensitivity": 50, - "start": "00:00:00", - "i": 0 - } - ], - "user_preferred_units": "mg/dL", - "units": "mg/dL" - }, - "sens": 50, - "bg_targets": { - "first": 1, - "targets": [ - { - "max_bg": 100, - "min_bg": 100, - "x": 0, - "offset": 0, - "low": 100, - "start": "00:00:00", - "high": 100, - "i": 0 - } - ], - "user_preferred_units": "mg/dL", - "units": "mg/dL" - }, - "max_bg": 100, - "min_bg": 100, - "out_units": "mg/dL", - "max_basal": 4, - "min_5m_carbimpact": 8, - "maxCOB": 120, - "max_iob": 6, - "max_daily_safety_multiplier": 4, - "current_basal_safety_multiplier": 5, - "autosens_max": 2, - "autosens_min": 0.5, - "remainingCarbsCap": 90, - "enableUAM": true, - "enableSMB_with_bolus": true, - "enableSMB_with_COB": true, - "enableSMB_with_temptarget": false, - "enableSMB_after_carbs": true, - "prime_indicates_pump_site_change": false, - "rewind_indicates_cartridge_change": false, - "battery_indicates_battery_change": false, - "maxSMBBasalMinutes": 75, - "curve": "rapid-acting", - "useCustomPeakTime": false, - "insulinPeakTime": 75, - "dia": 6, - "current_basal": 1.0, - "basalprofile": [ - { - "minutes": 0, - "rate": 1.0, - "start": "00:00:00", - "i": 0 - } - ], - "max_daily_basal": 1.0 - } - \ No newline at end of file From d5c70e592cbf2b5e1571552f4afd51f80219ac8a Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Thu, 14 Dec 2023 11:21:56 +0000 Subject: [PATCH 20/60] Abstract base classes, formatting and other comments --- causal_testing/surrogate/causal_surrogate_assisted.py | 10 ++++++---- .../surrogate/surrogate_search_algorithms.py | 5 ++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 9ddbc088..88480591 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -5,17 +5,18 @@ from dataclasses import dataclass from typing import Callable, Any +from abc import ABC @dataclass -class SimulationResult: +class SimulationResult(ABC): data: dict fault: bool relationship: str @dataclass -class SearchFitnessFunction: +class SearchFitnessFunction(ABC): fitness_function: Any surrogate_model: PolynomialRegressionEstimator @@ -78,9 +79,10 @@ def execute( print( f"Fault found between {surrogate.treatment} causing {surrogate.outcome}. Contradiction with expected {surrogate.expected_relationship}." ) - test_result.relationship = f"{surrogate.treatment} -> {surrogate.outcome} expected {surrogate.expected_relationship}" + test_result.relationship = ( + f"{surrogate.treatment} -> {surrogate.outcome} expected {surrogate.expected_relationship}" + ) return test_result, i + 1, data_collector.data - print("No fault found") return "No fault found", i + 1, data_collector.data diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index acaa2913..da6411ca 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -36,6 +36,7 @@ def generate_fitness_functions( for surrogate in surrogate_models: contradiction_function = self.contradiction_functions[surrogate.expected_relationship] + # The returned fitness function after including the contradiction function and surrogate model into the function's scope def fitness_function(_ga, solution, idx): surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta @@ -212,9 +213,7 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: while len(pool_vals) < num and len(all_fitness_function) > 0: fitness_function = all_fitness_function.pop() - pool_vals.append( - (fitness_function.surrogate_model, self.delta, specification.scenario, self.config) - ) + pool_vals.append((fitness_function.surrogate_model, self.delta, specification.scenario, self.config)) with mp.Pool(processes=num) as pool: solutions.extend(pool.map(threaded_search_function, range(len(pool_vals)))) From 5a41b1b0381e7bb27bcbcf804e28995077474483 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Fri, 15 Dec 2023 13:37:01 +0000 Subject: [PATCH 21/60] Added fitness function comment + removed multiprocessing code --- .../surrogate/surrogate_search_algorithms.py | 120 +----------------- 1 file changed, 1 insertion(+), 119 deletions(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index da6411ca..47d1999b 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -4,15 +4,6 @@ from pygad import GA from operator import itemgetter -import multiprocessing as mp -import warnings - -contradiction_functions = { - "positive": lambda x: -1 * x, - "negative": lambda x: x, - "no_effect": lambda x: abs(x), - "some_effect": lambda x: abs(1 / x), -} class GeneticSearchAlgorithm(SearchAlgorithm): @@ -36,7 +27,7 @@ def generate_fitness_functions( for surrogate in surrogate_models: contradiction_function = self.contradiction_functions[surrogate.expected_relationship] - # The returned fitness function after including the contradiction function and surrogate model into the function's scope + # The returned fitness function after including required variables into the function's scope def fitness_function(_ga, solution, idx): surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta @@ -112,112 +103,3 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: return max( solutions, key=itemgetter(1) ) # TODO This can be done better with fitness normalisation between edges - - -pool_vals = [] - - -def build_fitness_func(surrogate, delta): - def diff_evo_fitness_function(_ga, solution, _idx): - surrogate.control_value = solution[0] - delta - surrogate.treatment_value = solution[0] + delta - - adjustment_dict = dict() - for i, adjustment in enumerate(surrogate.adjustment_set): - adjustment_dict[adjustment] = solution[i + 1] - - ate = surrogate.estimate_ate_calculated(adjustment_dict) - - return contradiction_functions[surrogate.expected_relationship](ate) - - return diff_evo_fitness_function - - -def threaded_search_function(idx): - surrogate, delta, scenario, config = pool_vals[idx] - - var_space = dict() - var_space[surrogate.treatment] = dict() - for adj in surrogate.adjustment_set: - var_space[adj] = dict() - - for relationship in list(scenario.constraints): - rel_split = str(relationship).split(" ") - - if rel_split[1] == ">=": - var_space[rel_split[0]]["low"] = int(rel_split[2]) - elif rel_split[1] == "<=": - var_space[rel_split[0]]["high"] = int(rel_split[2]) - - gene_space = [] - gene_space.append(var_space[surrogate.treatment]) - for adj in surrogate.adjustment_set: - gene_space.append(var_space[adj]) - - gene_types = [] - gene_types.append(scenario.variables.get(surrogate.treatment).datatype) - for adj in surrogate.adjustment_set: - gene_types.append(scenario.variables.get(adj).datatype) - - ga = GA( - num_generations=200, - num_parents_mating=4, - fitness_func=build_fitness_func(surrogate, delta), - sol_per_pop=10, - num_genes=1 + len(surrogate.adjustment_set), - gene_space=gene_space, - gene_type=gene_types, - ) - - if config is not None: - for k, v in config.items(): - if k == "gene_space": - raise Exception( - "Gene space should not be set through config. This is generated from the causal specification" - ) - setattr(ga, k, v) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - ga.run() - solution, fitness, _idx = ga.best_solution() - - solution_dict = dict() - solution_dict[surrogate.treatment] = solution[0] - for idx, adj in enumerate(surrogate.adjustment_set): - solution_dict[adj] = solution[idx + 1] - - return (solution_dict, fitness, surrogate) - - -class MultiProcessGeneticSearchAlgorithm(SearchAlgorithm): - def __init__(self, delta=0.05, config: dict = None, processes: int = 1) -> None: - self.delta = delta - self.config = config - self.processes = processes - - def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: - return [SearchFitnessFunction(None, model) for model in surrogate_models] - - def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: - global pool_vals - solutions = [] - all_fitness_function = fitness_functions.copy() - - while len(all_fitness_function) > 0: - num = self.processes - if num > len(all_fitness_function): - num = len(all_fitness_function) - - pool_vals.clear() - - while len(pool_vals) < num and len(all_fitness_function) > 0: - fitness_function = all_fitness_function.pop() - pool_vals.append((fitness_function.surrogate_model, self.delta, specification.scenario, self.config)) - - with mp.Pool(processes=num) as pool: - solutions.extend(pool.map(threaded_search_function, range(len(pool_vals)))) - - return min( - solutions, key=itemgetter(1) - ) # TODO This can be done better with fitness normalisation between edges From 6d04385e0b06deef2b3b7e115fc00e3a98ea0a16 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:17:22 +0000 Subject: [PATCH 22/60] Linting for estimators.py --- causal_testing/testing/estimators.py | 116 +++++++++++++-------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index df7c1a3d..cf6c63b7 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -40,16 +40,16 @@ class Estimator(ABC): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - df: pd.DataFrame = None, - effect_modifiers: dict[str:Any] = None, - alpha: float = 0.05, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[str:Any] = None, + alpha: float = 0.05, ): self.treatment = treatment self.treatment_value = treatment_value @@ -90,16 +90,16 @@ class LogisticRegressionEstimator(Estimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - df: pd.DataFrame = None, - effect_modifiers: dict[str:Any] = None, - formula: str = None, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[str:Any] = None, + formula: str = None, ): super().__init__(treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers) @@ -162,7 +162,7 @@ def estimate(self, data: pd.DataFrame, adjustment_config: dict = None) -> Regres return model.predict(x) def estimate_control_treatment( - self, adjustment_config: dict = None, bootstrap_size: int = 100 + self, adjustment_config: dict = None, bootstrap_size: int = 100 ) -> tuple[pd.Series, pd.Series]: """Estimate the outcomes under control and treatment. @@ -280,17 +280,17 @@ class LinearRegressionEstimator(Estimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - df: pd.DataFrame = None, - effect_modifiers: dict[Variable:Any] = None, - formula: str = None, - alpha: float = 0.05, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[Variable:Any] = None, + formula: str = None, + alpha: float = 0.05, ): super().__init__( treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers, alpha=alpha @@ -440,24 +440,24 @@ def _get_confidence_intervals(self, model, treatment): class PolynomialRegressionEstimator(LinearRegressionEstimator): - """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a polynomial - combination of parameters and functions of the variables (note these functions need not be polynomial). + """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a + polynomial combination of parameters and functions of the variables (note these functions need not be polynomial). """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - degree: int, - df: pd.DataFrame = None, - effect_modifiers: dict[Variable:Any] = None, - formula: str = None, - alpha: float = 0.05, - expected_relationship=None, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + degree: int, + df: pd.DataFrame = None, + effect_modifiers: dict[Variable:Any] = None, + formula: str = None, + alpha: float = 0.05, + expected_relationship=None, ): super().__init__( treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers, formula, alpha @@ -497,17 +497,17 @@ class InstrumentalVariableEstimator(Estimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - instrument: str, - df: pd.DataFrame = None, - intercept: int = 1, - effect_modifiers: dict = None, # Not used (yet?). Needed for compatibility + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + instrument: str, + df: pd.DataFrame = None, + intercept: int = 1, + effect_modifiers: dict = None, # Not used (yet?). Needed for compatibility ): super().__init__(treatment, treatment_value, control_value, adjustment_set, outcome, df, None) self.intercept = intercept From 0b8c38fa72917c5867afa113894b4b196197b68d Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:23:40 +0000 Subject: [PATCH 23/60] Make remove_hidden_adjustment_sets method static --- causal_testing/specification/causal_dag.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index aa6e41e7..28026d53 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -499,8 +499,9 @@ def depends_on_outputs(self, node: Node, scenario: Scenario) -> bool: if isinstance(scenario.variables[node], Output): return True return any((self.depends_on_outputs(n, scenario) for n in self.graph.predecessors(node))) - - def remove_hidden_adjustment_sets(self, minimal_adjustment_sets: list[str], scenario: Scenario): + + @staticmethod + def remove_hidden_adjustment_sets(minimal_adjustment_sets: list[str], scenario: Scenario): return [ adj for adj in minimal_adjustment_sets if all([not scenario.variables.get(x).hidden for x in adj]) ] From 219b5d53ad7d9f8eadbcbe1bfc7eff883a0fbf81 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:25:48 +0000 Subject: [PATCH 24/60] reformat & black causal_dag.py --- causal_testing/specification/causal_dag.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index 28026d53..82ef9ac0 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -125,7 +125,6 @@ def close_separator( class CausalDAG(nx.DiGraph): - """A causal DAG is a directed acyclic graph in which nodes represent random variables and edges represent causality between a pair of random variables. We implement a CausalDAG as a networkx DiGraph with an additional check that ensures it is acyclic. A CausalDAG must be specified as a dot file. @@ -502,9 +501,7 @@ def depends_on_outputs(self, node: Node, scenario: Scenario) -> bool: @staticmethod def remove_hidden_adjustment_sets(minimal_adjustment_sets: list[str], scenario: Scenario): - return [ - adj for adj in minimal_adjustment_sets if all([not scenario.variables.get(x).hidden for x in adj]) - ] + return [adj for adj in minimal_adjustment_sets if all(not scenario.variables.get(x).hidden for x in adj)] def identification(self, base_test_case: BaseTestCase, scenario: Scenario = None): """Identify and return the minimum adjustment set @@ -525,7 +522,7 @@ def identification(self, base_test_case: BaseTestCase, scenario: Scenario = None ) else: raise ValueError("Causal effect should be 'total' or 'direct'") - + if scenario is not None: minimal_adjustment_sets = self.remove_hidden_adjustment_sets(minimal_adjustment_sets, scenario) From bd0c44fae63921c4000a2d1db8b2ad15aa41a157 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:36:29 +0000 Subject: [PATCH 25/60] docstrings for causal_dag.py --- causal_testing/specification/causal_dag.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index 82ef9ac0..5fa98d26 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -501,6 +501,10 @@ def depends_on_outputs(self, node: Node, scenario: Scenario) -> bool: @staticmethod def remove_hidden_adjustment_sets(minimal_adjustment_sets: list[str], scenario: Scenario): + """Remove variables labelled as hidden from adjustment set(s) + :param minimal_adjustment_sets: list of minimal adjustment set(s) to have hidden variables removed from + :param scenario: The modelling scenario which informs the variables that are hidden + """ return [adj for adj in minimal_adjustment_sets if all(not scenario.variables.get(x).hidden for x in adj)] def identification(self, base_test_case: BaseTestCase, scenario: Scenario = None): @@ -508,6 +512,7 @@ def identification(self, base_test_case: BaseTestCase, scenario: Scenario = None :param base_test_case: A base test case instance containing the outcome_variable and the treatment_variable required for identification. + :param scenario: The modelling scenario relating to the tests :return minimal_adjustment_set: The smallest set of variables which can be adjusted for to obtain a causal estimate as opposed to a purely associational estimate. """ From e3f474ec9a638a4ebddb1a87a04940bda67d3457 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:46:10 +0000 Subject: [PATCH 26/60] linting + black for surrogate_search_algorithms.py --- .../surrogate/surrogate_search_algorithms.py | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 47d1999b..2dc0378c 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,12 +1,15 @@ -from causal_testing.specification.causal_specification import CausalSpecification -from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator -from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction +"""Module containing implementation of search algorithm for surrogate search """ -from pygad import GA from operator import itemgetter +from pygad import GA + +from causal_testing.specification.causal_specification import CausalSpecification +from causal_testing.testing.estimators import PolynomialRegressionEstimator +from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction class GeneticSearchAlgorithm(SearchAlgorithm): + """ Implementation of SearchAlgorithm class. Implements genetic search algorithm for surrogate models.""" def __init__(self, delta=0.05, config: dict = None) -> None: super().__init__() @@ -15,7 +18,7 @@ def __init__(self, delta=0.05, config: dict = None) -> None: self.contradiction_functions = { "positive": lambda x: -1 * x, "negative": lambda x: x, - "no_effect": lambda x: abs(x), + "no_effect": abs, "some_effect": lambda x: abs(1 / x), } @@ -28,11 +31,11 @@ def generate_fitness_functions( contradiction_function = self.contradiction_functions[surrogate.expected_relationship] # The returned fitness function after including required variables into the function's scope - def fitness_function(_ga, solution, idx): + def fitness_function(ga, solution, idx): surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta - adjustment_dict = dict() + adjustment_dict = {} for i, adjustment in enumerate(surrogate.adjustment_set): adjustment_dict[adjustment] = solution[i + 1] @@ -50,10 +53,10 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: solutions = [] for fitness_function in fitness_functions: - var_space = dict() - var_space[fitness_function.surrogate_model.treatment] = dict() + var_space = {} + var_space[fitness_function.surrogate_model.treatment] = {} for adj in fitness_function.surrogate_model.adjustment_set: - var_space[adj] = dict() + var_space[adj] = {} for relationship in list(specification.scenario.constraints): rel_split = str(relationship).split(" ") @@ -87,19 +90,18 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: for k, v in self.config.items(): if k == "gene_space": raise Exception( - "Gene space should not be set through config. This is generated from the causal specification" + "Gene space should not be set through config. This is generated from the causal " + "specification" ) setattr(ga, k, v) ga.run() - solution, fitness, _idx = ga.best_solution() + solution, fitness, _ = ga.best_solution() - solution_dict = dict() + solution_dict = {} solution_dict[fitness_function.surrogate_model.treatment] = solution[0] for idx, adj in enumerate(fitness_function.surrogate_model.adjustment_set): solution_dict[adj] = solution[idx + 1] solutions.append((solution_dict, fitness, fitness_function.surrogate_model)) - return max( - solutions, key=itemgetter(1) - ) # TODO This can be done better with fitness normalisation between edges + return max(solutions, key=itemgetter(1)) # This can be done better with fitness normalisation between edges From e3ad52df2927a502cd014fff61c846bac22e94bf Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 21 Dec 2023 08:55:23 +0000 Subject: [PATCH 27/60] Docstrings --- .../surrogate/causal_surrogate_assisted.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 88480591..5bc52cc6 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -1,3 +1,5 @@ +"""Module containing classes to define and run causal surrogate assisted test cases""" + from causal_testing.data_collection.data_collector import ObservationalDataCollector from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.base_test_case import BaseTestCase @@ -10,6 +12,7 @@ @dataclass class SimulationResult(ABC): + """Data class holding the data and result metadata of a simulation""" data: dict fault: bool relationship: str @@ -17,12 +20,18 @@ class SimulationResult(ABC): @dataclass class SearchFitnessFunction(ABC): + """Data class containing the Fitness function and related model""" fitness_function: Any surrogate_model: PolynomialRegressionEstimator class SearchAlgorithm: + """Class to be inherited with the search algorithm consisting of a search function and the fitness function of the + space to be searched""" + def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: + """Generates the fitness function of the search space + :param surrogate_models: """ pass def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: @@ -30,6 +39,9 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: class Simulator: + """Class to be inherited with Simulator specific functions to start, shutdown and run the simulation with the give + config file""" + def startup(self, **kwargs): pass @@ -41,21 +53,23 @@ def run_with_config(self, configuration) -> SimulationResult: class CausalSurrogateAssistedTestCase: + """A class representing a single causal surrogate assisted test case.""" + def __init__( - self, - specification: CausalSpecification, - search_algorithm: SearchAlgorithm, - simulator: Simulator, + self, + specification: CausalSpecification, + search_algorithm: SearchAlgorithm, + simulator: Simulator, ): self.specification = specification self.search_algorithm = search_algorithm self.simulator = simulator def execute( - self, - data_collector: ObservationalDataCollector, - max_executions: int = 200, - custom_data_aggregator: Callable[[dict, dict], dict] = None, + self, + data_collector: ObservationalDataCollector, + max_executions: int = 200, + custom_data_aggregator: Callable[[dict, dict], dict] = None, ): data_collector.collect_data() @@ -77,7 +91,8 @@ def execute( if test_result.fault: print( - f"Fault found between {surrogate.treatment} causing {surrogate.outcome}. Contradiction with expected {surrogate.expected_relationship}." + f"Fault found between {surrogate.treatment} causing {surrogate.outcome}. Contradiction with " + f"expected {surrogate.expected_relationship}." ) test_result.relationship = ( f"{surrogate.treatment} -> {surrogate.outcome} expected {surrogate.expected_relationship}" @@ -88,7 +103,7 @@ def execute( return "No fault found", i + 1, data_collector.data def generate_surrogates( - self, specification: CausalSpecification, data_collector: ObservationalDataCollector + self, specification: CausalSpecification, data_collector: ObservationalDataCollector ) -> list[SearchFitnessFunction]: surrogate_models = [] From e2444702099832a1f80af16a905c59fd056b5a70 Mon Sep 17 00:00:00 2001 From: f-allian Date: Fri, 22 Dec 2023 15:28:22 +0000 Subject: [PATCH 28/60] Add: initial unit tests for causal surrogate --- .../test_causal_surrogate_assisted.py | 42 +++++++++++++++++++ .../test_surrogate_search_algorithms.py | 0 tests/testing_tests/test_estimators.py | 40 ++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 tests/surrogate_tests/test_causal_surrogate_assisted.py create mode 100644 tests/surrogate_tests/test_surrogate_search_algorithms.py diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py new file mode 100644 index 00000000..c8295f9f --- /dev/null +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -0,0 +1,42 @@ +import unittest +from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, SearchFitnessFunction +from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator + +class TestSimulationResult(unittest.TestCase): + + def setUp(self): + + self.data = {'key': 'value'} + + def test_inputs(self): + + fault_values = [True, False] + + relationship_values = ["positive", "negative", None] + + for fault in fault_values: + + for relationship in relationship_values: + with self.subTest(fault=fault, relationship=relationship): + result = SimulationResult(data=self.data, fault=fault, relationship=relationship) + + self.assertIsInstance(result.data, dict) + + self.assertEqual(result.fault, fault) + + self.assertEqual(result.relationship, relationship) + +class TestSearchFitnessFunction(unittest.TestCase): + + #TODO: complete tests for causal surrogate + + def test_init_valid_values(self): + + test_function = lambda x: x **2 + + surrogate_model = PolynomialRegressionEstimator() + + search_function = SearchFitnessFunction(fitness_function=test_function, surrogate_model=surrogate_model) + + self.assertIsCallable(search_function.fitness_function) + self.assertIsInstance(search_function.surrogate_model, PolynomialRegressionEstimator) \ No newline at end of file diff --git a/tests/surrogate_tests/test_surrogate_search_algorithms.py b/tests/surrogate_tests/test_surrogate_search_algorithms.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/testing_tests/test_estimators.py b/tests/testing_tests/test_estimators.py index 835a1144..8d660c00 100644 --- a/tests/testing_tests/test_estimators.py +++ b/tests/testing_tests/test_estimators.py @@ -7,6 +7,7 @@ CausalForestEstimator, LogisticRegressionEstimator, InstrumentalVariableEstimator, + PolynomialRegressionEstimator ) from causal_testing.specification.variable import Input from causal_testing.utils.validation import CausalValidator @@ -409,6 +410,43 @@ def test_program_11_2_with_robustness_validation(self): self.assertEqual(round(cv.estimate_robustness(model)["treatments"], 4), 0.7353) +class TestPolynomialRegressionEstimator(TestLinearRegressionEstimator): + + @classmethod + + def setUpClass(cls): + + super().setUpClass() + def test_program_11_3_polynomial(self): + + """Test whether the polynomial regression implementation produces the same results as program 11.3 (p. 162). + https://www.hsph.harvard.edu/miguel-hernan/wp-content/uploads/sites/1268/2023/10/hernanrobins_WhatIf_30sep23.pdf + """ + + df = self.chapter_11_df.copy() + + polynomial_estimator = PolynomialRegressionEstimator( + "treatments", None, None, set(), "outcomes", 3, df) + + model = polynomial_estimator._run_linear_regression() + + ate, _ = polynomial_estimator.estimate_coefficient() + + self.assertEqual( + round( + model.params["Intercept"] + + 90 * model.params["treatments"] + + 90 * 90 * model.params["np.power(treatments, 2)"], + 1, + ), + 197.1, + ) + # Increasing treatments from 90 to 100 should be the same as 10 times the unit ATE + self.assertEqual(round(model.params["treatments"], 3), round(ate, 3)) + + + + class TestCausalForestEstimator(unittest.TestCase): """Test the linear regression estimator against the programming exercises in Section 2 of Hernán and Robins [1]. @@ -491,3 +529,5 @@ def test_X1_effect(self): test_results = lr_model.estimate_ate() ate = test_results[0] self.assertAlmostEqual(ate, 2.0) + + From 162e28d430bd59a6c01b867398ebf31a51803406 Mon Sep 17 00:00:00 2001 From: f-allian Date: Fri, 22 Dec 2023 21:39:50 +0000 Subject: [PATCH 29/60] Revert "Add: initial unit tests for causal surrogate" This reverts commit e2444702099832a1f80af16a905c59fd056b5a70. --- .../test_causal_surrogate_assisted.py | 42 ------------------- .../test_surrogate_search_algorithms.py | 0 tests/testing_tests/test_estimators.py | 40 ------------------ 3 files changed, 82 deletions(-) delete mode 100644 tests/surrogate_tests/test_causal_surrogate_assisted.py delete mode 100644 tests/surrogate_tests/test_surrogate_search_algorithms.py diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py deleted file mode 100644 index c8295f9f..00000000 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, SearchFitnessFunction -from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator - -class TestSimulationResult(unittest.TestCase): - - def setUp(self): - - self.data = {'key': 'value'} - - def test_inputs(self): - - fault_values = [True, False] - - relationship_values = ["positive", "negative", None] - - for fault in fault_values: - - for relationship in relationship_values: - with self.subTest(fault=fault, relationship=relationship): - result = SimulationResult(data=self.data, fault=fault, relationship=relationship) - - self.assertIsInstance(result.data, dict) - - self.assertEqual(result.fault, fault) - - self.assertEqual(result.relationship, relationship) - -class TestSearchFitnessFunction(unittest.TestCase): - - #TODO: complete tests for causal surrogate - - def test_init_valid_values(self): - - test_function = lambda x: x **2 - - surrogate_model = PolynomialRegressionEstimator() - - search_function = SearchFitnessFunction(fitness_function=test_function, surrogate_model=surrogate_model) - - self.assertIsCallable(search_function.fitness_function) - self.assertIsInstance(search_function.surrogate_model, PolynomialRegressionEstimator) \ No newline at end of file diff --git a/tests/surrogate_tests/test_surrogate_search_algorithms.py b/tests/surrogate_tests/test_surrogate_search_algorithms.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/testing_tests/test_estimators.py b/tests/testing_tests/test_estimators.py index 8d660c00..835a1144 100644 --- a/tests/testing_tests/test_estimators.py +++ b/tests/testing_tests/test_estimators.py @@ -7,7 +7,6 @@ CausalForestEstimator, LogisticRegressionEstimator, InstrumentalVariableEstimator, - PolynomialRegressionEstimator ) from causal_testing.specification.variable import Input from causal_testing.utils.validation import CausalValidator @@ -410,43 +409,6 @@ def test_program_11_2_with_robustness_validation(self): self.assertEqual(round(cv.estimate_robustness(model)["treatments"], 4), 0.7353) -class TestPolynomialRegressionEstimator(TestLinearRegressionEstimator): - - @classmethod - - def setUpClass(cls): - - super().setUpClass() - def test_program_11_3_polynomial(self): - - """Test whether the polynomial regression implementation produces the same results as program 11.3 (p. 162). - https://www.hsph.harvard.edu/miguel-hernan/wp-content/uploads/sites/1268/2023/10/hernanrobins_WhatIf_30sep23.pdf - """ - - df = self.chapter_11_df.copy() - - polynomial_estimator = PolynomialRegressionEstimator( - "treatments", None, None, set(), "outcomes", 3, df) - - model = polynomial_estimator._run_linear_regression() - - ate, _ = polynomial_estimator.estimate_coefficient() - - self.assertEqual( - round( - model.params["Intercept"] - + 90 * model.params["treatments"] - + 90 * 90 * model.params["np.power(treatments, 2)"], - 1, - ), - 197.1, - ) - # Increasing treatments from 90 to 100 should be the same as 10 times the unit ATE - self.assertEqual(round(model.params["treatments"], 3), round(ate, 3)) - - - - class TestCausalForestEstimator(unittest.TestCase): """Test the linear regression estimator against the programming exercises in Section 2 of Hernán and Robins [1]. @@ -529,5 +491,3 @@ def test_X1_effect(self): test_results = lr_model.estimate_ate() ate = test_results[0] self.assertAlmostEqual(ate, 2.0) - - From df20a4922519e0e88c5837ce1e0d9bb15f1cff27 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 2 Jan 2024 14:13:45 +0000 Subject: [PATCH 30/60] Included a condition for invalid data returned --- causal_testing/surrogate/causal_surrogate_assisted.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 88480591..6272d055 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -71,7 +71,8 @@ def execute( self.simulator.shutdown() if custom_data_aggregator is not None: - data_collector.data = custom_data_aggregator(data_collector.data, test_result.data) + if data_collector.data is not None: + data_collector.data = custom_data_aggregator(data_collector.data, test_result.data) else: data_collector.data = data_collector.data.append(test_result.data, ignore_index=True) From e66fc6e228a26806b7e9ca9ba00d92f018bb05af Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 3 Jan 2024 11:02:43 +0000 Subject: [PATCH 31/60] Renamed PolynomialRegressionEstimator to CubicSplineRegressionEstimator --- .../surrogate/causal_surrogate_assisted.py | 6 +++--- .../surrogate/surrogate_search_algorithms.py | 4 ++-- causal_testing/testing/estimators.py | 10 +++++----- .../test_causal_surrogate_assisted.py | 6 +++--- tests/testing_tests/test_estimators.py | 14 +++++++------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 6272d055..1c3fa5c0 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -1,7 +1,7 @@ from causal_testing.data_collection.data_collector import ObservationalDataCollector from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.base_test_case import BaseTestCase -from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator +from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator from dataclasses import dataclass from typing import Callable, Any @@ -18,7 +18,7 @@ class SimulationResult(ABC): @dataclass class SearchFitnessFunction(ABC): fitness_function: Any - surrogate_model: PolynomialRegressionEstimator + surrogate_model: CubicSplineRegressionEstimator class SearchAlgorithm: @@ -102,7 +102,7 @@ def generate_surrogates( minimal_adjustment_set = specification.causal_dag.identification(base_test_case, specification.scenario) - surrogate = PolynomialRegressionEstimator( + surrogate = CubicSplineRegressionEstimator( u, 0, 0, diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 47d1999b..ea5c5284 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,5 +1,5 @@ from causal_testing.specification.causal_specification import CausalSpecification -from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator +from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction from pygad import GA @@ -20,7 +20,7 @@ def __init__(self, delta=0.05, config: dict = None) -> None: } def generate_fitness_functions( - self, surrogate_models: list[PolynomialRegressionEstimator] + self, surrogate_models: list[CubicSplineRegressionEstimator] ) -> list[SearchFitnessFunction]: fitness_functions = [] diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index cf6c63b7..fa0a6eb7 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -439,9 +439,9 @@ def _get_confidence_intervals(self, model, treatment): return [ci_low, ci_high] -class PolynomialRegressionEstimator(LinearRegressionEstimator): - """A Polynomial Regression Estimator is a parametric estimator which restricts the variables in the data to a - polynomial combination of parameters and functions of the variables (note these functions need not be polynomial). +class CubicSplineRegressionEstimator(LinearRegressionEstimator): + """A Cubic Spline Regression Estimator is a parametric estimator which restricts the variables in the data to a + combination of parameters and basis functions of the variables. """ def __init__( @@ -452,7 +452,7 @@ def __init__( control_value: float, adjustment_set: set, outcome: str, - degree: int, + basis: int, df: pd.DataFrame = None, effect_modifiers: dict[Variable:Any] = None, formula: str = None, @@ -470,7 +470,7 @@ def __init__( if formula is None: terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) - self.formula = f"{outcome} ~ cr({'+'.join(terms)}, df={degree})" + self.formula = f"{outcome} ~ cr({'+'.join(terms)}, df={basis})" def estimate_ate_calculated(self, adjustment_config: dict = None) -> tuple[float, list[float]]: model = self._run_linear_regression() diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index c8295f9f..2fce12c9 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -1,6 +1,6 @@ import unittest from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, SearchFitnessFunction -from causal_testing.testing.estimators import Estimator, PolynomialRegressionEstimator +from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator class TestSimulationResult(unittest.TestCase): @@ -34,9 +34,9 @@ def test_init_valid_values(self): test_function = lambda x: x **2 - surrogate_model = PolynomialRegressionEstimator() + surrogate_model = CubicSplineRegressionEstimator() search_function = SearchFitnessFunction(fitness_function=test_function, surrogate_model=surrogate_model) self.assertIsCallable(search_function.fitness_function) - self.assertIsInstance(search_function.surrogate_model, PolynomialRegressionEstimator) \ No newline at end of file + self.assertIsInstance(search_function.surrogate_model, CubicSplineRegressionEstimator) \ No newline at end of file diff --git a/tests/testing_tests/test_estimators.py b/tests/testing_tests/test_estimators.py index 8d660c00..a20dd246 100644 --- a/tests/testing_tests/test_estimators.py +++ b/tests/testing_tests/test_estimators.py @@ -7,7 +7,7 @@ CausalForestEstimator, LogisticRegressionEstimator, InstrumentalVariableEstimator, - PolynomialRegressionEstimator + CubicSplineRegressionEstimator ) from causal_testing.specification.variable import Input from causal_testing.utils.validation import CausalValidator @@ -410,27 +410,27 @@ def test_program_11_2_with_robustness_validation(self): self.assertEqual(round(cv.estimate_robustness(model)["treatments"], 4), 0.7353) -class TestPolynomialRegressionEstimator(TestLinearRegressionEstimator): +class TestCubicSplineRegressionEstimator(TestLinearRegressionEstimator): @classmethod def setUpClass(cls): super().setUpClass() - def test_program_11_3_polynomial(self): + def test_program_11_3_cublic_spline(self): - """Test whether the polynomial regression implementation produces the same results as program 11.3 (p. 162). + """Test whether the cublic_spline regression implementation produces the same results as program 11.3 (p. 162). https://www.hsph.harvard.edu/miguel-hernan/wp-content/uploads/sites/1268/2023/10/hernanrobins_WhatIf_30sep23.pdf """ df = self.chapter_11_df.copy() - polynomial_estimator = PolynomialRegressionEstimator( + cublic_spline_estimator = CubicSplineRegressionEstimator( "treatments", None, None, set(), "outcomes", 3, df) - model = polynomial_estimator._run_linear_regression() + model = cublic_spline_estimator._run_linear_regression() - ate, _ = polynomial_estimator.estimate_coefficient() + ate, _ = cublic_spline_estimator.estimate_coefficient() self.assertEqual( round( From 051861ee72d05d3c2e586f2a6582982e42886d58 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:25:56 +0000 Subject: [PATCH 32/60] function & class strings for ABC classes --- .../surrogate/causal_surrogate_assisted.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 99e9a3b7..c095d06e 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -31,11 +31,14 @@ class SearchAlgorithm: def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: """Generates the fitness function of the search space - :param surrogate_models: """ - pass + :param surrogate_models: A list of CubicSplineRegressionEstimator generated for each edge of the DAG + :return: A list of fitness functions mapping to each of the surrogate models in the input""" def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: - pass + """Function which implements a search routine which searches for the optimal fitness value for the specified + scenario + :param fitness_functions: The fitness function to be optimised + :param specification: The Causal Specification (combination of Scenario and Causal Dag)""" class Simulator: @@ -43,13 +46,16 @@ class Simulator: config file""" def startup(self, **kwargs): - pass + """Function that when run, initialises and opens the Simulator""" def shutdown(self, **kwargs): - pass + """Function to safely exit and shutdown the Simulator""" def run_with_config(self, configuration) -> SimulationResult: - pass + """Run the simulator with the given configuration and return the results in the structure of a + SimulationResult + :param configuration: + :return: Simulation results in the structure of the SimulationResult data class""" class CausalSurrogateAssistedTestCase: From 7692084113e56ecf1e2759c69a39d01a6d26b0d3 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:29:28 +0000 Subject: [PATCH 33/60] Add all docstrings to causal_surrogate_assisted.py --- .../surrogate/causal_surrogate_assisted.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index c095d06e..25ff238d 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -54,7 +54,7 @@ def shutdown(self, **kwargs): def run_with_config(self, configuration) -> SimulationResult: """Run the simulator with the given configuration and return the results in the structure of a SimulationResult - :param configuration: + :param configuration: the configuration required to initialise the Simulation :return: Simulation results in the structure of the SimulationResult data class""" @@ -77,6 +77,12 @@ def execute( max_executions: int = 200, custom_data_aggregator: Callable[[dict, dict], dict] = None, ): + """ For this specific test case, collect the data, run the simulator, check for faults and return the result + and collected data + :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario + :param max_executions: Maximum number of executions + :param custom_data_aggregator: + :return: tuple containing SimulationResult or str, execution number and collected data """ data_collector.collect_data() for i in range(max_executions): @@ -112,6 +118,11 @@ def execute( def generate_surrogates( self, specification: CausalSpecification, data_collector: ObservationalDataCollector ) -> list[SearchFitnessFunction]: + """ Generate a surrogate model for each edge of the dag that specifies it is included in the DAG metadata. + :param specification: The Causal Specification (combination of Scenario and Causal Dag) + :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario + :return: A list of surrogate models + """ surrogate_models = [] for u, v in specification.causal_dag.graph.edges: From 1e22537880cea2168d8e759aced8890b590fe3d3 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:43:02 +0000 Subject: [PATCH 34/60] remaining linting + black for causal_surrogate_assisted.py --- .../surrogate/causal_surrogate_assisted.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 25ff238d..d50f5339 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -1,18 +1,19 @@ """Module containing classes to define and run causal surrogate assisted test cases""" +from abc import ABC +from dataclasses import dataclass +from typing import Callable, Any + from causal_testing.data_collection.data_collector import ObservationalDataCollector from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator -from dataclasses import dataclass -from typing import Callable, Any -from abc import ABC - @dataclass class SimulationResult(ABC): """Data class holding the data and result metadata of a simulation""" + data: dict fault: bool relationship: str @@ -21,13 +22,14 @@ class SimulationResult(ABC): @dataclass class SearchFitnessFunction(ABC): """Data class containing the Fitness function and related model""" + fitness_function: Any surrogate_model: CubicSplineRegressionEstimator class SearchAlgorithm: """Class to be inherited with the search algorithm consisting of a search function and the fitness function of the - space to be searched""" + space to be searched""" def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: """Generates the fitness function of the search space @@ -43,7 +45,7 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: class Simulator: """Class to be inherited with Simulator specific functions to start, shutdown and run the simulation with the give - config file""" + config file""" def startup(self, **kwargs): """Function that when run, initialises and opens the Simulator""" @@ -62,35 +64,33 @@ class CausalSurrogateAssistedTestCase: """A class representing a single causal surrogate assisted test case.""" def __init__( - self, - specification: CausalSpecification, - search_algorithm: SearchAlgorithm, - simulator: Simulator, + self, + specification: CausalSpecification, + search_algorithm: SearchAlgorithm, + simulator: Simulator, ): self.specification = specification self.search_algorithm = search_algorithm self.simulator = simulator def execute( - self, - data_collector: ObservationalDataCollector, - max_executions: int = 200, - custom_data_aggregator: Callable[[dict, dict], dict] = None, + self, + data_collector: ObservationalDataCollector, + max_executions: int = 200, + custom_data_aggregator: Callable[[dict, dict], dict] = None, ): - """ For this specific test case, collect the data, run the simulator, check for faults and return the result - and collected data - :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario - :param max_executions: Maximum number of executions - :param custom_data_aggregator: - :return: tuple containing SimulationResult or str, execution number and collected data """ + """For this specific test case, collect the data, run the simulator, check for faults and return the result + and collected data + :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario + :param max_executions: Maximum number of executions + :param custom_data_aggregator: + :return: tuple containing SimulationResult or str, execution number and collected data""" data_collector.collect_data() for i in range(max_executions): surrogate_models = self.generate_surrogates(self.specification, data_collector) fitness_functions = self.search_algorithm.generate_fitness_functions(surrogate_models) - candidate_test_case, _fitness, surrogate = self.search_algorithm.search( - fitness_functions, self.specification - ) + candidate_test_case, _, surrogate = self.search_algorithm.search(fitness_functions, self.specification) self.simulator.startup() test_result = self.simulator.run_with_config(candidate_test_case) @@ -116,9 +116,9 @@ def execute( return "No fault found", i + 1, data_collector.data def generate_surrogates( - self, specification: CausalSpecification, data_collector: ObservationalDataCollector + self, specification: CausalSpecification, data_collector: ObservationalDataCollector ) -> list[SearchFitnessFunction]: - """ Generate a surrogate model for each edge of the dag that specifies it is included in the DAG metadata. + """Generate a surrogate model for each edge of the dag that specifies it is included in the DAG metadata. :param specification: The Causal Specification (combination of Scenario and Causal Dag) :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario :return: A list of surrogate models From 2d473e5b46a6df56c395931f379805c20b943280 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:16:51 +0000 Subject: [PATCH 35/60] Move functionality into static method called create_gene_types --- .../surrogate/surrogate_search_algorithms.py | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index c0ed1336..3413ae98 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -4,12 +4,13 @@ from pygad import GA from causal_testing.specification.causal_specification import CausalSpecification -from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator +from causal_testing.testing.estimators import CubicSplineRegressionEstimator from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction class GeneticSearchAlgorithm(SearchAlgorithm): """ Implementation of SearchAlgorithm class. Implements genetic search algorithm for surrogate models.""" + def __init__(self, delta=0.05, config: dict = None) -> None: super().__init__() @@ -23,7 +24,7 @@ def __init__(self, delta=0.05, config: dict = None) -> None: } def generate_fitness_functions( - self, surrogate_models: list[CubicSplineRegressionEstimator] + self, surrogate_models: list[CubicSplineRegressionEstimator] ) -> list[SearchFitnessFunction]: fitness_functions = [] @@ -53,28 +54,8 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: solutions = [] for fitness_function in fitness_functions: - var_space = {} - var_space[fitness_function.surrogate_model.treatment] = {} - for adj in fitness_function.surrogate_model.adjustment_set: - var_space[adj] = {} - - for relationship in list(specification.scenario.constraints): - rel_split = str(relationship).split(" ") - - if rel_split[1] == ">=": - var_space[rel_split[0]]["low"] = int(rel_split[2]) - elif rel_split[1] == "<=": - var_space[rel_split[0]]["high"] = int(rel_split[2]) - gene_space = [] - gene_space.append(var_space[fitness_function.surrogate_model.treatment]) - for adj in fitness_function.surrogate_model.adjustment_set: - gene_space.append(var_space[adj]) - - gene_types = [] - gene_types.append(specification.scenario.variables.get(fitness_function.surrogate_model.treatment).datatype) - for adj in fitness_function.surrogate_model.adjustment_set: - gene_types.append(specification.scenario.variables.get(adj).datatype) + gene_types, gene_space = self.create_gene_types(fitness_function, specification) ga = GA( num_generations=200, @@ -105,3 +86,34 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: solutions.append((solution_dict, fitness, fitness_function.surrogate_model)) return max(solutions, key=itemgetter(1)) # This can be done better with fitness normalisation between edges + + @staticmethod + def create_gene_types(fitness_function: SearchFitnessFunction, specification: CausalSpecification) -> tuple[ + list, list]: + """Generate the gene_types and gene_space for a given fitness function and specification + :param fitness_function: Instance of SearchFitnessFunction + :param specification: The Causal Specification (combination of Scenario and Causal Dag)""" + + var_space = {} + var_space[fitness_function.surrogate_model.treatment] = {} + for adj in fitness_function.surrogate_model.adjustment_set: + var_space[adj] = {} + + for relationship in list(specification.scenario.constraints): + rel_split = str(relationship).split(" ") + + if rel_split[1] == ">=": + var_space[rel_split[0]]["low"] = int(rel_split[2]) + elif rel_split[1] == "<=": + var_space[rel_split[0]]["high"] = int(rel_split[2]) + + gene_space = [] + gene_space.append(var_space[fitness_function.surrogate_model.treatment]) + for adj in fitness_function.surrogate_model.adjustment_set: + gene_space.append(var_space[adj]) + + gene_types = [] + gene_types.append(specification.scenario.variables.get(fitness_function.surrogate_model.treatment).datatype) + for adj in fitness_function.surrogate_model.adjustment_set: + gene_types.append(specification.scenario.variables.get(adj).datatype) + return gene_types, gene_space From 055b0640cc67dc32f5867c2be8a9627dda3af20a Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:29:40 +0000 Subject: [PATCH 36/60] Add type parameter to configuration variable --- causal_testing/surrogate/causal_surrogate_assisted.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index d50f5339..4bad6895 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -53,10 +53,10 @@ def startup(self, **kwargs): def shutdown(self, **kwargs): """Function to safely exit and shutdown the Simulator""" - def run_with_config(self, configuration) -> SimulationResult: + def run_with_config(self, configuration: Any) -> SimulationResult: """Run the simulator with the given configuration and return the results in the structure of a SimulationResult - :param configuration: the configuration required to initialise the Simulation + :param configuration: The configuration required to initialise the Simulation :return: Simulation results in the structure of the SimulationResult data class""" From 94ce3c97f6b06b40d4c029a04de52830c9608bec Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:35:55 +0000 Subject: [PATCH 37/60] Ignore cell-var-from-loop for surrogate_search_algorithms.py --- causal_testing/surrogate/surrogate_search_algorithms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 3413ae98..4a443f25 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,4 +1,6 @@ """Module containing implementation of search algorithm for surrogate search """ +# pylint: disable=cell-var-from-loop +# Fitness functions are required to be iteratively defined, including all variables within. from operator import itemgetter from pygad import GA From 2289ae48bc01681f701d62320fee50053ed18105 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:36:19 +0000 Subject: [PATCH 38/60] black --- .../surrogate/surrogate_search_algorithms.py | 10 +- causal_testing/testing/estimators.py | 112 +++++++++--------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 4a443f25..63f041f2 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -11,7 +11,7 @@ class GeneticSearchAlgorithm(SearchAlgorithm): - """ Implementation of SearchAlgorithm class. Implements genetic search algorithm for surrogate models.""" + """Implementation of SearchAlgorithm class. Implements genetic search algorithm for surrogate models.""" def __init__(self, delta=0.05, config: dict = None) -> None: super().__init__() @@ -26,7 +26,7 @@ def __init__(self, delta=0.05, config: dict = None) -> None: } def generate_fitness_functions( - self, surrogate_models: list[CubicSplineRegressionEstimator] + self, surrogate_models: list[CubicSplineRegressionEstimator] ) -> list[SearchFitnessFunction]: fitness_functions = [] @@ -56,7 +56,6 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: solutions = [] for fitness_function in fitness_functions: - gene_types, gene_space = self.create_gene_types(fitness_function, specification) ga = GA( @@ -90,8 +89,9 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: return max(solutions, key=itemgetter(1)) # This can be done better with fitness normalisation between edges @staticmethod - def create_gene_types(fitness_function: SearchFitnessFunction, specification: CausalSpecification) -> tuple[ - list, list]: + def create_gene_types( + fitness_function: SearchFitnessFunction, specification: CausalSpecification + ) -> tuple[list, list]: """Generate the gene_types and gene_space for a given fitness function and specification :param fitness_function: Instance of SearchFitnessFunction :param specification: The Causal Specification (combination of Scenario and Causal Dag)""" diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index fa0a6eb7..104866a7 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -40,16 +40,16 @@ class Estimator(ABC): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - df: pd.DataFrame = None, - effect_modifiers: dict[str:Any] = None, - alpha: float = 0.05, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[str:Any] = None, + alpha: float = 0.05, ): self.treatment = treatment self.treatment_value = treatment_value @@ -90,16 +90,16 @@ class LogisticRegressionEstimator(Estimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - df: pd.DataFrame = None, - effect_modifiers: dict[str:Any] = None, - formula: str = None, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[str:Any] = None, + formula: str = None, ): super().__init__(treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers) @@ -162,7 +162,7 @@ def estimate(self, data: pd.DataFrame, adjustment_config: dict = None) -> Regres return model.predict(x) def estimate_control_treatment( - self, adjustment_config: dict = None, bootstrap_size: int = 100 + self, adjustment_config: dict = None, bootstrap_size: int = 100 ) -> tuple[pd.Series, pd.Series]: """Estimate the outcomes under control and treatment. @@ -280,17 +280,17 @@ class LinearRegressionEstimator(Estimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - df: pd.DataFrame = None, - effect_modifiers: dict[Variable:Any] = None, - formula: str = None, - alpha: float = 0.05, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + df: pd.DataFrame = None, + effect_modifiers: dict[Variable:Any] = None, + formula: str = None, + alpha: float = 0.05, ): super().__init__( treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers, alpha=alpha @@ -445,19 +445,19 @@ class CubicSplineRegressionEstimator(LinearRegressionEstimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - basis: int, - df: pd.DataFrame = None, - effect_modifiers: dict[Variable:Any] = None, - formula: str = None, - alpha: float = 0.05, - expected_relationship=None, + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + basis: int, + df: pd.DataFrame = None, + effect_modifiers: dict[Variable:Any] = None, + formula: str = None, + alpha: float = 0.05, + expected_relationship=None, ): super().__init__( treatment, treatment_value, control_value, adjustment_set, outcome, df, effect_modifiers, formula, alpha @@ -497,17 +497,17 @@ class InstrumentalVariableEstimator(Estimator): """ def __init__( - # pylint: disable=too-many-arguments - self, - treatment: str, - treatment_value: float, - control_value: float, - adjustment_set: set, - outcome: str, - instrument: str, - df: pd.DataFrame = None, - intercept: int = 1, - effect_modifiers: dict = None, # Not used (yet?). Needed for compatibility + # pylint: disable=too-many-arguments + self, + treatment: str, + treatment_value: float, + control_value: float, + adjustment_set: set, + outcome: str, + instrument: str, + df: pd.DataFrame = None, + intercept: int = 1, + effect_modifiers: dict = None, # Not used (yet?). Needed for compatibility ): super().__init__(treatment, treatment_value, control_value, adjustment_set, outcome, df, None) self.intercept = intercept From c8f9dd3782bde10d8016564fe94da104b1a205f1 Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:41:12 +0000 Subject: [PATCH 39/60] Remove generic exception --- causal_testing/surrogate/surrogate_search_algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 63f041f2..8bebc055 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -71,7 +71,7 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: if self.config is not None: for k, v in self.config.items(): if k == "gene_space": - raise Exception( + raise ValueError( "Gene space should not be set through config. This is generated from the causal " "specification" ) From 49b80a76f0d0b612339927e58aa7bdaec5d9db5e Mon Sep 17 00:00:00 2001 From: cwild-UoS <93984046+cwild-UoS@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:54:32 +0000 Subject: [PATCH 40/60] Formalise ABCs --- .../surrogate/causal_surrogate_assisted.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 4bad6895..46224ef1 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -1,6 +1,6 @@ """Module containing classes to define and run causal surrogate assisted test cases""" -from abc import ABC +from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Callable, Any @@ -11,7 +11,7 @@ @dataclass -class SimulationResult(ABC): +class SimulationResult: """Data class holding the data and result metadata of a simulation""" data: dict @@ -20,22 +20,24 @@ class SimulationResult(ABC): @dataclass -class SearchFitnessFunction(ABC): +class SearchFitnessFunction: """Data class containing the Fitness function and related model""" fitness_function: Any surrogate_model: CubicSplineRegressionEstimator -class SearchAlgorithm: +class SearchAlgorithm(ABC): """Class to be inherited with the search algorithm consisting of a search function and the fitness function of the space to be searched""" + @abstractmethod def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: """Generates the fitness function of the search space :param surrogate_models: A list of CubicSplineRegressionEstimator generated for each edge of the DAG :return: A list of fitness functions mapping to each of the surrogate models in the input""" + @abstractmethod def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: """Function which implements a search routine which searches for the optimal fitness value for the specified scenario @@ -43,16 +45,19 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: :param specification: The Causal Specification (combination of Scenario and Causal Dag)""" -class Simulator: +class Simulator(ABC): """Class to be inherited with Simulator specific functions to start, shutdown and run the simulation with the give config file""" + @abstractmethod def startup(self, **kwargs): """Function that when run, initialises and opens the Simulator""" + @abstractmethod def shutdown(self, **kwargs): """Function to safely exit and shutdown the Simulator""" + @abstractmethod def run_with_config(self, configuration: Any) -> SimulationResult: """Run the simulator with the given configuration and return the results in the structure of a SimulationResult From 20cad36980ba002201d9b247f78b1a99d2e469d3 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Thu, 4 Jan 2024 10:15:02 +0000 Subject: [PATCH 41/60] Pylinting + updating doc string and typing --- causal_testing/surrogate/causal_surrogate_assisted.py | 7 ++++--- causal_testing/surrogate/surrogate_search_algorithms.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 46224ef1..937bfdfe 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -58,7 +58,7 @@ def shutdown(self, **kwargs): """Function to safely exit and shutdown the Simulator""" @abstractmethod - def run_with_config(self, configuration: Any) -> SimulationResult: + def run_with_config(self, configuration: dict) -> SimulationResult: """Run the simulator with the given configuration and return the results in the structure of a SimulationResult :param configuration: The configuration required to initialise the Simulation @@ -84,8 +84,9 @@ def execute( max_executions: int = 200, custom_data_aggregator: Callable[[dict, dict], dict] = None, ): - """For this specific test case, collect the data, run the simulator, check for faults and return the result - and collected data + """For this specific test case, a search algorithm is used to find the most contradictory point in the input + space which is, therefore, most likely to indicate incorrect behaviour. This cadidate test case is run against + the simulator, checked for faults and the result returned with collected data :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario :param max_executions: Maximum number of executions :param custom_data_aggregator: diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 8bebc055..e5a5e398 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -34,7 +34,7 @@ def generate_fitness_functions( contradiction_function = self.contradiction_functions[surrogate.expected_relationship] # The returned fitness function after including required variables into the function's scope - def fitness_function(ga, solution, idx): + def fitness_function(ga, solution, idx): # pylint: disable=unused-argument surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta From 5df6b9d6a2c49de4712d1f7077ac2cb72cbf81b7 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Thu, 4 Jan 2024 10:16:18 +0000 Subject: [PATCH 42/60] Pygad signature comment --- causal_testing/surrogate/surrogate_search_algorithms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index e5a5e398..95d1a469 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -34,6 +34,7 @@ def generate_fitness_functions( contradiction_function = self.contradiction_functions[surrogate.expected_relationship] # The returned fitness function after including required variables into the function's scope + # Unused arguments are required for pygad's fitness function signature def fitness_function(ga, solution, idx): # pylint: disable=unused-argument surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta From eeda6e7153a4446eda55f9cca1748d1048480780 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Thu, 4 Jan 2024 10:18:07 +0000 Subject: [PATCH 43/60] Updated doc string max executions --- causal_testing/surrogate/causal_surrogate_assisted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 937bfdfe..4c450184 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -88,7 +88,7 @@ def execute( space which is, therefore, most likely to indicate incorrect behaviour. This cadidate test case is run against the simulator, checked for faults and the result returned with collected data :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario - :param max_executions: Maximum number of executions + :param max_executions: Maximum number of simulator executions before exiting the search :param custom_data_aggregator: :return: tuple containing SimulationResult or str, execution number and collected data""" data_collector.collect_data() From 27c737a985e2d16a03d42ebe908bde6685fdaa2c Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Thu, 4 Jan 2024 14:03:41 +0000 Subject: [PATCH 44/60] Basic tests to stop tests failing --- causal_testing/testing/estimators.py | 7 ++++--- .../test_causal_surrogate_assisted.py | 4 ++-- tests/testing_tests/test_estimators.py | 21 +++++++++++-------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/causal_testing/testing/estimators.py b/causal_testing/testing/estimators.py index 104866a7..e52b841b 100644 --- a/causal_testing/testing/estimators.py +++ b/causal_testing/testing/estimators.py @@ -472,12 +472,13 @@ def __init__( terms = [treatment] + sorted(list(adjustment_set)) + sorted(list(effect_modifiers)) self.formula = f"{outcome} ~ cr({'+'.join(terms)}, df={basis})" - def estimate_ate_calculated(self, adjustment_config: dict = None) -> tuple[float, list[float]]: + def estimate_ate_calculated(self, adjustment_config: dict = None) -> float: model = self._run_linear_regression() x = {"Intercept": 1, self.treatment: self.treatment_value} - for k, v in adjustment_config.items(): - x[k] = v + if adjustment_config is not None: + for k, v in adjustment_config.items(): + x[k] = v if self.effect_modifiers is not None: for k, v in self.effect_modifiers.items(): x[k] = v diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index 2fce12c9..99884e35 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -34,9 +34,9 @@ def test_init_valid_values(self): test_function = lambda x: x **2 - surrogate_model = CubicSplineRegressionEstimator() + surrogate_model = CubicSplineRegressionEstimator("", 0, 0, set(), "", 4) search_function = SearchFitnessFunction(fitness_function=test_function, surrogate_model=surrogate_model) - self.assertIsCallable(search_function.fitness_function) + self.assertTrue(callable(search_function.fitness_function)) self.assertIsInstance(search_function.surrogate_model, CubicSplineRegressionEstimator) \ No newline at end of file diff --git a/tests/testing_tests/test_estimators.py b/tests/testing_tests/test_estimators.py index a20dd246..95f809ef 100644 --- a/tests/testing_tests/test_estimators.py +++ b/tests/testing_tests/test_estimators.py @@ -421,28 +421,31 @@ def test_program_11_3_cublic_spline(self): """Test whether the cublic_spline regression implementation produces the same results as program 11.3 (p. 162). https://www.hsph.harvard.edu/miguel-hernan/wp-content/uploads/sites/1268/2023/10/hernanrobins_WhatIf_30sep23.pdf + Slightly modified as Hernan et al. use linear regression for this example. """ df = self.chapter_11_df.copy() cublic_spline_estimator = CubicSplineRegressionEstimator( - "treatments", None, None, set(), "outcomes", 3, df) + "treatments", 1, 0, set(), "outcomes", 3, df) model = cublic_spline_estimator._run_linear_regression() - ate, _ = cublic_spline_estimator.estimate_coefficient() - self.assertEqual( round( - model.params["Intercept"] - + 90 * model.params["treatments"] - + 90 * 90 * model.params["np.power(treatments, 2)"], + cublic_spline_estimator.model.predict({"Intercept": 1, "treatments": 90}).iloc[0], 1, ), - 197.1, + 195.6, ) - # Increasing treatments from 90 to 100 should be the same as 10 times the unit ATE - self.assertEqual(round(model.params["treatments"], 3), round(ate, 3)) + + ate_1 = cublic_spline_estimator.estimate_ate_calculated() + cublic_spline_estimator.treatment_value = 2 + ate_2 = cublic_spline_estimator.estimate_ate_calculated() + + # Doubling the treatemebnt value should roughly but not exactly double the ATE + self.assertNotEqual(ate_1 * 2, ate_2) + self.assertAlmostEqual(ate_1 * 2, ate_2) From 8c3708282b566f4a52e522b4dbb1534f03637d32 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 13:44:02 +0000 Subject: [PATCH 45/60] Adding tests --- causal_testing/specification/causal_dag.py | 3 + .../surrogate/surrogate_search_algorithms.py | 9 +- tests/specification_tests/test_causal_dag.py | 34 ++++++ .../test_causal_surrogate_assisted.py | 100 +++++++++++++++++- 4 files changed, 138 insertions(+), 8 deletions(-) diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index 5fa98d26..1b47d3af 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -531,6 +531,9 @@ def identification(self, base_test_case: BaseTestCase, scenario: Scenario = None if scenario is not None: minimal_adjustment_sets = self.remove_hidden_adjustment_sets(minimal_adjustment_sets, scenario) + if len(minimal_adjustment_sets) == 0: + return set() + minimal_adjustment_set = min(minimal_adjustment_sets, key=len) return minimal_adjustment_set diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 95d1a469..938f403e 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -105,10 +105,11 @@ def create_gene_types( for relationship in list(specification.scenario.constraints): rel_split = str(relationship).split(" ") - if rel_split[1] == ">=": - var_space[rel_split[0]]["low"] = int(rel_split[2]) - elif rel_split[1] == "<=": - var_space[rel_split[0]]["high"] = int(rel_split[2]) + if rel_split[0] in var_space.keys(): + if rel_split[1] == ">=": + var_space[rel_split[0]]["low"] = int(rel_split[2]) + elif rel_split[1] == "<=": + var_space[rel_split[0]]["high"] = int(rel_split[2]) gene_space = [] gene_space.append(var_space[fitness_function.surrogate_model.treatment]) diff --git a/tests/specification_tests/test_causal_dag.py b/tests/specification_tests/test_causal_dag.py index 84e79852..f88a56a7 100644 --- a/tests/specification_tests/test_causal_dag.py +++ b/tests/specification_tests/test_causal_dag.py @@ -2,6 +2,9 @@ import os import networkx as nx from causal_testing.specification.causal_dag import CausalDAG, close_separator, list_all_min_sep +from causal_testing.specification.scenario import Scenario +from causal_testing.specification.variable import Input, Output +from causal_testing.testing.base_test_case import BaseTestCase from tests.test_helpers import create_temp_dir_if_non_existent, remove_temp_dir_if_existent @@ -428,3 +431,34 @@ def test_list_all_min_sep(self): def tearDown(self) -> None: remove_temp_dir_if_existent() + + +class TestHiddenVariableDAG(unittest.TestCase): + """ + Test the CausalDAG identification for the exclusion of hidden variables. + """ + + def setUp(self) -> None: + temp_dir_path = create_temp_dir_if_non_existent() + self.dag_dot_path = os.path.join(temp_dir_path, "dag.dot") + dag_dot = """digraph DAG { rankdir=LR; Z -> X; X -> M; M -> Y; Z -> M; }""" + with open(self.dag_dot_path, "w") as f: + f.write(dag_dot) + + def test_hidden_varaible_adjustment_sets(self): + """Test whether identification produces different adjustment sets depending on if a variable is hidden.""" + causal_dag = CausalDAG(self.dag_dot_path) + z = Input("Z", int) + x = Input("X", int) + m = Input("M", int) + + scenario = Scenario(variables={z, x, m}) + adjustment_sets = causal_dag.identification(BaseTestCase(x, m), scenario) + + z.hidden = True + adjustment_sets_with_hidden = causal_dag.identification(BaseTestCase(x, m), scenario) + + self.assertNotEqual(adjustment_sets, adjustment_sets_with_hidden) + + def tearDown(self) -> None: + remove_temp_dir_if_existent() diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index 99884e35..5fe7b511 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -1,6 +1,16 @@ import unittest -from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, SearchFitnessFunction -from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator +from causal_testing.data_collection.data_collector import ObservationalDataCollector +from causal_testing.specification.causal_dag import CausalDAG +from causal_testing.specification.causal_specification import CausalSpecification +from causal_testing.specification.scenario import Scenario +from causal_testing.specification.variable import Input +from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, SearchFitnessFunction, CausalSurrogateAssistedTestCase, Simulator +from causal_testing.surrogate.surrogate_search_algorithms import GeneticSearchAlgorithm +from causal_testing.testing.estimators import CubicSplineRegressionEstimator +from tests.test_helpers import create_temp_dir_if_non_existent, remove_temp_dir_if_existent +import os +import pandas as pd +import numpy as np class TestSimulationResult(unittest.TestCase): @@ -28,7 +38,16 @@ def test_inputs(self): class TestSearchFitnessFunction(unittest.TestCase): - #TODO: complete tests for causal surrogate + @classmethod + def setUpClass(cls) -> None: + cls.class_df = load_class_df() + + def setUp(self): + temp_dir_path = create_temp_dir_if_non_existent() + self.dag_dot_path = os.path.join(temp_dir_path, "dag.dot") + dag_dot = """digraph DAG { rankdir=LR; Z -> X; X -> M [included=1, expected=positive]; M -> Y [included=1, expected=negative]; Z -> M; }""" + with open(self.dag_dot_path, "w") as f: + f.write(dag_dot) def test_init_valid_values(self): @@ -39,4 +58,77 @@ def test_init_valid_values(self): search_function = SearchFitnessFunction(fitness_function=test_function, surrogate_model=surrogate_model) self.assertTrue(callable(search_function.fitness_function)) - self.assertIsInstance(search_function.surrogate_model, CubicSplineRegressionEstimator) \ No newline at end of file + self.assertIsInstance(search_function.surrogate_model, CubicSplineRegressionEstimator) + + def test_surrogate_model_generation(self): + c_s_a_test_case = CausalSurrogateAssistedTestCase(None, None, None) + + df = self.class_df.copy() + + causal_dag = CausalDAG(self.dag_dot_path) + z = Input("Z", int) + x = Input("X", int) + m = Input("M", int) + y = Input("Y", int) + scenario = Scenario(variables={z, x, m, y}) + specification = CausalSpecification(scenario, causal_dag) + + surrogate_models = c_s_a_test_case.generate_surrogates(specification, ObservationalDataCollector(scenario, df)) + self.assertEqual(len(surrogate_models), 2) + + for surrogate in surrogate_models: + self.assertIsInstance(surrogate, CubicSplineRegressionEstimator) + self.assertNotEqual(surrogate.treatment, "Z") + self.assertNotEqual(surrogate.outcome, "Z") + + def test_causal_surrogate_assisted_execution(self): + df = self.class_df.copy() + + causal_dag = CausalDAG(self.dag_dot_path) + z = Input("Z", int) + x = Input("X", int) + m = Input("M", int) + y = Input("Y", int) + scenario = Scenario(variables={z, x, m, y}, constraints={ + z <= 0, z >= 3, + x <= 0, x >= 3, + m <= 0, m >= 3 + }) + specification = CausalSpecification(scenario, causal_dag) + + search_algorithm = GeneticSearchAlgorithm(config= { + "parent_selection_type": "tournament", + "K_tournament": 4, + "mutation_type": "random", + "mutation_percent_genes": 50, + "mutation_by_replacement": True, + }) + simulator = TestSimulator() + + c_s_a_test_case = CausalSurrogateAssistedTestCase(specification, search_algorithm, simulator) + + result, iterations, result_data = c_s_a_test_case.execute(ObservationalDataCollector(scenario, df)) + + self.assertIsInstance(result, SimulationResult) + self.assertEqual(iterations, 1) + self.assertEqual(len(result_data), 17) + + def tearDown(self) -> None: + remove_temp_dir_if_existent() + +def load_class_df(): + """Get the testing data and put into a dataframe.""" + + class_df = pd.DataFrame({"Z": np.arange(16), "X": np.arange(16), "M": np.arange(16, 32), "Y": np.arange(32,16,-1)}) + return class_df + +class TestSimulator(): + + def run_with_config(self, configuration: dict) -> SimulationResult: + return SimulationResult({"Z": 1, "X": 1, "M": 1, "Y": 1}, True, None) + + def startup(self): + pass + + def shutdown(self): + pass \ No newline at end of file From 2e97af10701c1de9cf9ac545e623dfdeff7bd1f0 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 13:51:26 +0000 Subject: [PATCH 46/60] Updated dependencies --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 46fa2c0b..31634e71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ dependencies = [ "scipy~=1.7", "statsmodels~=0.13", "tabulate~=0.8", - "pydot~=1.4" + "pydot~=1.4", + pygad~=3.2 ] dynamic = ["version"] From ae8b86cdaceb2ca22a242f640ff9fa6fd8492c64 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 13:53:15 +0000 Subject: [PATCH 47/60] Updated dependencies --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 31634e71..dde98953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "statsmodels~=0.13", "tabulate~=0.8", "pydot~=1.4", - pygad~=3.2 + "pygad~=3.2" ] dynamic = ["version"] From c8ebb29e40a5c316e49c9b985cf10caf124eda84 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 14:18:37 +0000 Subject: [PATCH 48/60] Test coverage --- .../test_causal_surrogate_assisted.py | 115 +++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index 5fe7b511..d8567bac 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -4,7 +4,7 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input -from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, SearchFitnessFunction, CausalSurrogateAssistedTestCase, Simulator +from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SimulationResult, SearchFitnessFunction, CausalSurrogateAssistedTestCase, Simulator from causal_testing.surrogate.surrogate_search_algorithms import GeneticSearchAlgorithm from causal_testing.testing.estimators import CubicSplineRegressionEstimator from tests.test_helpers import create_temp_dir_if_non_existent, remove_temp_dir_if_existent @@ -113,6 +113,101 @@ def test_causal_surrogate_assisted_execution(self): self.assertEqual(iterations, 1) self.assertEqual(len(result_data), 17) + def test_causal_surrogate_assisted_execution_failure(self): + df = self.class_df.copy() + + causal_dag = CausalDAG(self.dag_dot_path) + z = Input("Z", int) + x = Input("X", int) + m = Input("M", int) + y = Input("Y", int) + scenario = Scenario(variables={z, x, m, y}, constraints={ + z <= 0, z >= 3, + x <= 0, x >= 3, + m <= 0, m >= 3 + }) + specification = CausalSpecification(scenario, causal_dag) + + search_algorithm = GeneticSearchAlgorithm(config= { + "parent_selection_type": "tournament", + "K_tournament": 4, + "mutation_type": "random", + "mutation_percent_genes": 50, + "mutation_by_replacement": True, + }) + simulator = TestSimulatorFailing() + + c_s_a_test_case = CausalSurrogateAssistedTestCase(specification, search_algorithm, simulator) + + result, iterations, result_data = c_s_a_test_case.execute(ObservationalDataCollector(scenario, df), 1) + + self.assertIsInstance(result, str) + self.assertEqual(iterations, 1) + self.assertEqual(len(result_data), 17) + + def test_causal_surrogate_assisted_execution_custom_aggregator(self): + df = self.class_df.copy() + + causal_dag = CausalDAG(self.dag_dot_path) + z = Input("Z", int) + x = Input("X", int) + m = Input("M", int) + y = Input("Y", int) + scenario = Scenario(variables={z, x, m, y}, constraints={ + z <= 0, z >= 3, + x <= 0, x >= 3, + m <= 0, m >= 3 + }) + specification = CausalSpecification(scenario, causal_dag) + + search_algorithm = GeneticSearchAlgorithm(config= { + "parent_selection_type": "tournament", + "K_tournament": 4, + "mutation_type": "random", + "mutation_percent_genes": 50, + "mutation_by_replacement": True, + }) + simulator = TestSimulator() + + c_s_a_test_case = CausalSurrogateAssistedTestCase(specification, search_algorithm, simulator) + + result, iterations, result_data = c_s_a_test_case.execute(ObservationalDataCollector(scenario, df), + custom_data_aggregator=data_double_aggregator) + + self.assertIsInstance(result, SimulationResult) + self.assertEqual(iterations, 1) + self.assertEqual(len(result_data), 18) + + def test_causal_surrogate_assisted_execution_incorrect_search_config(self): + df = self.class_df.copy() + + causal_dag = CausalDAG(self.dag_dot_path) + z = Input("Z", int) + x = Input("X", int) + m = Input("M", int) + y = Input("Y", int) + scenario = Scenario(variables={z, x, m, y}, constraints={ + z <= 0, z >= 3, + x <= 0, x >= 3, + m <= 0, m >= 3 + }) + specification = CausalSpecification(scenario, causal_dag) + + search_algorithm = GeneticSearchAlgorithm(config= { + "parent_selection_type": "tournament", + "K_tournament": 4, + "mutation_type": "random", + "mutation_percent_genes": 50, + "mutation_by_replacement": True, + "gene_space": "Something" + }) + simulator = TestSimulator() + + c_s_a_test_case = CausalSurrogateAssistedTestCase(specification, search_algorithm, simulator) + + self.assertRaises(c_s_a_test_case.execute(ValueError, ObservationalDataCollector(scenario, df), + custom_data_aggregator=data_double_aggregator)) + def tearDown(self) -> None: remove_temp_dir_if_existent() @@ -122,7 +217,7 @@ def load_class_df(): class_df = pd.DataFrame({"Z": np.arange(16), "X": np.arange(16), "M": np.arange(16, 32), "Y": np.arange(32,16,-1)}) return class_df -class TestSimulator(): +class TestSimulator(Simulator): def run_with_config(self, configuration: dict) -> SimulationResult: return SimulationResult({"Z": 1, "X": 1, "M": 1, "Y": 1}, True, None) @@ -131,4 +226,18 @@ def startup(self): pass def shutdown(self): - pass \ No newline at end of file + pass + +class TestSimulatorFailing(Simulator): + + def run_with_config(self, configuration: dict) -> SimulationResult: + return SimulationResult({"Z": 1, "X": 1, "M": 1, "Y": 1}, False, None) + + def startup(self): + pass + + def shutdown(self): + pass + +def data_double_aggregator(data, new_data): + return data.append(new_data, ignore_index=True).append(new_data, ignore_index=True) \ No newline at end of file From 0c852d3357497337aa963c29a3e1d7948c43f230 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 14:20:56 +0000 Subject: [PATCH 49/60] Linting --- causal_testing/surrogate/surrogate_search_algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 938f403e..d93962d0 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -105,7 +105,7 @@ def create_gene_types( for relationship in list(specification.scenario.constraints): rel_split = str(relationship).split(" ") - if rel_split[0] in var_space.keys(): + if rel_split[0] in var_space: if rel_split[1] == ">=": var_space[rel_split[0]]["low"] = int(rel_split[2]) elif rel_split[1] == "<=": From ee62aa0948f73515405856f0f7a06dce2bcc3618 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 14:31:31 +0000 Subject: [PATCH 50/60] Fixed test --- tests/surrogate_tests/test_causal_surrogate_assisted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index d8567bac..bf44edbc 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -205,7 +205,7 @@ def test_causal_surrogate_assisted_execution_incorrect_search_config(self): c_s_a_test_case = CausalSurrogateAssistedTestCase(specification, search_algorithm, simulator) - self.assertRaises(c_s_a_test_case.execute(ValueError, ObservationalDataCollector(scenario, df), + self.assertRaises(ValueError, c_s_a_test_case.execute(ObservationalDataCollector(scenario, df), custom_data_aggregator=data_double_aggregator)) def tearDown(self) -> None: From 0a816a3e76d7a4125fff40e8d177fd686802ad67 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Mon, 15 Jan 2024 14:41:24 +0000 Subject: [PATCH 51/60] Fixed test --- tests/surrogate_tests/test_causal_surrogate_assisted.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index bf44edbc..bb9f2b09 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -205,8 +205,9 @@ def test_causal_surrogate_assisted_execution_incorrect_search_config(self): c_s_a_test_case = CausalSurrogateAssistedTestCase(specification, search_algorithm, simulator) - self.assertRaises(ValueError, c_s_a_test_case.execute(ObservationalDataCollector(scenario, df), - custom_data_aggregator=data_double_aggregator)) + self.assertRaises(ValueError, c_s_a_test_case.execute, + data_collector=ObservationalDataCollector(scenario, df), + custom_data_aggregator=data_double_aggregator) def tearDown(self) -> None: remove_temp_dir_if_existent() From cc0485fcde18887565305d3dba5ed275c5c723be Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 17 Jan 2024 09:21:47 +0000 Subject: [PATCH 52/60] Fixed bug in fitness function scope --- .../surrogate/causal_surrogate_assisted.py | 23 ++------ .../surrogate/surrogate_search_algorithms.py | 52 ++++++++----------- .../test_causal_surrogate_assisted.py | 15 +----- 3 files changed, 27 insertions(+), 63 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 4c450184..c19d696d 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -19,29 +19,15 @@ class SimulationResult: relationship: str -@dataclass -class SearchFitnessFunction: - """Data class containing the Fitness function and related model""" - - fitness_function: Any - surrogate_model: CubicSplineRegressionEstimator - - class SearchAlgorithm(ABC): """Class to be inherited with the search algorithm consisting of a search function and the fitness function of the space to be searched""" @abstractmethod - def generate_fitness_functions(self, surrogate_models: list[Estimator]) -> list[SearchFitnessFunction]: - """Generates the fitness function of the search space - :param surrogate_models: A list of CubicSplineRegressionEstimator generated for each edge of the DAG - :return: A list of fitness functions mapping to each of the surrogate models in the input""" - - @abstractmethod - def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: + def search(self, surrogate_models: list[CubicSplineRegressionEstimator], specification: CausalSpecification) -> list: """Function which implements a search routine which searches for the optimal fitness value for the specified scenario - :param fitness_functions: The fitness function to be optimised + :param surrogate_models: The surrogate models to be searched :param specification: The Causal Specification (combination of Scenario and Causal Dag)""" @@ -95,8 +81,7 @@ def execute( for i in range(max_executions): surrogate_models = self.generate_surrogates(self.specification, data_collector) - fitness_functions = self.search_algorithm.generate_fitness_functions(surrogate_models) - candidate_test_case, _, surrogate = self.search_algorithm.search(fitness_functions, self.specification) + candidate_test_case, _, surrogate = self.search_algorithm.search(surrogate_models, self.specification) self.simulator.startup() test_result = self.simulator.run_with_config(candidate_test_case) @@ -123,7 +108,7 @@ def execute( def generate_surrogates( self, specification: CausalSpecification, data_collector: ObservationalDataCollector - ) -> list[SearchFitnessFunction]: + ) -> list[CubicSplineRegressionEstimator]: """Generate a surrogate model for each edge of the dag that specifies it is included in the DAG metadata. :param specification: The Causal Specification (combination of Scenario and Causal Dag) :param data_collector: An ObservationalDataCollector which gathers data relevant to the specified scenario diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index d93962d0..6f3a0346 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -7,7 +7,7 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.estimators import CubicSplineRegressionEstimator -from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SearchFitnessFunction +from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm class GeneticSearchAlgorithm(SearchAlgorithm): @@ -25,15 +25,15 @@ def __init__(self, delta=0.05, config: dict = None) -> None: "some_effect": lambda x: abs(1 / x), } - def generate_fitness_functions( - self, surrogate_models: list[CubicSplineRegressionEstimator] - ) -> list[SearchFitnessFunction]: - fitness_functions = [] + def search( + self, surrogate_models: list[CubicSplineRegressionEstimator], specification: CausalSpecification + ) -> list: + solutions = [] for surrogate in surrogate_models: contradiction_function = self.contradiction_functions[surrogate.expected_relationship] - # The returned fitness function after including required variables into the function's scope + # The GA fitness function after including required variables into the function's scope # Unused arguments are required for pygad's fitness function signature def fitness_function(ga, solution, idx): # pylint: disable=unused-argument surrogate.control_value = solution[0] - self.delta @@ -46,25 +46,15 @@ def fitness_function(ga, solution, idx): # pylint: disable=unused-argument ate = surrogate.estimate_ate_calculated(adjustment_dict) return contradiction_function(ate) - - search_fitness_function = SearchFitnessFunction(fitness_function, surrogate) - - fitness_functions.append(search_fitness_function) - - return fitness_functions - - def search(self, fitness_functions: list[SearchFitnessFunction], specification: CausalSpecification) -> list: - solutions = [] - - for fitness_function in fitness_functions: - gene_types, gene_space = self.create_gene_types(fitness_function, specification) + + gene_types, gene_space = self.create_gene_types(surrogate, specification) ga = GA( num_generations=200, num_parents_mating=4, - fitness_func=fitness_function.fitness_function, + fitness_func=fitness_function, sol_per_pop=10, - num_genes=1 + len(fitness_function.surrogate_model.adjustment_set), + num_genes=1 + len(surrogate.adjustment_set), gene_space=gene_space, gene_type=gene_types, ) @@ -82,24 +72,24 @@ def search(self, fitness_functions: list[SearchFitnessFunction], specification: solution, fitness, _ = ga.best_solution() solution_dict = {} - solution_dict[fitness_function.surrogate_model.treatment] = solution[0] - for idx, adj in enumerate(fitness_function.surrogate_model.adjustment_set): + solution_dict[surrogate.treatment] = solution[0] + for idx, adj in enumerate(surrogate.adjustment_set): solution_dict[adj] = solution[idx + 1] - solutions.append((solution_dict, fitness, fitness_function.surrogate_model)) + solutions.append((solution_dict, fitness, surrogate)) return max(solutions, key=itemgetter(1)) # This can be done better with fitness normalisation between edges @staticmethod def create_gene_types( - fitness_function: SearchFitnessFunction, specification: CausalSpecification + surrogate_model: CubicSplineRegressionEstimator, specification: CausalSpecification ) -> tuple[list, list]: """Generate the gene_types and gene_space for a given fitness function and specification - :param fitness_function: Instance of SearchFitnessFunction + :param surrogate_model: Instance of a CubicSplineRegressionEstimator :param specification: The Causal Specification (combination of Scenario and Causal Dag)""" var_space = {} - var_space[fitness_function.surrogate_model.treatment] = {} - for adj in fitness_function.surrogate_model.adjustment_set: + var_space[surrogate_model.treatment] = {} + for adj in surrogate_model.adjustment_set: var_space[adj] = {} for relationship in list(specification.scenario.constraints): @@ -112,12 +102,12 @@ def create_gene_types( var_space[rel_split[0]]["high"] = int(rel_split[2]) gene_space = [] - gene_space.append(var_space[fitness_function.surrogate_model.treatment]) - for adj in fitness_function.surrogate_model.adjustment_set: + gene_space.append(var_space[surrogate_model.treatment]) + for adj in surrogate_model.adjustment_set: gene_space.append(var_space[adj]) gene_types = [] - gene_types.append(specification.scenario.variables.get(fitness_function.surrogate_model.treatment).datatype) - for adj in fitness_function.surrogate_model.adjustment_set: + gene_types.append(specification.scenario.variables.get(surrogate_model.treatment).datatype) + for adj in surrogate_model.adjustment_set: gene_types.append(specification.scenario.variables.get(adj).datatype) return gene_types, gene_space diff --git a/tests/surrogate_tests/test_causal_surrogate_assisted.py b/tests/surrogate_tests/test_causal_surrogate_assisted.py index bb9f2b09..43afe0e4 100644 --- a/tests/surrogate_tests/test_causal_surrogate_assisted.py +++ b/tests/surrogate_tests/test_causal_surrogate_assisted.py @@ -4,7 +4,7 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input -from causal_testing.surrogate.causal_surrogate_assisted import SearchAlgorithm, SimulationResult, SearchFitnessFunction, CausalSurrogateAssistedTestCase, Simulator +from causal_testing.surrogate.causal_surrogate_assisted import SimulationResult, CausalSurrogateAssistedTestCase, Simulator from causal_testing.surrogate.surrogate_search_algorithms import GeneticSearchAlgorithm from causal_testing.testing.estimators import CubicSplineRegressionEstimator from tests.test_helpers import create_temp_dir_if_non_existent, remove_temp_dir_if_existent @@ -36,7 +36,7 @@ def test_inputs(self): self.assertEqual(result.relationship, relationship) -class TestSearchFitnessFunction(unittest.TestCase): +class TestCausalSurrogate(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -49,17 +49,6 @@ def setUp(self): with open(self.dag_dot_path, "w") as f: f.write(dag_dot) - def test_init_valid_values(self): - - test_function = lambda x: x **2 - - surrogate_model = CubicSplineRegressionEstimator("", 0, 0, set(), "", 4) - - search_function = SearchFitnessFunction(fitness_function=test_function, surrogate_model=surrogate_model) - - self.assertTrue(callable(search_function.fitness_function)) - self.assertIsInstance(search_function.surrogate_model, CubicSplineRegressionEstimator) - def test_surrogate_model_generation(self): c_s_a_test_case = CausalSurrogateAssistedTestCase(None, None, None) From 04b377d6763a5b9965ebecc8da5d3d248466ee54 Mon Sep 17 00:00:00 2001 From: Richard Somers Date: Wed, 17 Jan 2024 09:25:37 +0000 Subject: [PATCH 53/60] linting --- causal_testing/surrogate/causal_surrogate_assisted.py | 8 +++++--- causal_testing/surrogate/surrogate_search_algorithms.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index c19d696d..932a91e2 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -2,12 +2,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Callable, Any +from typing import Callable from causal_testing.data_collection.data_collector import ObservationalDataCollector from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.testing.base_test_case import BaseTestCase -from causal_testing.testing.estimators import Estimator, CubicSplineRegressionEstimator +from causal_testing.testing.estimators import CubicSplineRegressionEstimator @dataclass @@ -24,7 +24,9 @@ class SearchAlgorithm(ABC): space to be searched""" @abstractmethod - def search(self, surrogate_models: list[CubicSplineRegressionEstimator], specification: CausalSpecification) -> list: + def search( + self, surrogate_models: list[CubicSplineRegressionEstimator], specification: CausalSpecification + ) -> list: """Function which implements a search routine which searches for the optimal fitness value for the specified scenario :param surrogate_models: The surrogate models to be searched diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 6f3a0346..ab74f501 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -46,7 +46,7 @@ def fitness_function(ga, solution, idx): # pylint: disable=unused-argument ate = surrogate.estimate_ate_calculated(adjustment_dict) return contradiction_function(ate) - + gene_types, gene_space = self.create_gene_types(surrogate, specification) ga = GA( From bbafb24b6b956d28ff971991170f2f6fdd923588 Mon Sep 17 00:00:00 2001 From: f-allian Date: Wed, 17 Jan 2024 11:51:45 +0000 Subject: [PATCH 54/60] Fix: structure and files uploaded to dafni. --- dafni/.dockerignore | 5 +- dafni/.env | 5 + dafni/Dockerfile | 23 +- dafni/README.md | 29 +++ dafni/{ => data}/inputs/causal_tests.json | 0 dafni/{ => data}/inputs/dag.dot | 0 .../inputs/runtime_data.csv} | 0 dafni/{ => data}/inputs/variables.json | 0 dafni/data/outputs/causal_tests_results.json | 215 ++++++++++++++++++ dafni/docker-compose.yaml | 2 - dafni/main_dafni.py | 27 ++- dafni/model_definition.yaml | 65 +++--- 12 files changed, 315 insertions(+), 56 deletions(-) create mode 100644 dafni/.env create mode 100644 dafni/README.md rename dafni/{ => data}/inputs/causal_tests.json (100%) rename dafni/{ => data}/inputs/dag.dot (100%) rename dafni/{inputs/simulated_data.csv => data/inputs/runtime_data.csv} (100%) rename dafni/{ => data}/inputs/variables.json (100%) create mode 100644 dafni/data/outputs/causal_tests_results.json diff --git a/dafni/.dockerignore b/dafni/.dockerignore index 0af95714..52a1d65d 100644 --- a/dafni/.dockerignore +++ b/dafni/.dockerignore @@ -1,5 +1,4 @@ ../* ../!causal_testing -../!LICENSE -./!inputs -./!main_dafni.py \ No newline at end of file +./!main_dafni.py +./!data/ \ No newline at end of file diff --git a/dafni/.env b/dafni/.env new file mode 100644 index 00000000..3143724b --- /dev/null +++ b/dafni/.env @@ -0,0 +1,5 @@ +#.env +VARIABLES_PATH=./data/inputs/variables.json +CAUSAL_TESTS=./data/inputs/causal_tests.json +DATA_PATH=./data/inputs/runtime_data.csv +DAG_PATH=./data/inputs/dag.dot \ No newline at end of file diff --git a/dafni/Dockerfile b/dafni/Dockerfile index 0d83569a..8969ad3f 100644 --- a/dafni/Dockerfile +++ b/dafni/Dockerfile @@ -11,22 +11,23 @@ ENV PYTHONUNBUFFERED=1 #Label maintainer LABEL maintainer="Dr. Farhad Allian - The University of Sheffield" -# Copy the source code and test files from build into the container -COPY --chown=nobody ./causal_testing /usr/src/app/ -COPY --chown=nobody ./dafni/inputs /usr/src/app/inputs/ -COPY --chown=nobody ./dafni/main_dafni.py /usr/src/app/ +# Create a folder for the source code/outputs +RUN mkdir -p ./causal_testing +RUN mkdir -p ./data/outputs -# Change the working directory -WORKDIR /usr/src/app/ +# Copy the source code and test files from build into the container +COPY --chown=nobody ../causal_testing ./causal_testing +COPY --chown=nobody ./dafni/main_dafni.py ./ +COPY --chown=nobody ./dafni/data/inputs ./data/inputs # Install core dependencies using PyPi RUN pip install causal-testing-framework --no-cache-dir -# Use the necessaary environment variables for the script's inputs -ENV VARIABLES=./inputs/variables.json \ - CAUSAL_TESTS=./inputs/causal_tests.json \ - DATA_PATH=./inputs/simulated_data.csv \ - DAG_PATH=./inputs/dag.dot +#For local testing purposes +ENV VARIABLES_PATH=./data/inputs/variables.json \ + CAUSAL_TESTS=./data/inputs/causal_tests.json \ + DATA_PATH=./data/inputs/runtime_data.csv \ + DAG_PATH=./data/inputs/dag.dot # Define the entrypoint/commands CMD python main_dafni.py --variables_path $VARIABLES_PATH --dag_path $DAG_PATH --data_path $DATA_PATH --tests_path $CAUSAL_TESTS diff --git a/dafni/README.md b/dafni/README.md new file mode 100644 index 00000000..29833d61 --- /dev/null +++ b/dafni/README.md @@ -0,0 +1,29 @@ +# Causal Testing Framework on DAFNI + +- This directory contains the containerisation files of the causal testing framework using Docker, which is used +to upload the framework onto [DAFNI](https://www.dafni.ac.uk). +- It is **not** recommended to install the causal testing framework using Docker, and should only be installed + using [PyPI](https://pypi.org/project/causal-testing-framework/). + +### Folders + +- `data` contains two sub-folders (the structure is important for DAFNI). + - `inputs` is a folder that contains the input files that are (separately) uploaded to DAFNI. + - `causal_tests.json` is a JSON file that contains the causal tests. + - `variables.json` is a JSON file that contains the variables and constraints to be used. + - `dag.dot` is a dot file that contains the directed acyclc graph (dag) file. + - `runtime_data.csv` is a csv file that contains the runtime data. + + - `outputs` is a folder where the `causal_tests_results.json` output file is created. + +### Docker files +- `main_dafni.py` is the entry-point to the causal testing framework that is used by Docker. +- `model_definition.yaml` is the model metadata that is required to be uploaded to DAFNI. +- `.env` is an example of a configuration file containing the environment variables. This is only required + if using `docker-compose` to build the image. +- `Dockerfile` is the main blueprint that builds the image. +- `.dockerignore` tells the Dockerfile which files to not include in the image. +- `docker-compose.yaml` is another method of building the image and running the container in one line. + Note: the `.env` file that contains the environment variables for `main_dafni.py` is only used here. + + diff --git a/dafni/inputs/causal_tests.json b/dafni/data/inputs/causal_tests.json similarity index 100% rename from dafni/inputs/causal_tests.json rename to dafni/data/inputs/causal_tests.json diff --git a/dafni/inputs/dag.dot b/dafni/data/inputs/dag.dot similarity index 100% rename from dafni/inputs/dag.dot rename to dafni/data/inputs/dag.dot diff --git a/dafni/inputs/simulated_data.csv b/dafni/data/inputs/runtime_data.csv similarity index 100% rename from dafni/inputs/simulated_data.csv rename to dafni/data/inputs/runtime_data.csv diff --git a/dafni/inputs/variables.json b/dafni/data/inputs/variables.json similarity index 100% rename from dafni/inputs/variables.json rename to dafni/data/inputs/variables.json diff --git a/dafni/data/outputs/causal_tests_results.json b/dafni/data/outputs/causal_tests_results.json new file mode 100644 index 00000000..f5e503aa --- /dev/null +++ b/dafni/data/outputs/causal_tests_results.json @@ -0,0 +1,215 @@ +[ + { + "name": "max_doses _||_ cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinations": "NoEffect" + }, + "formula": "cum_vaccinations ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 252628.1066666667, + "ci_low": 252271.33332001517, + "ci_high": 252984.8800133182 + } + }, + { + "name": "max_doses _||_ cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 213111.93333333335, + "ci_low": 212755.15056812647, + "ci_high": 213468.71609854023 + } + }, + { + "name": "max_doses _||_ cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 2666.3066666666664, + "ci_low": 2619.972040648758, + "ci_high": 2712.6412926845746 + } + }, + { + "name": "vaccine --> cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 315785.1333333332, + "ci_low": 315339.1666500188, + "ci_high": 316231.1000166476 + } + }, + { + "name": "vaccine --> cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 266389.91666666657, + "ci_low": 265943.93821015797, + "ci_high": 266835.89512317517 + } + }, + { + "name": "vaccine --> cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 3332.883333333332, + "ci_low": 3274.9650508109467, + "ci_high": 3390.801615855717 + } + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 0.9998656401531605, + "ci_low": 0.9929245394499968, + "ci_high": 1.0068067408563242 + } + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "failed": false, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": -0.006416682407515084, + "ci_low": -0.05663010083886572, + "ci_high": 0.043796736023835554 + } + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinated" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false, + "failed": false, + "result": { + "treatment": "cum_vaccinated", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": -0.006176900588291234, + "ci_low": -0.05639349612119588, + "ci_high": 0.04403969494461341 + } + } +] diff --git a/dafni/docker-compose.yaml b/dafni/docker-compose.yaml index a1228bad..9a38d1c4 100644 --- a/dafni/docker-compose.yaml +++ b/dafni/docker-compose.yaml @@ -8,5 +8,3 @@ services: - .env volumes: - .:/usr/src/app - - ./inputs:/usr/src/app/inputs/ - - ./outputs:/usr/src/app/outputs/ \ No newline at end of file diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py index 5ab40c8d..c7983ca3 100644 --- a/dafni/main_dafni.py +++ b/dafni/main_dafni.py @@ -1,5 +1,3 @@ -import warnings -warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*") import os from pathlib import Path import argparse @@ -70,16 +68,15 @@ def get_args(test_args=None) -> argparse.Namespace: if args.output_path is None: - args.output_path = "./outputs/results/"+"_".join([os.path.splitext(os.path.basename(x))[0] - for x in args.data_path]) + "_results.json" + args.output_path = "./data/outputs/causal_tests_results.json" - Path(args.output_path).parent.mkdir(exist_ok=True, parents=True) + Path(args.output_path).parent.mkdir(exist_ok=True) else: args.output_path = Path(args.output_path) - args.output_path.parent.mkdir(exist_ok=True, parents=True) + args.output_path.parent.mkdir(exist_ok=True) return args @@ -93,7 +90,9 @@ def read_variables(variables_path: Path) -> dict: """ if not variables_path.exists() or variables_path.is_dir(): - raise ValidationError(f"Cannot find a valid settings file at {variables_path.absolute()}.") + raise FileNotFoundError + + print(f"JSON file not found at the specified location: {variables_path}") else: @@ -126,8 +125,13 @@ def validate_variables(data_dict: dict) -> tuple: for variable, _inputs in zip(variables, inputs): if "constraint" in variable: + constraints.add(_inputs.z3 == variable["constraint"]) + else: + + raise ValidationError("Cannot find the variables defined by the causal tests.") + return inputs, outputs, constraints @@ -137,10 +141,10 @@ def main(): """ args = get_args() - # Step 0: Read in the runtime dataset(s) - try: + # Step 0: Read in the runtime dataset(s) + data_frame = pd.concat([pd.read_csv(d) for d in args.data_path]) # Step 1: Read in the JSON input/output variables and parse io arguments @@ -191,7 +195,8 @@ def main(): test["result"].pop("control_value") - with open(f"{args.output_path}", "w") as f: + + with open(args.output_path, "w") as f: print(json.dumps(test_outcomes, indent=2), file=f) @@ -207,4 +212,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml index a5e6d602..901e2570 100644 --- a/dafni/model_definition.yaml +++ b/dafni/model_definition.yaml @@ -5,7 +5,7 @@ kind: M api_version: v1beta3 metadata: display_name: Causal Testing Framework - name: causal_testing + name: causal-testing-framework publisher: The CITCOM Team, The University of Sheffield type: model summary: A Causal Inference-Driven Software Testing Framework @@ -16,43 +16,50 @@ metadata: the anticipated cause-effect relationships amongst the inputs and outputs of the system-under-test and the supporting mathematical framework to design statistical procedures capable of making causal inferences. Each causal test case focuses on the causal effect of an intervention made to the system-under test. - contact_point_name: Dr. Farhad Allian + contact_point_name: Farhad Allian contact_point_email: farhad.allian@sheffield.ac.uk source_code: https://github.com/CITCOM-project/CausalTestingFramework - license: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme - + licence: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme + spec: inputs: - parameters: - - name: causal_tests - title: Causal tests filename - description: A .json file containing the causal tests to be used - type: string - required: true - - name: variables - title: Variables filename - description: A .json file containing the input and output variables to be used - type: string - required: true - - name: dag_file - title: DAG filename - description: A .dot file containing the input DAG to be used - type: string - required: true - dataslots: - - name: runtime_data + - name: Runtime csv data description: > A .csv file containing the input runtime data to be used default: - - #TODO - path: /inputs/dataslots - required: false - - name: dag_file + - 2b7336cd-eb68-4c1f-8f91-26d8969b8cb3 + path: inputs/ + required: true + + - name: DAG data description: > A .dot file containing the input DAG to be used default: - - #TODO - path: /data/tests/inputs - required: false \ No newline at end of file + - 74665fdb-43a2-4c51-b81e-d5299b38bf8c + path: inputs/ + required: true + + - name: Causal tests + description: > + A .JSON file containing the input causal tests to be used + default: + - 6f2f7c1f-81b4-4804-8f86-cca304dc7f66 + path: inputs/ + required: true + + - name: Variables + description: > + A .JSON file containing the input variables to be used + default: + - 02e755c8-952b-461a-a914-4f4ffbe2edf1 + path: inputs/ + required: true + + outputs: + datasets: + - name: causal_test_results.json + type: json + description: > + A JSON file containing the output causal test results. \ No newline at end of file From 743b130f6f06f06ef23d091cf02e3f0df3d5dfc2 Mon Sep 17 00:00:00 2001 From: f-allian Date: Wed, 17 Jan 2024 13:40:40 +0000 Subject: [PATCH 55/60] Fix: linting and added __init__.py to surrogate --- causal_testing/surrogate/__init__.py | 0 causal_testing/surrogate/surrogate_search_algorithms.py | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 causal_testing/surrogate/__init__.py diff --git a/causal_testing/surrogate/__init__.py b/causal_testing/surrogate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index ab74f501..558e876d 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,5 +1,7 @@ """Module containing implementation of search algorithm for surrogate search """ # pylint: disable=cell-var-from-loop +# pylint: disable=too-few-public-methods +# pylint: disable=too-many-locals # Fitness functions are required to be iteratively defined, including all variables within. from operator import itemgetter From 5e5055d9850a9858edb11f328992170982590713 Mon Sep 17 00:00:00 2001 From: f-allian Date: Wed, 17 Jan 2024 13:55:56 +0000 Subject: [PATCH 56/60] Fix: linting error --- causal_testing/surrogate/surrogate_search_algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 558e876d..c1780fce 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,7 +1,7 @@ """Module containing implementation of search algorithm for surrogate search """ # pylint: disable=cell-var-from-loop -# pylint: disable=too-few-public-methods # pylint: disable=too-many-locals +# pylint: disable=R0903 # Fitness functions are required to be iteratively defined, including all variables within. from operator import itemgetter From 996d013227c300310ac78708cd7cf7f23498c6a0 Mon Sep 17 00:00:00 2001 From: f-allian Date: Wed, 17 Jan 2024 14:00:15 +0000 Subject: [PATCH 57/60] Fix: linting error #2 --- causal_testing/surrogate/surrogate_search_algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index c1780fce..c8c5f691 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,7 +1,6 @@ """Module containing implementation of search algorithm for surrogate search """ # pylint: disable=cell-var-from-loop # pylint: disable=too-many-locals -# pylint: disable=R0903 # Fitness functions are required to be iteratively defined, including all variables within. from operator import itemgetter @@ -15,6 +14,7 @@ class GeneticSearchAlgorithm(SearchAlgorithm): """Implementation of SearchAlgorithm class. Implements genetic search algorithm for surrogate models.""" + # pylint: disable=too-few-public-methods def __init__(self, delta=0.05, config: dict = None) -> None: super().__init__() From eaa4360567983a6e5a533117dde1356bbd656540 Mon Sep 17 00:00:00 2001 From: f-allian Date: Wed, 17 Jan 2024 14:13:48 +0000 Subject: [PATCH 58/60] Fix: linting error #3 --- causal_testing/surrogate/causal_surrogate_assisted.py | 2 ++ causal_testing/surrogate/surrogate_search_algorithms.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index 932a91e2..a6767164 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -1,3 +1,5 @@ +# pylint: disable=too-few-public-methods + """Module containing classes to define and run causal surrogate assisted test cases""" from abc import ABC, abstractmethod diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index c8c5f691..9261479f 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -14,7 +14,6 @@ class GeneticSearchAlgorithm(SearchAlgorithm): """Implementation of SearchAlgorithm class. Implements genetic search algorithm for surrogate models.""" - # pylint: disable=too-few-public-methods def __init__(self, delta=0.05, config: dict = None) -> None: super().__init__() From d4322364b8fd3c0e254dbc766b4e3549d9852288 Mon Sep 17 00:00:00 2001 From: f-allian Date: Fri, 26 Jan 2024 17:01:33 +0000 Subject: [PATCH 59/60] Fix: supress lintings locally --- causal_testing/surrogate/causal_surrogate_assisted.py | 4 +--- causal_testing/surrogate/surrogate_search_algorithms.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/causal_testing/surrogate/causal_surrogate_assisted.py b/causal_testing/surrogate/causal_surrogate_assisted.py index a6767164..77ef88f7 100644 --- a/causal_testing/surrogate/causal_surrogate_assisted.py +++ b/causal_testing/surrogate/causal_surrogate_assisted.py @@ -1,5 +1,3 @@ -# pylint: disable=too-few-public-methods - """Module containing classes to define and run causal surrogate assisted test cases""" from abc import ABC, abstractmethod @@ -21,7 +19,7 @@ class SimulationResult: relationship: str -class SearchAlgorithm(ABC): +class SearchAlgorithm(ABC): # pylint: disable=too-few-public-methods """Class to be inherited with the search algorithm consisting of a search function and the fitness function of the space to be searched""" diff --git a/causal_testing/surrogate/surrogate_search_algorithms.py b/causal_testing/surrogate/surrogate_search_algorithms.py index 9261479f..71e5d655 100644 --- a/causal_testing/surrogate/surrogate_search_algorithms.py +++ b/causal_testing/surrogate/surrogate_search_algorithms.py @@ -1,6 +1,4 @@ """Module containing implementation of search algorithm for surrogate search """ -# pylint: disable=cell-var-from-loop -# pylint: disable=too-many-locals # Fitness functions are required to be iteratively defined, including all variables within. from operator import itemgetter @@ -26,6 +24,7 @@ def __init__(self, delta=0.05, config: dict = None) -> None: "some_effect": lambda x: abs(1 / x), } + # pylint: disable=too-many-locals def search( self, surrogate_models: list[CubicSplineRegressionEstimator], specification: CausalSpecification ) -> list: @@ -36,6 +35,7 @@ def search( # The GA fitness function after including required variables into the function's scope # Unused arguments are required for pygad's fitness function signature + #pylint: disable=cell-var-from-loop def fitness_function(ga, solution, idx): # pylint: disable=unused-argument surrogate.control_value = solution[0] - self.delta surrogate.treatment_value = solution[0] + self.delta From 46216aa4711324a23a1b77b3243819a49a1c4e92 Mon Sep 17 00:00:00 2001 From: f-allian Date: Tue, 30 Jan 2024 14:52:18 +0000 Subject: [PATCH 60/60] Fix: final linting --- dafni/main_dafni.py | 51 ++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py index c7983ca3..e6b142f3 100644 --- a/dafni/main_dafni.py +++ b/dafni/main_dafni.py @@ -1,4 +1,9 @@ -import os +""" + +Entrypoint script to run the causal testing framework on DAFNI + +""" + from pathlib import Path import argparse import json @@ -14,7 +19,6 @@ class ValidationError(Exception): """ Custom class to capture validation errors in this script """ - pass def get_args(test_args=None) -> argparse.Namespace: @@ -24,20 +28,22 @@ def get_args(test_args=None) -> argparse.Namespace: :returns: - argparse.Namespace - A Namsespace consisting of the arguments to this script """ - parser = argparse.ArgumentParser(description="A script for running the causal testing famework on DAFNI.") + parser = argparse.ArgumentParser(description="A script for running the CTF on DAFNI.") parser.add_argument( "--data_path", required=True, help="Path to the input runtime data (.csv)", nargs="+") parser.add_argument('--tests_path', required=True, - help='Path to the input configuration file containing the causal tests (.json)') + help='Input configuration file path ' + 'containing the causal tests (.json)') parser.add_argument('--variables_path', required=True, - help='Path to the input configuration file containing the predefined variables (.json)') + help='Input configuration file path ' + 'containing the predefined variables (.json)') parser.add_argument("--dag_path", required=True, - help="Path to the input file containing a valid DAG (.dot). " + help="Input configuration file path containing a valid DAG (.dot). " "Note: this must be supplied if the --tests argument isn't provided.") parser.add_argument('--output_path', required=False, help='Path to the output directory.') @@ -81,7 +87,7 @@ def get_args(test_args=None) -> argparse.Namespace: return args -def read_variables(variables_path: Path) -> dict: +def read_variables(variables_path: Path) -> FileNotFoundError | dict: """ Function to read the variables.json file specified by the user :param variables_path: A Path object of the user-specified file path @@ -90,17 +96,15 @@ def read_variables(variables_path: Path) -> dict: """ if not variables_path.exists() or variables_path.is_dir(): - raise FileNotFoundError - print(f"JSON file not found at the specified location: {variables_path}") - else: + raise FileNotFoundError - with variables_path.open('r') as file: + with variables_path.open('r') as file: - inputs = json.load(file) + inputs = json.load(file) - return inputs + return inputs def validate_variables(data_dict: dict) -> tuple: @@ -108,26 +112,27 @@ def validate_variables(data_dict: dict) -> tuple: Function to validate the variables defined in the causal tests :param data_dict: A dictionary consisting of the pre-defined variables for the causal tests :returns: - - tuple - Tuple consisting of the inputs, outputs and constraints to pass into the modelling scenario + - Tuple containing the inputs, outputs and constraints to pass into the modelling scenario """ if data_dict["variables"]: variables = data_dict["variables"] - inputs = [Input(variable["name"], eval(variable["datatype"])) for variable in variables if + inputs = [Input(variable["name"], eval(variable["datatype"])) + for variable in variables if variable["typestring"] == "Input"] - outputs = [Output(variable["name"], eval(variable["datatype"])) for variable in variables if + outputs = [Output(variable["name"], eval(variable["datatype"])) + for variable in variables if variable["typestring"] == "Output"] constraints = set() - for variable, _inputs in zip(variables, inputs): + for variable, input_var in zip(variables, inputs): if "constraint" in variable: - constraints.add(_inputs.z3 == variable["constraint"]) - + constraints.add(input_var.z3 == variable["constraint"]) else: raise ValidationError("Cannot find the variables defined by the causal tests.") @@ -180,7 +185,8 @@ def main(): json_utility.setup(scenario=modelling_scenario, data=data_frame) # Step 7: Run the causal tests - test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, mutates={}, estimators=estimators, + test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, + mutates={}, estimators=estimators, f_flag=args.f) # Step 8: Update, print and save the final outputs @@ -196,7 +202,7 @@ def main(): test["result"].pop("control_value") - with open(args.output_path, "w") as f: + with open(args.output_path, "w", encoding="utf-8") as f: print(json.dumps(test_outcomes, indent=2), file=f) @@ -208,7 +214,8 @@ def main(): else: - print(f"Execution successful. Output file saved at {Path(args.output_path).parent.resolve()}") + print(f"Execution successful. " + f"Output file saved at {Path(args.output_path).parent.resolve()}") if __name__ == "__main__":