diff --git a/dafni/.dockerignore b/dafni/.dockerignore new file mode 100644 index 00000000..52a1d65d --- /dev/null +++ b/dafni/.dockerignore @@ -0,0 +1,4 @@ +../* +../!causal_testing +./!main_dafni.py +./!data/ \ No newline at end of file diff --git a/dafni/.env b/dafni/.env new file mode 100644 index 00000000..3143724b --- /dev/null +++ b/dafni/.env @@ -0,0 +1,5 @@ +#.env +VARIABLES_PATH=./data/inputs/variables.json +CAUSAL_TESTS=./data/inputs/causal_tests.json +DATA_PATH=./data/inputs/runtime_data.csv +DAG_PATH=./data/inputs/dag.dot \ No newline at end of file diff --git a/dafni/Dockerfile b/dafni/Dockerfile new file mode 100644 index 00000000..8969ad3f --- /dev/null +++ b/dafni/Dockerfile @@ -0,0 +1,33 @@ +# Define the Python version neded for CTF +FROM python:3.10-slim + +## Prevents Python from writing pyc files +ENV PYTHONDONTWRITEBYTECODE=1 +# +## Keeps Python from buffering stdout and stderr to avoid the framework +## from crashing without emitting any logs due to buffering +ENV PYTHONUNBUFFERED=1 + +#Label maintainer +LABEL maintainer="Dr. Farhad Allian - The University of Sheffield" + +# Create a folder for the source code/outputs +RUN mkdir -p ./causal_testing +RUN mkdir -p ./data/outputs + +# Copy the source code and test files from build into the container +COPY --chown=nobody ../causal_testing ./causal_testing +COPY --chown=nobody ./dafni/main_dafni.py ./ +COPY --chown=nobody ./dafni/data/inputs ./data/inputs + +# Install core dependencies using PyPi +RUN pip install causal-testing-framework --no-cache-dir + +#For local testing purposes +ENV VARIABLES_PATH=./data/inputs/variables.json \ + CAUSAL_TESTS=./data/inputs/causal_tests.json \ + DATA_PATH=./data/inputs/runtime_data.csv \ + DAG_PATH=./data/inputs/dag.dot + +# Define the entrypoint/commands +CMD python main_dafni.py --variables_path $VARIABLES_PATH --dag_path $DAG_PATH --data_path $DATA_PATH --tests_path $CAUSAL_TESTS diff --git a/dafni/README.md b/dafni/README.md new file mode 100644 index 00000000..29833d61 --- /dev/null +++ b/dafni/README.md @@ -0,0 +1,29 @@ +# Causal Testing Framework on DAFNI + +- This directory contains the containerisation files of the causal testing framework using Docker, which is used +to upload the framework onto [DAFNI](https://www.dafni.ac.uk). +- It is **not** recommended to install the causal testing framework using Docker, and should only be installed + using [PyPI](https://pypi.org/project/causal-testing-framework/). + +### Folders + +- `data` contains two sub-folders (the structure is important for DAFNI). + - `inputs` is a folder that contains the input files that are (separately) uploaded to DAFNI. + - `causal_tests.json` is a JSON file that contains the causal tests. + - `variables.json` is a JSON file that contains the variables and constraints to be used. + - `dag.dot` is a dot file that contains the directed acyclc graph (dag) file. + - `runtime_data.csv` is a csv file that contains the runtime data. + + - `outputs` is a folder where the `causal_tests_results.json` output file is created. + +### Docker files +- `main_dafni.py` is the entry-point to the causal testing framework that is used by Docker. +- `model_definition.yaml` is the model metadata that is required to be uploaded to DAFNI. +- `.env` is an example of a configuration file containing the environment variables. This is only required + if using `docker-compose` to build the image. +- `Dockerfile` is the main blueprint that builds the image. +- `.dockerignore` tells the Dockerfile which files to not include in the image. +- `docker-compose.yaml` is another method of building the image and running the container in one line. + Note: the `.env` file that contains the environment variables for `main_dafni.py` is only used here. + + diff --git a/dafni/data/inputs/causal_tests.json b/dafni/data/inputs/causal_tests.json new file mode 100644 index 00000000..6c78c57f --- /dev/null +++ b/dafni/data/inputs/causal_tests.json @@ -0,0 +1,136 @@ +{ + "tests": [ + { + "name": "max_doses _||_ cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinations": "NoEffect" + }, + "formula": "cum_vaccinations ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "max_doses _||_ cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "max_doses _||_ cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "vaccine --> cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinated" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false + } + ] +} \ No newline at end of file diff --git a/dafni/data/inputs/dag.dot b/dafni/data/inputs/dag.dot new file mode 100644 index 00000000..43628817 --- /dev/null +++ b/dafni/data/inputs/dag.dot @@ -0,0 +1,7 @@ +digraph CausalDAG { + rankdir=LR; + "vaccine" -> "cum_vaccinations"; + "vaccine" -> "cum_vaccinated"; + "vaccine" -> "cum_infections"; + "max_doses"; +} \ No newline at end of file diff --git a/dafni/data/inputs/runtime_data.csv b/dafni/data/inputs/runtime_data.csv new file mode 100644 index 00000000..3ac0a3ac --- /dev/null +++ b/dafni/data/inputs/runtime_data.csv @@ -0,0 +1,31 @@ +pop_size,pop_type,pop_infected,n_days,vaccine_type,use_waning,rand_seed,cum_infections,cum_deaths,cum_recoveries,cum_vaccinations,cum_vaccinated,target_elderly,vaccine,max_doses +50000,hybrid,1000,50,pfizer,True,1169,6277.0,15.0,6175.0,629466.0,530715.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,8888,6381.0,18.0,6274.0,630796.0,532010.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,370,6738.0,15.0,6621.0,631705.0,532864.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,9981,6784.0,18.0,6682.0,634582.0,535795.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6305,6757.0,20.0,6659.0,631292.0,532464.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1993,5844.0,17.0,5755.0,633314.0,534478.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1938,6465.0,19.0,6353.0,627724.0,528993.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4797,7044.0,15.0,6919.0,631246.0,532433.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2308,6878.0,6.0,6801.0,628865.0,530038.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4420,6429.0,11.0,6348.0,633803.0,535030.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2314,6566.0,15.0,6477.0,629288.0,530550.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7813,6913.0,17.0,6818.0,629290.0,530512.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1050,6963.0,14.0,6860.0,627981.0,529212.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3215,6671.0,17.0,6577.0,628802.0,530038.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2286,6597.0,13.0,6505.0,628986.0,530195.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3080,6926.0,16.0,6834.0,633636.0,534904.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7405,6438.0,15.0,6347.0,630353.0,531540.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,9668,6577.0,15.0,6485.0,631257.0,532409.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,8211,6197.0,13.0,6103.0,633827.0,535056.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4686,6761.0,16.0,6653.0,630557.0,531737.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3591,7328.0,24.0,7214.0,629949.0,531124.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4834,6617.0,22.0,6512.0,632609.0,533705.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6142,7017.0,17.0,6902.0,635965.0,537252.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6877,6845.0,15.0,6753.0,635678.0,536925.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1878,6480.0,20.0,6390.0,630807.0,531999.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3761,6972.0,16.0,6890.0,631100.0,532329.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1741,6581.0,20.0,6491.0,632835.0,534088.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,5592,6561.0,19.0,6461.0,636799.0,537959.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7979,7075.0,17.0,6966.0,632902.0,534140.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,71,6291.0,13.0,6203.0,631694.0,532901.0,True,1,2 diff --git a/dafni/data/inputs/variables.json b/dafni/data/inputs/variables.json new file mode 100644 index 00000000..8e4a1add --- /dev/null +++ b/dafni/data/inputs/variables.json @@ -0,0 +1,47 @@ +{ + "variables": [ + { + "name": "pop_size", + "datatype": "int", + "typestring": "Input", + "constraint": 50000 + }, + { + "name": "pop_infected", + "datatype": "int", + "typestring": "Input", + "constraint": 1000 + }, + { + "name": "n_days", + "datatype": "int", + "typestring": "Input", + "constraint": 50 + }, + { + "name": "vaccine", + "datatype": "int", + "typestring": "Input" + }, + { + "name": "cum_infections", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "cum_vaccinations", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "cum_vaccinated", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "max_doses", + "datatype": "int", + "typestring": "Output" + } + ] +} \ No newline at end of file diff --git a/dafni/data/outputs/causal_tests_results.json b/dafni/data/outputs/causal_tests_results.json new file mode 100644 index 00000000..f5e503aa --- /dev/null +++ b/dafni/data/outputs/causal_tests_results.json @@ -0,0 +1,215 @@ +[ + { + "name": "max_doses _||_ cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinations": "NoEffect" + }, + "formula": "cum_vaccinations ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 252628.1066666667, + "ci_low": 252271.33332001517, + "ci_high": 252984.8800133182 + } + }, + { + "name": "max_doses _||_ cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 213111.93333333335, + "ci_low": 212755.15056812647, + "ci_high": 213468.71609854023 + } + }, + { + "name": "max_doses _||_ cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 2666.3066666666664, + "ci_low": 2619.972040648758, + "ci_high": 2712.6412926845746 + } + }, + { + "name": "vaccine --> cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 315785.1333333332, + "ci_low": 315339.1666500188, + "ci_high": 316231.1000166476 + } + }, + { + "name": "vaccine --> cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 266389.91666666657, + "ci_low": 265943.93821015797, + "ci_high": 266835.89512317517 + } + }, + { + "name": "vaccine --> cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 3332.883333333332, + "ci_low": 3274.9650508109467, + "ci_high": 3390.801615855717 + } + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 0.9998656401531605, + "ci_low": 0.9929245394499968, + "ci_high": 1.0068067408563242 + } + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "failed": false, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": -0.006416682407515084, + "ci_low": -0.05663010083886572, + "ci_high": 0.043796736023835554 + } + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinated" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false, + "failed": false, + "result": { + "treatment": "cum_vaccinated", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": -0.006176900588291234, + "ci_low": -0.05639349612119588, + "ci_high": 0.04403969494461341 + } + } +] diff --git a/dafni/docker-compose.yaml b/dafni/docker-compose.yaml new file mode 100644 index 00000000..9a38d1c4 --- /dev/null +++ b/dafni/docker-compose.yaml @@ -0,0 +1,10 @@ +version: '3' +services: + causal-testing-framework: + build: + context: ../ + dockerfile: ./dafni/Dockerfile + env_file: + - .env + volumes: + - .:/usr/src/app diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py new file mode 100644 index 00000000..e6b142f3 --- /dev/null +++ b/dafni/main_dafni.py @@ -0,0 +1,222 @@ +""" + +Entrypoint script to run the causal testing framework on DAFNI + +""" + +from pathlib import Path +import argparse +import json +import pandas as pd +from causal_testing.specification.scenario import Scenario +from causal_testing.specification.variable import Input, Output +from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect, SomeEffect +from causal_testing.testing.estimators import LinearRegressionEstimator +from causal_testing.json_front.json_class import JsonUtility + + +class ValidationError(Exception): + """ + Custom class to capture validation errors in this script + """ + + +def get_args(test_args=None) -> argparse.Namespace: + """ + Function to parse arguments from the user using the CLI + :param test_args: None + :returns: + - argparse.Namespace - A Namsespace consisting of the arguments to this script + """ + parser = argparse.ArgumentParser(description="A script for running the CTF on DAFNI.") + + parser.add_argument( + "--data_path", required=True, + help="Path to the input runtime data (.csv)", nargs="+") + + parser.add_argument('--tests_path', required=True, + help='Input configuration file path ' + 'containing the causal tests (.json)') + + parser.add_argument('--variables_path', required=True, + help='Input configuration file path ' + 'containing the predefined variables (.json)') + + parser.add_argument("--dag_path", required=True, + help="Input configuration file path containing a valid DAG (.dot). " + "Note: this must be supplied if the --tests argument isn't provided.") + + parser.add_argument('--output_path', required=False, help='Path to the output directory.') + + parser.add_argument( + "-f", + default=False, + help="(Optional) Failure flag to step the framework from running if a test has failed.") + + parser.add_argument( + "-w", + default=False, + help="(Optional) Specify to overwrite any existing output files. " + "This can lead to the loss of existing outputs if not " + "careful") + + args = parser.parse_args(test_args) + + # Convert these to Path objects for main() + + args.variables_path = Path(args.variables_path) + + args.tests_path = Path(args.tests_path) + + if args.dag_path is not None: + + args.dag_path = Path(args.dag_path) + + if args.output_path is None: + + args.output_path = "./data/outputs/causal_tests_results.json" + + Path(args.output_path).parent.mkdir(exist_ok=True) + + else: + + args.output_path = Path(args.output_path) + + args.output_path.parent.mkdir(exist_ok=True) + + return args + + +def read_variables(variables_path: Path) -> FileNotFoundError | dict: + """ + Function to read the variables.json file specified by the user + :param variables_path: A Path object of the user-specified file path + :returns: + - dict - A valid dictionary consisting of the causal tests + """ + if not variables_path.exists() or variables_path.is_dir(): + + print(f"JSON file not found at the specified location: {variables_path}") + + raise FileNotFoundError + + with variables_path.open('r') as file: + + inputs = json.load(file) + + return inputs + + +def validate_variables(data_dict: dict) -> tuple: + """ + Function to validate the variables defined in the causal tests + :param data_dict: A dictionary consisting of the pre-defined variables for the causal tests + :returns: + - Tuple containing the inputs, outputs and constraints to pass into the modelling scenario + """ + if data_dict["variables"]: + + variables = data_dict["variables"] + + inputs = [Input(variable["name"], eval(variable["datatype"])) + for variable in variables if + variable["typestring"] == "Input"] + + outputs = [Output(variable["name"], eval(variable["datatype"])) + for variable in variables if + variable["typestring"] == "Output"] + + constraints = set() + + for variable, input_var in zip(variables, inputs): + + if "constraint" in variable: + + constraints.add(input_var.z3 == variable["constraint"]) + else: + + raise ValidationError("Cannot find the variables defined by the causal tests.") + + return inputs, outputs, constraints + + +def main(): + """ + Main entrypoint of the script: + """ + args = get_args() + + try: + + # Step 0: Read in the runtime dataset(s) + + data_frame = pd.concat([pd.read_csv(d) for d in args.data_path]) + + # Step 1: Read in the JSON input/output variables and parse io arguments + + variables_dict = read_variables(args.variables_path) + + inputs, outputs, constraints = validate_variables(variables_dict) + + # Step 2: Set up the modeling scenario and estimator + + modelling_scenario = Scenario(variables=inputs + outputs, constraints=constraints) + + modelling_scenario.setup_treatment_variables() + + estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} + + # Step 3: Define the expected variables + + expected_outcome_effects = { + "Positive": Positive(), + "Negative": Negative(), + "NoEffect": NoEffect(), + "SomeEffect": SomeEffect()} + + # Step 4: Call the JSONUtility class to perform the causal tests + + json_utility = JsonUtility(args.output_path, output_overwrite=True) + + # Step 5: Set the path to the data.csv, dag.dot and causal_tests.json file + json_utility.set_paths(args.tests_path, args.dag_path, args.data_path) + + # Step 6: Sets up all the necessary parts of the json_class needed to execute tests + json_utility.setup(scenario=modelling_scenario, data=data_frame) + + # Step 7: Run the causal tests + test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, + mutates={}, estimators=estimators, + f_flag=args.f) + + # Step 8: Update, print and save the final outputs + + for test in test_outcomes: + + test.pop("estimator") + + test["result"] = test["result"].to_dict(json=True) + + test["result"].pop("treatment_value") + + test["result"].pop("control_value") + + + with open(args.output_path, "w", encoding="utf-8") as f: + + print(json.dumps(test_outcomes, indent=2), file=f) + + print(json.dumps(test_outcomes, indent=2)) + + except ValidationError as ve: + + print(f"Cannot validate the specified input configurations: {ve}") + + else: + + print(f"Execution successful. " + f"Output file saved at {Path(args.output_path).parent.resolve()}") + + +if __name__ == "__main__": + main() diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml new file mode 100644 index 00000000..901e2570 --- /dev/null +++ b/dafni/model_definition.yaml @@ -0,0 +1,65 @@ +# Model definition file to run the causal testing framework on DAFNI +# https://docs.secure.dafni.rl.ac.uk/docs/how-to/how-to-write-a-model-definition-file + +kind: M +api_version: v1beta3 +metadata: + display_name: Causal Testing Framework + name: causal-testing-framework + publisher: The CITCOM Team, The University of Sheffield + type: model + summary: A Causal Inference-Driven Software Testing Framework + description: > + Causal Testing is a causal inference-driven framework for functional black-box testing. + This framework utilises graphical causal inference (CI) techniques for the specification and functional testing of + software from a black-box perspective. In this framework, we use causal directed acyclic graphs (DAGs) to express + the anticipated cause-effect relationships amongst the inputs and outputs of the system-under-test and the + supporting mathematical framework to design statistical procedures capable of making causal inferences. + Each causal test case focuses on the causal effect of an intervention made to the system-under test. + contact_point_name: Farhad Allian + contact_point_email: farhad.allian@sheffield.ac.uk + source_code: https://github.com/CITCOM-project/CausalTestingFramework + licence: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme + + +spec: + inputs: + dataslots: + - name: Runtime csv data + description: > + A .csv file containing the input runtime data to be used + default: + - 2b7336cd-eb68-4c1f-8f91-26d8969b8cb3 + path: inputs/ + required: true + + - name: DAG data + description: > + A .dot file containing the input DAG to be used + default: + - 74665fdb-43a2-4c51-b81e-d5299b38bf8c + path: inputs/ + required: true + + - name: Causal tests + description: > + A .JSON file containing the input causal tests to be used + default: + - 6f2f7c1f-81b4-4804-8f86-cca304dc7f66 + path: inputs/ + required: true + + - name: Variables + description: > + A .JSON file containing the input variables to be used + default: + - 02e755c8-952b-461a-a914-4f4ffbe2edf1 + path: inputs/ + required: true + + outputs: + datasets: + - name: causal_test_results.json + type: json + description: > + A JSON file containing the output causal test results. \ No newline at end of file