From 1670195a4802ba89ce3a760834d11c3669b53568 Mon Sep 17 00:00:00 2001 From: f-allian Date: Tue, 12 Dec 2023 16:21:49 +0000 Subject: [PATCH 1/3] Add: initial commit for the dockerised ctf --- dafni/.dockerignore | 5 + dafni/Dockerfile | 32 +++++ dafni/docker-compose.yaml | 12 ++ dafni/inputs/causal_tests.json | 136 +++++++++++++++++++++ dafni/inputs/dag.dot | 7 ++ dafni/inputs/simulated_data.csv | 31 +++++ dafni/inputs/variables.json | 47 +++++++ dafni/main_dafni.py | 210 ++++++++++++++++++++++++++++++++ dafni/model_definition.yaml | 58 +++++++++ 9 files changed, 538 insertions(+) create mode 100644 dafni/.dockerignore create mode 100644 dafni/Dockerfile create mode 100644 dafni/docker-compose.yaml create mode 100644 dafni/inputs/causal_tests.json create mode 100644 dafni/inputs/dag.dot create mode 100644 dafni/inputs/simulated_data.csv create mode 100644 dafni/inputs/variables.json create mode 100644 dafni/main_dafni.py create mode 100644 dafni/model_definition.yaml diff --git a/dafni/.dockerignore b/dafni/.dockerignore new file mode 100644 index 00000000..0af95714 --- /dev/null +++ b/dafni/.dockerignore @@ -0,0 +1,5 @@ +../* +../!causal_testing +../!LICENSE +./!inputs +./!main_dafni.py \ No newline at end of file diff --git a/dafni/Dockerfile b/dafni/Dockerfile new file mode 100644 index 00000000..0d83569a --- /dev/null +++ b/dafni/Dockerfile @@ -0,0 +1,32 @@ +# Define the Python version neded for CTF +FROM python:3.10-slim + +## Prevents Python from writing pyc files +ENV PYTHONDONTWRITEBYTECODE=1 +# +## Keeps Python from buffering stdout and stderr to avoid the framework +## from crashing without emitting any logs due to buffering +ENV PYTHONUNBUFFERED=1 + +#Label maintainer +LABEL maintainer="Dr. Farhad Allian - The University of Sheffield" + +# Copy the source code and test files from build into the container +COPY --chown=nobody ./causal_testing /usr/src/app/ +COPY --chown=nobody ./dafni/inputs /usr/src/app/inputs/ +COPY --chown=nobody ./dafni/main_dafni.py /usr/src/app/ + +# Change the working directory +WORKDIR /usr/src/app/ + +# Install core dependencies using PyPi +RUN pip install causal-testing-framework --no-cache-dir + +# Use the necessaary environment variables for the script's inputs +ENV VARIABLES=./inputs/variables.json \ + CAUSAL_TESTS=./inputs/causal_tests.json \ + DATA_PATH=./inputs/simulated_data.csv \ + DAG_PATH=./inputs/dag.dot + +# Define the entrypoint/commands +CMD python main_dafni.py --variables_path $VARIABLES_PATH --dag_path $DAG_PATH --data_path $DATA_PATH --tests_path $CAUSAL_TESTS diff --git a/dafni/docker-compose.yaml b/dafni/docker-compose.yaml new file mode 100644 index 00000000..a1228bad --- /dev/null +++ b/dafni/docker-compose.yaml @@ -0,0 +1,12 @@ +version: '3' +services: + causal-testing-framework: + build: + context: ../ + dockerfile: ./dafni/Dockerfile + env_file: + - .env + volumes: + - .:/usr/src/app + - ./inputs:/usr/src/app/inputs/ + - ./outputs:/usr/src/app/outputs/ \ No newline at end of file diff --git a/dafni/inputs/causal_tests.json b/dafni/inputs/causal_tests.json new file mode 100644 index 00000000..6c78c57f --- /dev/null +++ b/dafni/inputs/causal_tests.json @@ -0,0 +1,136 @@ +{ + "tests": [ + { + "name": "max_doses _||_ cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinations": "NoEffect" + }, + "formula": "cum_vaccinations ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "max_doses _||_ cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "max_doses _||_ cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ max_doses", + "alpha": 0.05, + "skip": false + }, + { + "name": "vaccine --> cum_vaccinations", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_vaccinated", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false + }, + { + "name": "vaccine --> cum_infections", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinated" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false + } + ] +} \ No newline at end of file diff --git a/dafni/inputs/dag.dot b/dafni/inputs/dag.dot new file mode 100644 index 00000000..43628817 --- /dev/null +++ b/dafni/inputs/dag.dot @@ -0,0 +1,7 @@ +digraph CausalDAG { + rankdir=LR; + "vaccine" -> "cum_vaccinations"; + "vaccine" -> "cum_vaccinated"; + "vaccine" -> "cum_infections"; + "max_doses"; +} \ No newline at end of file diff --git a/dafni/inputs/simulated_data.csv b/dafni/inputs/simulated_data.csv new file mode 100644 index 00000000..3ac0a3ac --- /dev/null +++ b/dafni/inputs/simulated_data.csv @@ -0,0 +1,31 @@ +pop_size,pop_type,pop_infected,n_days,vaccine_type,use_waning,rand_seed,cum_infections,cum_deaths,cum_recoveries,cum_vaccinations,cum_vaccinated,target_elderly,vaccine,max_doses +50000,hybrid,1000,50,pfizer,True,1169,6277.0,15.0,6175.0,629466.0,530715.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,8888,6381.0,18.0,6274.0,630796.0,532010.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,370,6738.0,15.0,6621.0,631705.0,532864.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,9981,6784.0,18.0,6682.0,634582.0,535795.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6305,6757.0,20.0,6659.0,631292.0,532464.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1993,5844.0,17.0,5755.0,633314.0,534478.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1938,6465.0,19.0,6353.0,627724.0,528993.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4797,7044.0,15.0,6919.0,631246.0,532433.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2308,6878.0,6.0,6801.0,628865.0,530038.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4420,6429.0,11.0,6348.0,633803.0,535030.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2314,6566.0,15.0,6477.0,629288.0,530550.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7813,6913.0,17.0,6818.0,629290.0,530512.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1050,6963.0,14.0,6860.0,627981.0,529212.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3215,6671.0,17.0,6577.0,628802.0,530038.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,2286,6597.0,13.0,6505.0,628986.0,530195.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3080,6926.0,16.0,6834.0,633636.0,534904.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7405,6438.0,15.0,6347.0,630353.0,531540.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,9668,6577.0,15.0,6485.0,631257.0,532409.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,8211,6197.0,13.0,6103.0,633827.0,535056.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4686,6761.0,16.0,6653.0,630557.0,531737.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3591,7328.0,24.0,7214.0,629949.0,531124.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,4834,6617.0,22.0,6512.0,632609.0,533705.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6142,7017.0,17.0,6902.0,635965.0,537252.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,6877,6845.0,15.0,6753.0,635678.0,536925.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1878,6480.0,20.0,6390.0,630807.0,531999.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,3761,6972.0,16.0,6890.0,631100.0,532329.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,1741,6581.0,20.0,6491.0,632835.0,534088.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,5592,6561.0,19.0,6461.0,636799.0,537959.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,7979,7075.0,17.0,6966.0,632902.0,534140.0,True,1,2 +50000,hybrid,1000,50,pfizer,True,71,6291.0,13.0,6203.0,631694.0,532901.0,True,1,2 diff --git a/dafni/inputs/variables.json b/dafni/inputs/variables.json new file mode 100644 index 00000000..8e4a1add --- /dev/null +++ b/dafni/inputs/variables.json @@ -0,0 +1,47 @@ +{ + "variables": [ + { + "name": "pop_size", + "datatype": "int", + "typestring": "Input", + "constraint": 50000 + }, + { + "name": "pop_infected", + "datatype": "int", + "typestring": "Input", + "constraint": 1000 + }, + { + "name": "n_days", + "datatype": "int", + "typestring": "Input", + "constraint": 50 + }, + { + "name": "vaccine", + "datatype": "int", + "typestring": "Input" + }, + { + "name": "cum_infections", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "cum_vaccinations", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "cum_vaccinated", + "datatype": "int", + "typestring": "Output" + }, + { + "name": "max_doses", + "datatype": "int", + "typestring": "Output" + } + ] +} \ No newline at end of file diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py new file mode 100644 index 00000000..5ab40c8d --- /dev/null +++ b/dafni/main_dafni.py @@ -0,0 +1,210 @@ +import warnings +warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*") +import os +from pathlib import Path +import argparse +import json +import pandas as pd +from causal_testing.specification.scenario import Scenario +from causal_testing.specification.variable import Input, Output +from causal_testing.testing.causal_test_outcome import Positive, Negative, NoEffect, SomeEffect +from causal_testing.testing.estimators import LinearRegressionEstimator +from causal_testing.json_front.json_class import JsonUtility + + +class ValidationError(Exception): + """ + Custom class to capture validation errors in this script + """ + pass + + +def get_args(test_args=None) -> argparse.Namespace: + """ + Function to parse arguments from the user using the CLI + :param test_args: None + :returns: + - argparse.Namespace - A Namsespace consisting of the arguments to this script + """ + parser = argparse.ArgumentParser(description="A script for running the causal testing famework on DAFNI.") + + parser.add_argument( + "--data_path", required=True, + help="Path to the input runtime data (.csv)", nargs="+") + + parser.add_argument('--tests_path', required=True, + help='Path to the input configuration file containing the causal tests (.json)') + + parser.add_argument('--variables_path', required=True, + help='Path to the input configuration file containing the predefined variables (.json)') + + parser.add_argument("--dag_path", required=True, + help="Path to the input file containing a valid DAG (.dot). " + "Note: this must be supplied if the --tests argument isn't provided.") + + parser.add_argument('--output_path', required=False, help='Path to the output directory.') + + parser.add_argument( + "-f", + default=False, + help="(Optional) Failure flag to step the framework from running if a test has failed.") + + parser.add_argument( + "-w", + default=False, + help="(Optional) Specify to overwrite any existing output files. " + "This can lead to the loss of existing outputs if not " + "careful") + + args = parser.parse_args(test_args) + + # Convert these to Path objects for main() + + args.variables_path = Path(args.variables_path) + + args.tests_path = Path(args.tests_path) + + if args.dag_path is not None: + + args.dag_path = Path(args.dag_path) + + if args.output_path is None: + + args.output_path = "./outputs/results/"+"_".join([os.path.splitext(os.path.basename(x))[0] + for x in args.data_path]) + "_results.json" + + Path(args.output_path).parent.mkdir(exist_ok=True, parents=True) + + else: + + args.output_path = Path(args.output_path) + + args.output_path.parent.mkdir(exist_ok=True, parents=True) + + return args + + +def read_variables(variables_path: Path) -> dict: + """ + Function to read the variables.json file specified by the user + :param variables_path: A Path object of the user-specified file path + :returns: + - dict - A valid dictionary consisting of the causal tests + """ + if not variables_path.exists() or variables_path.is_dir(): + + raise ValidationError(f"Cannot find a valid settings file at {variables_path.absolute()}.") + + else: + + with variables_path.open('r') as file: + + inputs = json.load(file) + + return inputs + + +def validate_variables(data_dict: dict) -> tuple: + """ + Function to validate the variables defined in the causal tests + :param data_dict: A dictionary consisting of the pre-defined variables for the causal tests + :returns: + - tuple - Tuple consisting of the inputs, outputs and constraints to pass into the modelling scenario + """ + if data_dict["variables"]: + + variables = data_dict["variables"] + + inputs = [Input(variable["name"], eval(variable["datatype"])) for variable in variables if + variable["typestring"] == "Input"] + + outputs = [Output(variable["name"], eval(variable["datatype"])) for variable in variables if + variable["typestring"] == "Output"] + + constraints = set() + + for variable, _inputs in zip(variables, inputs): + + if "constraint" in variable: + constraints.add(_inputs.z3 == variable["constraint"]) + + return inputs, outputs, constraints + + +def main(): + """ + Main entrypoint of the script: + """ + args = get_args() + + # Step 0: Read in the runtime dataset(s) + + try: + + data_frame = pd.concat([pd.read_csv(d) for d in args.data_path]) + + # Step 1: Read in the JSON input/output variables and parse io arguments + + variables_dict = read_variables(args.variables_path) + + inputs, outputs, constraints = validate_variables(variables_dict) + + # Step 2: Set up the modeling scenario and estimator + + modelling_scenario = Scenario(variables=inputs + outputs, constraints=constraints) + + modelling_scenario.setup_treatment_variables() + + estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} + + # Step 3: Define the expected variables + + expected_outcome_effects = { + "Positive": Positive(), + "Negative": Negative(), + "NoEffect": NoEffect(), + "SomeEffect": SomeEffect()} + + # Step 4: Call the JSONUtility class to perform the causal tests + + json_utility = JsonUtility(args.output_path, output_overwrite=True) + + # Step 5: Set the path to the data.csv, dag.dot and causal_tests.json file + json_utility.set_paths(args.tests_path, args.dag_path, args.data_path) + + # Step 6: Sets up all the necessary parts of the json_class needed to execute tests + json_utility.setup(scenario=modelling_scenario, data=data_frame) + + # Step 7: Run the causal tests + test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, mutates={}, estimators=estimators, + f_flag=args.f) + + # Step 8: Update, print and save the final outputs + + for test in test_outcomes: + + test.pop("estimator") + + test["result"] = test["result"].to_dict(json=True) + + test["result"].pop("treatment_value") + + test["result"].pop("control_value") + + with open(f"{args.output_path}", "w") as f: + + print(json.dumps(test_outcomes, indent=2), file=f) + + print(json.dumps(test_outcomes, indent=2)) + + except ValidationError as ve: + + print(f"Cannot validate the specified input configurations: {ve}") + + else: + + print(f"Execution successful. Output file saved at {Path(args.output_path).parent.resolve()}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml new file mode 100644 index 00000000..a5e6d602 --- /dev/null +++ b/dafni/model_definition.yaml @@ -0,0 +1,58 @@ +# Model definition file to run the causal testing framework on DAFNI +# https://docs.secure.dafni.rl.ac.uk/docs/how-to/how-to-write-a-model-definition-file + +kind: M +api_version: v1beta3 +metadata: + display_name: Causal Testing Framework + name: causal_testing + publisher: The CITCOM Team, The University of Sheffield + type: model + summary: A Causal Inference-Driven Software Testing Framework + description: > + Causal Testing is a causal inference-driven framework for functional black-box testing. + This framework utilises graphical causal inference (CI) techniques for the specification and functional testing of + software from a black-box perspective. In this framework, we use causal directed acyclic graphs (DAGs) to express + the anticipated cause-effect relationships amongst the inputs and outputs of the system-under-test and the + supporting mathematical framework to design statistical procedures capable of making causal inferences. + Each causal test case focuses on the causal effect of an intervention made to the system-under test. + contact_point_name: Dr. Farhad Allian + contact_point_email: farhad.allian@sheffield.ac.uk + source_code: https://github.com/CITCOM-project/CausalTestingFramework + license: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme + + +spec: + inputs: + parameters: + - name: causal_tests + title: Causal tests filename + description: A .json file containing the causal tests to be used + type: string + required: true + - name: variables + title: Variables filename + description: A .json file containing the input and output variables to be used + type: string + required: true + - name: dag_file + title: DAG filename + description: A .dot file containing the input DAG to be used + type: string + required: true + + dataslots: + - name: runtime_data + description: > + A .csv file containing the input runtime data to be used + default: + - #TODO + path: /inputs/dataslots + required: false + - name: dag_file + description: > + A .dot file containing the input DAG to be used + default: + - #TODO + path: /data/tests/inputs + required: false \ No newline at end of file From bbafb24b6b956d28ff971991170f2f6fdd923588 Mon Sep 17 00:00:00 2001 From: f-allian Date: Wed, 17 Jan 2024 11:51:45 +0000 Subject: [PATCH 2/3] Fix: structure and files uploaded to dafni. --- dafni/.dockerignore | 5 +- dafni/.env | 5 + dafni/Dockerfile | 23 +- dafni/README.md | 29 +++ dafni/{ => data}/inputs/causal_tests.json | 0 dafni/{ => data}/inputs/dag.dot | 0 .../inputs/runtime_data.csv} | 0 dafni/{ => data}/inputs/variables.json | 0 dafni/data/outputs/causal_tests_results.json | 215 ++++++++++++++++++ dafni/docker-compose.yaml | 2 - dafni/main_dafni.py | 27 ++- dafni/model_definition.yaml | 65 +++--- 12 files changed, 315 insertions(+), 56 deletions(-) create mode 100644 dafni/.env create mode 100644 dafni/README.md rename dafni/{ => data}/inputs/causal_tests.json (100%) rename dafni/{ => data}/inputs/dag.dot (100%) rename dafni/{inputs/simulated_data.csv => data/inputs/runtime_data.csv} (100%) rename dafni/{ => data}/inputs/variables.json (100%) create mode 100644 dafni/data/outputs/causal_tests_results.json diff --git a/dafni/.dockerignore b/dafni/.dockerignore index 0af95714..52a1d65d 100644 --- a/dafni/.dockerignore +++ b/dafni/.dockerignore @@ -1,5 +1,4 @@ ../* ../!causal_testing -../!LICENSE -./!inputs -./!main_dafni.py \ No newline at end of file +./!main_dafni.py +./!data/ \ No newline at end of file diff --git a/dafni/.env b/dafni/.env new file mode 100644 index 00000000..3143724b --- /dev/null +++ b/dafni/.env @@ -0,0 +1,5 @@ +#.env +VARIABLES_PATH=./data/inputs/variables.json +CAUSAL_TESTS=./data/inputs/causal_tests.json +DATA_PATH=./data/inputs/runtime_data.csv +DAG_PATH=./data/inputs/dag.dot \ No newline at end of file diff --git a/dafni/Dockerfile b/dafni/Dockerfile index 0d83569a..8969ad3f 100644 --- a/dafni/Dockerfile +++ b/dafni/Dockerfile @@ -11,22 +11,23 @@ ENV PYTHONUNBUFFERED=1 #Label maintainer LABEL maintainer="Dr. Farhad Allian - The University of Sheffield" -# Copy the source code and test files from build into the container -COPY --chown=nobody ./causal_testing /usr/src/app/ -COPY --chown=nobody ./dafni/inputs /usr/src/app/inputs/ -COPY --chown=nobody ./dafni/main_dafni.py /usr/src/app/ +# Create a folder for the source code/outputs +RUN mkdir -p ./causal_testing +RUN mkdir -p ./data/outputs -# Change the working directory -WORKDIR /usr/src/app/ +# Copy the source code and test files from build into the container +COPY --chown=nobody ../causal_testing ./causal_testing +COPY --chown=nobody ./dafni/main_dafni.py ./ +COPY --chown=nobody ./dafni/data/inputs ./data/inputs # Install core dependencies using PyPi RUN pip install causal-testing-framework --no-cache-dir -# Use the necessaary environment variables for the script's inputs -ENV VARIABLES=./inputs/variables.json \ - CAUSAL_TESTS=./inputs/causal_tests.json \ - DATA_PATH=./inputs/simulated_data.csv \ - DAG_PATH=./inputs/dag.dot +#For local testing purposes +ENV VARIABLES_PATH=./data/inputs/variables.json \ + CAUSAL_TESTS=./data/inputs/causal_tests.json \ + DATA_PATH=./data/inputs/runtime_data.csv \ + DAG_PATH=./data/inputs/dag.dot # Define the entrypoint/commands CMD python main_dafni.py --variables_path $VARIABLES_PATH --dag_path $DAG_PATH --data_path $DATA_PATH --tests_path $CAUSAL_TESTS diff --git a/dafni/README.md b/dafni/README.md new file mode 100644 index 00000000..29833d61 --- /dev/null +++ b/dafni/README.md @@ -0,0 +1,29 @@ +# Causal Testing Framework on DAFNI + +- This directory contains the containerisation files of the causal testing framework using Docker, which is used +to upload the framework onto [DAFNI](https://www.dafni.ac.uk). +- It is **not** recommended to install the causal testing framework using Docker, and should only be installed + using [PyPI](https://pypi.org/project/causal-testing-framework/). + +### Folders + +- `data` contains two sub-folders (the structure is important for DAFNI). + - `inputs` is a folder that contains the input files that are (separately) uploaded to DAFNI. + - `causal_tests.json` is a JSON file that contains the causal tests. + - `variables.json` is a JSON file that contains the variables and constraints to be used. + - `dag.dot` is a dot file that contains the directed acyclc graph (dag) file. + - `runtime_data.csv` is a csv file that contains the runtime data. + + - `outputs` is a folder where the `causal_tests_results.json` output file is created. + +### Docker files +- `main_dafni.py` is the entry-point to the causal testing framework that is used by Docker. +- `model_definition.yaml` is the model metadata that is required to be uploaded to DAFNI. +- `.env` is an example of a configuration file containing the environment variables. This is only required + if using `docker-compose` to build the image. +- `Dockerfile` is the main blueprint that builds the image. +- `.dockerignore` tells the Dockerfile which files to not include in the image. +- `docker-compose.yaml` is another method of building the image and running the container in one line. + Note: the `.env` file that contains the environment variables for `main_dafni.py` is only used here. + + diff --git a/dafni/inputs/causal_tests.json b/dafni/data/inputs/causal_tests.json similarity index 100% rename from dafni/inputs/causal_tests.json rename to dafni/data/inputs/causal_tests.json diff --git a/dafni/inputs/dag.dot b/dafni/data/inputs/dag.dot similarity index 100% rename from dafni/inputs/dag.dot rename to dafni/data/inputs/dag.dot diff --git a/dafni/inputs/simulated_data.csv b/dafni/data/inputs/runtime_data.csv similarity index 100% rename from dafni/inputs/simulated_data.csv rename to dafni/data/inputs/runtime_data.csv diff --git a/dafni/inputs/variables.json b/dafni/data/inputs/variables.json similarity index 100% rename from dafni/inputs/variables.json rename to dafni/data/inputs/variables.json diff --git a/dafni/data/outputs/causal_tests_results.json b/dafni/data/outputs/causal_tests_results.json new file mode 100644 index 00000000..f5e503aa --- /dev/null +++ b/dafni/data/outputs/causal_tests_results.json @@ -0,0 +1,215 @@ +[ + { + "name": "max_doses _||_ cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinations": "NoEffect" + }, + "formula": "cum_vaccinations ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 252628.1066666667, + "ci_low": 252271.33332001517, + "ci_high": 252984.8800133182 + } + }, + { + "name": "max_doses _||_ cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 213111.93333333335, + "ci_low": 212755.15056812647, + "ci_high": 213468.71609854023 + } + }, + { + "name": "max_doses _||_ cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "max_doses" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ max_doses", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "max_doses", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 2666.3066666666664, + "ci_low": 2619.972040648758, + "ci_high": 2712.6412926845746 + } + }, + { + "name": "vaccine --> cum_vaccinations", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinations": "SomeEffect" + }, + "formula": "cum_vaccinations ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinations", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 315785.1333333332, + "ci_low": 315339.1666500188, + "ci_high": 316231.1000166476 + } + }, + { + "name": "vaccine --> cum_vaccinated", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_vaccinated": "SomeEffect" + }, + "formula": "cum_vaccinated ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 266389.91666666657, + "ci_low": 265943.93821015797, + "ci_high": 266835.89512317517 + } + }, + { + "name": "vaccine --> cum_infections", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "vaccine" + ], + "expected_effect": { + "cum_infections": "SomeEffect" + }, + "formula": "cum_infections ~ vaccine", + "skip": false, + "failed": false, + "result": { + "treatment": "vaccine", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 3332.883333333332, + "ci_low": 3274.9650508109467, + "ci_high": 3390.801615855717 + } + }, + { + "name": "cum_vaccinations _||_ cum_vaccinated | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_vaccinated": "NoEffect" + }, + "formula": "cum_vaccinated ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "failed": true, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_vaccinated", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": 0.9998656401531605, + "ci_low": 0.9929245394499968, + "ci_high": 1.0068067408563242 + } + }, + { + "name": "cum_vaccinations _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinations" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinations + vaccine", + "alpha": 0.05, + "skip": false, + "failed": false, + "result": { + "treatment": "cum_vaccinations", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": -0.006416682407515084, + "ci_low": -0.05663010083886572, + "ci_high": 0.043796736023835554 + } + }, + { + "name": "cum_vaccinated _||_ cum_infections | ['vaccine']", + "estimate_type": "coefficient", + "effect": "direct", + "mutations": [ + "cum_vaccinated" + ], + "expected_effect": { + "cum_infections": "NoEffect" + }, + "formula": "cum_infections ~ cum_vaccinated + vaccine", + "alpha": 0.05, + "skip": false, + "failed": false, + "result": { + "treatment": "cum_vaccinated", + "outcome": "cum_infections", + "adjustment_set": [], + "effect_measure": "coefficient", + "effect_estimate": -0.006176900588291234, + "ci_low": -0.05639349612119588, + "ci_high": 0.04403969494461341 + } + } +] diff --git a/dafni/docker-compose.yaml b/dafni/docker-compose.yaml index a1228bad..9a38d1c4 100644 --- a/dafni/docker-compose.yaml +++ b/dafni/docker-compose.yaml @@ -8,5 +8,3 @@ services: - .env volumes: - .:/usr/src/app - - ./inputs:/usr/src/app/inputs/ - - ./outputs:/usr/src/app/outputs/ \ No newline at end of file diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py index 5ab40c8d..c7983ca3 100644 --- a/dafni/main_dafni.py +++ b/dafni/main_dafni.py @@ -1,5 +1,3 @@ -import warnings -warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*") import os from pathlib import Path import argparse @@ -70,16 +68,15 @@ def get_args(test_args=None) -> argparse.Namespace: if args.output_path is None: - args.output_path = "./outputs/results/"+"_".join([os.path.splitext(os.path.basename(x))[0] - for x in args.data_path]) + "_results.json" + args.output_path = "./data/outputs/causal_tests_results.json" - Path(args.output_path).parent.mkdir(exist_ok=True, parents=True) + Path(args.output_path).parent.mkdir(exist_ok=True) else: args.output_path = Path(args.output_path) - args.output_path.parent.mkdir(exist_ok=True, parents=True) + args.output_path.parent.mkdir(exist_ok=True) return args @@ -93,7 +90,9 @@ def read_variables(variables_path: Path) -> dict: """ if not variables_path.exists() or variables_path.is_dir(): - raise ValidationError(f"Cannot find a valid settings file at {variables_path.absolute()}.") + raise FileNotFoundError + + print(f"JSON file not found at the specified location: {variables_path}") else: @@ -126,8 +125,13 @@ def validate_variables(data_dict: dict) -> tuple: for variable, _inputs in zip(variables, inputs): if "constraint" in variable: + constraints.add(_inputs.z3 == variable["constraint"]) + else: + + raise ValidationError("Cannot find the variables defined by the causal tests.") + return inputs, outputs, constraints @@ -137,10 +141,10 @@ def main(): """ args = get_args() - # Step 0: Read in the runtime dataset(s) - try: + # Step 0: Read in the runtime dataset(s) + data_frame = pd.concat([pd.read_csv(d) for d in args.data_path]) # Step 1: Read in the JSON input/output variables and parse io arguments @@ -191,7 +195,8 @@ def main(): test["result"].pop("control_value") - with open(f"{args.output_path}", "w") as f: + + with open(args.output_path, "w") as f: print(json.dumps(test_outcomes, indent=2), file=f) @@ -207,4 +212,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/dafni/model_definition.yaml b/dafni/model_definition.yaml index a5e6d602..901e2570 100644 --- a/dafni/model_definition.yaml +++ b/dafni/model_definition.yaml @@ -5,7 +5,7 @@ kind: M api_version: v1beta3 metadata: display_name: Causal Testing Framework - name: causal_testing + name: causal-testing-framework publisher: The CITCOM Team, The University of Sheffield type: model summary: A Causal Inference-Driven Software Testing Framework @@ -16,43 +16,50 @@ metadata: the anticipated cause-effect relationships amongst the inputs and outputs of the system-under-test and the supporting mathematical framework to design statistical procedures capable of making causal inferences. Each causal test case focuses on the causal effect of an intervention made to the system-under test. - contact_point_name: Dr. Farhad Allian + contact_point_name: Farhad Allian contact_point_email: farhad.allian@sheffield.ac.uk source_code: https://github.com/CITCOM-project/CausalTestingFramework - license: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme - + licence: https://github.com/CITCOM-project/CausalTestingFramework?tab=MIT-1-ov-file#readme + spec: inputs: - parameters: - - name: causal_tests - title: Causal tests filename - description: A .json file containing the causal tests to be used - type: string - required: true - - name: variables - title: Variables filename - description: A .json file containing the input and output variables to be used - type: string - required: true - - name: dag_file - title: DAG filename - description: A .dot file containing the input DAG to be used - type: string - required: true - dataslots: - - name: runtime_data + - name: Runtime csv data description: > A .csv file containing the input runtime data to be used default: - - #TODO - path: /inputs/dataslots - required: false - - name: dag_file + - 2b7336cd-eb68-4c1f-8f91-26d8969b8cb3 + path: inputs/ + required: true + + - name: DAG data description: > A .dot file containing the input DAG to be used default: - - #TODO - path: /data/tests/inputs - required: false \ No newline at end of file + - 74665fdb-43a2-4c51-b81e-d5299b38bf8c + path: inputs/ + required: true + + - name: Causal tests + description: > + A .JSON file containing the input causal tests to be used + default: + - 6f2f7c1f-81b4-4804-8f86-cca304dc7f66 + path: inputs/ + required: true + + - name: Variables + description: > + A .JSON file containing the input variables to be used + default: + - 02e755c8-952b-461a-a914-4f4ffbe2edf1 + path: inputs/ + required: true + + outputs: + datasets: + - name: causal_test_results.json + type: json + description: > + A JSON file containing the output causal test results. \ No newline at end of file From 46216aa4711324a23a1b77b3243819a49a1c4e92 Mon Sep 17 00:00:00 2001 From: f-allian Date: Tue, 30 Jan 2024 14:52:18 +0000 Subject: [PATCH 3/3] Fix: final linting --- dafni/main_dafni.py | 51 ++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/dafni/main_dafni.py b/dafni/main_dafni.py index c7983ca3..e6b142f3 100644 --- a/dafni/main_dafni.py +++ b/dafni/main_dafni.py @@ -1,4 +1,9 @@ -import os +""" + +Entrypoint script to run the causal testing framework on DAFNI + +""" + from pathlib import Path import argparse import json @@ -14,7 +19,6 @@ class ValidationError(Exception): """ Custom class to capture validation errors in this script """ - pass def get_args(test_args=None) -> argparse.Namespace: @@ -24,20 +28,22 @@ def get_args(test_args=None) -> argparse.Namespace: :returns: - argparse.Namespace - A Namsespace consisting of the arguments to this script """ - parser = argparse.ArgumentParser(description="A script for running the causal testing famework on DAFNI.") + parser = argparse.ArgumentParser(description="A script for running the CTF on DAFNI.") parser.add_argument( "--data_path", required=True, help="Path to the input runtime data (.csv)", nargs="+") parser.add_argument('--tests_path', required=True, - help='Path to the input configuration file containing the causal tests (.json)') + help='Input configuration file path ' + 'containing the causal tests (.json)') parser.add_argument('--variables_path', required=True, - help='Path to the input configuration file containing the predefined variables (.json)') + help='Input configuration file path ' + 'containing the predefined variables (.json)') parser.add_argument("--dag_path", required=True, - help="Path to the input file containing a valid DAG (.dot). " + help="Input configuration file path containing a valid DAG (.dot). " "Note: this must be supplied if the --tests argument isn't provided.") parser.add_argument('--output_path', required=False, help='Path to the output directory.') @@ -81,7 +87,7 @@ def get_args(test_args=None) -> argparse.Namespace: return args -def read_variables(variables_path: Path) -> dict: +def read_variables(variables_path: Path) -> FileNotFoundError | dict: """ Function to read the variables.json file specified by the user :param variables_path: A Path object of the user-specified file path @@ -90,17 +96,15 @@ def read_variables(variables_path: Path) -> dict: """ if not variables_path.exists() or variables_path.is_dir(): - raise FileNotFoundError - print(f"JSON file not found at the specified location: {variables_path}") - else: + raise FileNotFoundError - with variables_path.open('r') as file: + with variables_path.open('r') as file: - inputs = json.load(file) + inputs = json.load(file) - return inputs + return inputs def validate_variables(data_dict: dict) -> tuple: @@ -108,26 +112,27 @@ def validate_variables(data_dict: dict) -> tuple: Function to validate the variables defined in the causal tests :param data_dict: A dictionary consisting of the pre-defined variables for the causal tests :returns: - - tuple - Tuple consisting of the inputs, outputs and constraints to pass into the modelling scenario + - Tuple containing the inputs, outputs and constraints to pass into the modelling scenario """ if data_dict["variables"]: variables = data_dict["variables"] - inputs = [Input(variable["name"], eval(variable["datatype"])) for variable in variables if + inputs = [Input(variable["name"], eval(variable["datatype"])) + for variable in variables if variable["typestring"] == "Input"] - outputs = [Output(variable["name"], eval(variable["datatype"])) for variable in variables if + outputs = [Output(variable["name"], eval(variable["datatype"])) + for variable in variables if variable["typestring"] == "Output"] constraints = set() - for variable, _inputs in zip(variables, inputs): + for variable, input_var in zip(variables, inputs): if "constraint" in variable: - constraints.add(_inputs.z3 == variable["constraint"]) - + constraints.add(input_var.z3 == variable["constraint"]) else: raise ValidationError("Cannot find the variables defined by the causal tests.") @@ -180,7 +185,8 @@ def main(): json_utility.setup(scenario=modelling_scenario, data=data_frame) # Step 7: Run the causal tests - test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, mutates={}, estimators=estimators, + test_outcomes = json_utility.run_json_tests(effects=expected_outcome_effects, + mutates={}, estimators=estimators, f_flag=args.f) # Step 8: Update, print and save the final outputs @@ -196,7 +202,7 @@ def main(): test["result"].pop("control_value") - with open(args.output_path, "w") as f: + with open(args.output_path, "w", encoding="utf-8") as f: print(json.dumps(test_outcomes, indent=2), file=f) @@ -208,7 +214,8 @@ def main(): else: - print(f"Execution successful. Output file saved at {Path(args.output_path).parent.resolve()}") + print(f"Execution successful. " + f"Output file saved at {Path(args.output_path).parent.resolve()}") if __name__ == "__main__":