From 20a8d14de88cbeb5f2bc98894511f54e84e7bec9 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Tue, 27 Aug 2024 04:18:43 -0400
Subject: [PATCH 01/17] feat: setup devcontainer + implement basic performance
 testing

This commit adds a basic devcontainer configuration and implements a simple naive framework for performance testing.
---
 .devcontainer/devcontainer.json |  30 +++++
 .github/dependabot.yml          |  12 ++
 .gitignore                      |   5 +
 performance_test.py             | 194 ++++++++++++++++++++++++++++++++
 pipeline/inspection.py          |   6 +
 test_data/README.md             |  19 ++++
 6 files changed, 266 insertions(+)
 create mode 100644 .devcontainer/devcontainer.json
 create mode 100644 .github/dependabot.yml
 create mode 100644 performance_test.py
 create mode 100644 test_data/README.md

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 0000000..9bfa4c5
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,30 @@
+{
+  "name": "Python 3",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {}
+  },
+  //"forwardPorts": [],
+  "postCreateCommand": "pip3 install --user -r requirements.txt",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "ms-toolsai.jupyter",
+        "stkb.rewrap",
+        "DavidAnson.vscode-markdownlint",
+        "charliermarsh.ruff",
+        "GitHub.vscode-pull-request-github"
+      ]
+    }
+  },
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+  ],
+  "remoteUser": "vscode", // Use a non-root user
+  "containerEnv": {
+    "PYTHONUNBUFFERED": "1",
+    "PYTHONPATH": "/workspace/src"
+  }
+}
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..f33a02c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly
diff --git a/.gitignore b/.gitignore
index c793664..06b1ab7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -169,3 +169,8 @@ cython_debug/
 
 # Testing artifacts
 end_to_end_pipeline_artifacts
+reports
+test_outputs
+
+
+logs
diff --git a/performance_test.py b/performance_test.py
new file mode 100644
index 0000000..425baad
--- /dev/null
+++ b/performance_test.py
@@ -0,0 +1,194 @@
+import os
+import time
+import json
+import shutil
+import datetime
+from typing import Dict, List
+
+from dotenv import load_dotenv
+from pipeline import analyze, LabelStorage, OCR, GPT
+from tests import levenshtein_similarity
+
+
+def validate_env_vars() -> None:
+    """Ensure all required environment variables are set.
+
+    This function checks for the presence of the environment variables needed for 
+    the pipeline to function correctly. If any required variables are missing, 
+    it raises an EnvironmentError.
+    """
+    required_vars = [
+        "AZURE_API_ENDPOINT",
+        "AZURE_API_KEY",
+        "AZURE_OPENAI_ENDPOINT",
+        "AZURE_OPENAI_KEY",
+        "AZURE_OPENAI_DEPLOYMENT",
+    ]
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    if missing_vars:
+        raise EnvironmentError(f"Missing required environment variables: {
+                               ', '.join(missing_vars)}")
+
+
+# Load environment variables
+load_dotenv()
+
+DOCUMENT_INTELLIGENCE_API_ENDPOINT = os.getenv("AZURE_API_ENDPOINT")
+DOCUMENT_INTELLIGENCE_API_KEY = os.getenv("AZURE_API_KEY")
+OPENAI_API_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
+OPENAI_API_KEY = os.getenv("AZURE_OPENAI_KEY")
+OPENAI_DEPLOYMENT_ID = os.getenv("AZURE_OPENAI_DEPLOYMENT")
+
+
+class TestCase:
+    def __init__(self, image_paths: List[str], expected_json_path: str):
+        """
+        Initialize a test case with image paths and the expected JSON output path.
+        """
+        self.original_image_paths = image_paths
+
+        # Note: The pipeline's `LabelStorage.clear()` method removes added images.
+        # To preserve the original test data for future runs, copies are created and used instead.
+        # This is clearly a workaround that will need to be addressed properly eventually.
+        self.image_paths = self._create_image_copies(image_paths)
+
+        self.expected_json_path = expected_json_path
+
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        os.makedirs("test_outputs", exist_ok=True)
+        self.actual_json_path = os.path.join(
+            "test_outputs", f'actual_output_{timestamp}.json')
+
+        self.results: Dict[str, float] = {}
+        self.label_storage = LabelStorage()
+        for image_path in self.image_paths:
+            self.label_storage.add_image(image_path)
+        self.ocr = OCR(DOCUMENT_INTELLIGENCE_API_ENDPOINT,
+                       DOCUMENT_INTELLIGENCE_API_KEY)
+        self.gpt = GPT(OPENAI_API_ENDPOINT, OPENAI_API_KEY,
+                       OPENAI_DEPLOYMENT_ID)
+
+    def _create_image_copies(self, image_paths: List[str]) -> List[str]:
+        """
+        Create copies of the input images to prevent deletion of original files 
+        done by the clear method of LabelStorage.
+        """
+        copied_paths = []
+        for image_path in image_paths:
+            # Create a copy of the image with a '_copy' suffix
+            base, ext = os.path.splitext(image_path)
+            copy_path = f"{base}_copy{ext}"
+            shutil.copy2(image_path, copy_path)
+            copied_paths.append(copy_path)
+        return copied_paths
+
+    def run_tests(self) -> None:
+        """
+        Run performance and accuracy tests for the pipeline.
+        """
+        self.run_performance_test()
+        self.run_accuracy_test()
+
+    def run_performance_test(self) -> None:
+        """Measure the time taken to run the pipeline analysis."""
+        start_time = time.time()
+        actual_output = analyze(self.label_storage, self.ocr, self.gpt)
+        end_time = time.time()
+        self.results['performance'] = end_time - start_time
+
+        # Save actual output to JSON file
+        self.save_json_output(actual_output.model_dump_json(indent=2))
+
+    def run_accuracy_test(self) -> None:
+        """Calculate and store the accuracy of the pipeline's output."""
+        self.results['accuracy'] = self.calculate_levenshtein_accuracy()
+
+    def calculate_levenshtein_accuracy(self) -> float:
+        """Calculate Levenshtein accuracy between expected and actual output."""
+        expected_json = self.load_json(self.expected_json_path)
+        actual_json = self.load_json(self.actual_json_path)
+
+        expected_str = json.dumps(expected_json, sort_keys=True)
+        actual_str = json.dumps(actual_json, sort_keys=True)
+
+        return levenshtein_similarity(expected_str, actual_str)
+
+    def save_json_output(self, output: str) -> None:
+        """Save the actual output JSON to a file."""
+        with open(self.actual_json_path, 'w') as f:
+            f.write(output)
+
+    @staticmethod
+    def load_json(path: str) -> Dict:
+        """Load and return JSON content from a file."""
+        with open(path, 'r') as f:
+            return json.load(f)
+
+
+class TestRunner:
+    def __init__(self, test_cases: List[TestCase]):
+        """
+        Initialize the test runner with a list of test cases.
+        """
+        self.test_cases = test_cases
+
+    def run_tests(self) -> None:
+        """Run all test cases."""
+        for current_test_case in self.test_cases:
+            current_test_case.run_tests()
+
+    def generate_report(self) -> None:
+        """Generate a report of the test results and save it as a timestamped .md file."""
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        os.makedirs("reports", exist_ok=True)
+        report_path = os.path.join(
+            "reports", f"performance_report_{timestamp}.md")
+
+        with open(report_path, 'w') as f:
+            f.write("# Performance Test Report\n\n")
+            f.write(f"Generated on: {
+                    datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
+
+            for i, test_case in enumerate(self.test_cases):
+                f.write(f"## Test Case {i+1}\n\n")
+                f.write(
+                    f"- Performance: {test_case.results['performance']:.4f} seconds\n")
+                f.write(f"- Accuracy: {test_case.results['accuracy']:.2f}\n")
+                f.write(
+                    f"- Actual output saved to: {test_case.actual_json_path}\n")
+                f.write(
+                    f"- Expected output path: {test_case.expected_json_path}\n")
+                f.write(
+                    f"- Original image paths: {', '.join(test_case.original_image_paths)}\n\n")
+
+        print(f"Report generated and saved to: {report_path}")
+
+
+if __name__ == "__main__":
+    validate_env_vars()
+
+    test_cases = []
+    labels_folder = "test_data/labels"
+
+    # Recursively analyze the labels folder
+    for root, dirs, files in os.walk(labels_folder):
+        for dir_name in dirs:
+            label_folder = os.path.join(root, dir_name)
+            image_paths = []
+            expected_json_path = ""
+
+            # Find image paths and expected output for each label folder
+            for file_name in os.listdir(label_folder):
+                file_path = os.path.join(label_folder, file_name)
+                if file_name.endswith((".png", ".jpg")):
+                    image_paths.append(file_path)
+                elif file_name == "expected_output.json":
+                    expected_json_path = file_path
+
+            # Create a test case if image paths and expected output are found
+            if image_paths and expected_json_path:
+                test_cases.append(TestCase(image_paths, expected_json_path))
+
+    runner = TestRunner(test_cases)
+    runner.run_tests()
+    runner.generate_report()
diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index ba60087..b8989e8 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -75,6 +75,12 @@ class FertilizerInspection(BaseModel):
     specifications_fr: List[Specification] = []
     first_aid_fr: List[str] = None
 
+    @field_validator('company_phone_number', 'manufacturer_phone_number', mode='before', check_fields=False)
+    def handle_phone_numbers(cls, v):
+        if not isinstance(v, str):
+            return ''
+        return v
+
     @field_validator('weight_kg', 'weight_lb', 'density', 'volume', mode='before', check_fields=False)
     def convert_values(cls, v):
         if isinstance(v, (int, float)):
diff --git a/test_data/README.md b/test_data/README.md
new file mode 100644
index 0000000..43e56c6
--- /dev/null
+++ b/test_data/README.md
@@ -0,0 +1,19 @@
+### Test Data
+
+This folder hosts all the data needed to perform testing on the processing pipeline (label images and their expected output).
+
+Please follow the following structure when adding new test cases:
+
+```
+├── test_data/                # Test images and related data
+│   ├── labels/               # Folders organized by test case
+│   │   ├── label_001/        # Each folder contains images and expected output JSON
+│   │   │   ├── img_001.jpg
+│   │   │   ├── img_002.jpg
+│   │   │   └── expected_output.json
+│   │   ├── label_002/
+│   │   │   ├── img_001.jpg
+│   │   │   ├── img_002.jpg
+│   │   │   └── expected_output.json
+│   │   └── ...
+```
\ No newline at end of file

From 4745032c5fdd712df88be5bd7de190b80e14f6fc Mon Sep 17 00:00:00 2001
From: Bryan Ndjeutcha <ndjeutcha@gmail.com>
Date: Wed, 28 Aug 2024 10:22:34 -0400
Subject: [PATCH 02/17] feat: add a new input field for the form signature for
 the json_schema

---
 pipeline/gpt.py | 44 ++++++++------------------------------------
 1 file changed, 8 insertions(+), 36 deletions(-)

diff --git a/pipeline/gpt.py b/pipeline/gpt.py
index ae3a8ec..b33f67e 100644
--- a/pipeline/gpt.py
+++ b/pipeline/gpt.py
@@ -3,44 +3,14 @@
 import dspy.adapters
 import dspy.utils
 
+from pipeline.inspection import FertilizerInspection
+
 MODELS_WITH_RESPONSE_FORMAT = [
     "ailab-llm",
     "ailab-llm-gpt-4o"
 ]  # List of models that support the response_format option
 
-SPECIFICATION = """
-Keys:
-"company_name"
-"company_address"
-"company_website"
-"company_phone_number"
-"manufacturer_name"
-"manufacturer_address"
-"manufacturer_website"
-"manufacturer_phone_number"
-"fertiliser_name"
-"registration_number" (a series of letters and numbers)
-"lot_number"
-"weight" (array of objects with "value", and "unit")
-"density" (an object with "value", and "unit")
-"volume" (an object with "value", and "unit")
-"npk" (format: "number-number-number") **important
-"guaranteed_analysis" (array of objects with "nutrient", "value", and "unit") **important
-"warranty"
-"cautions_en"  (array of strings)
-"instructions_en" (array of strings)
-"micronutrients_en" (array of objects with "nutrient", "value", and "unit")
-"ingredients_en" (array of objects with "nutrient", "value", and "unit")
-"specifications_en" (array of objects with "humidity", "ph", and "solubility")
-"first_aid_en"  (array of strings)
-"cautions_fr" (array of strings)
-"instructions_fr" (array of strings)
-"micronutrients_fr" (array of objects with "nutrient", "value", and "unit")
-"ingredients_fr" (array of objects with "nutrient", "value", and "unit")
-"specifications_fr" (array of objects with "humidity", "ph", and "solubility")
-"first_aid_fr" (array of strings)
-
-Requirements:
+REQUIREMENTS = """
 The content of keys with the suffix _en must be in English.
 The content of keys with the suffix _fr must be in French.
 Translation of the text is prohibited.
@@ -56,7 +26,8 @@ class ProduceLabelForm(dspy.Signature):
     """
     
     text = dspy.InputField(desc="The text of the fertilizer label extracted using OCR.")
-    specification = dspy.InputField(desc="The specification containing the fields to highlight and their requirements.")
+    json_schema = dspy.InputField(desc="The JSON schema of the object to be returned.")
+    requirements = dspy.InputField(desc="The instructions and guidelines to follow.")
     inspection = dspy.OutputField(desc="Only a complete JSON.")
 
 class GPT:
@@ -87,9 +58,10 @@ def __init__(self, api_endpoint, api_key, deployment_id):
             response_format=response_format,
         )
 
-    def create_inspection(self, prompt) -> Prediction:
+    def create_inspection(self, text) -> Prediction:
         with dspy.context(lm=self.dspy_client, experimental=True):
+            json_schema = FertilizerInspection.model_json_schema()
             signature = dspy.ChainOfThought(ProduceLabelForm)
-            prediction = signature(specification=SPECIFICATION, text=prompt)
+            prediction = signature(text=text, json_schema=json_schema, requirements=REQUIREMENTS)
 
         return prediction

From 67d81836aefe5c8cfff196570cb6a23c506faf8c Mon Sep 17 00:00:00 2001
From: Bryan Ndjeutcha <49378990+snakedye@users.noreply.github.com>
Date: Mon, 26 Aug 2024 10:46:21 -0400
Subject: [PATCH 03/17] fix: clean up expected.json (#17)

---
 expected.json | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/expected.json b/expected.json
index 786b7a6..8e629fa 100644
--- a/expected.json
+++ b/expected.json
@@ -135,19 +135,13 @@
       "unit":"%"
     },
     {
-      "nutrient":"Argile",
-      "value":"",
-      "unit":""
+      "nutrient":"Argile"
     },
     {
-      "nutrient":"Sable",
-      "value":"",
-      "unit":""
+      "nutrient":"Sable"
     },
     {
-      "nutrient":"Perlite",
-      "value":"",
-      "unit":""
+      "nutrient":"Perlite"
     }
   ],
   "specifications_fr":[

From f8edef8c137d9d4387f58f94ccfd325cf4893c87 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Tue, 27 Aug 2024 04:18:43 -0400
Subject: [PATCH 04/17] feat: setup devcontainer + implement basic performance
 testing

This commit adds a basic devcontainer configuration and implements a simple naive framework for performance testing.
---
 .devcontainer/devcontainer.json |  30 +++++
 .github/dependabot.yml          |  12 ++
 .gitignore                      |   5 +
 performance_test.py             | 194 ++++++++++++++++++++++++++++++++
 pipeline/inspection.py          |   6 +
 test_data/README.md             |  19 ++++
 6 files changed, 266 insertions(+)
 create mode 100644 .devcontainer/devcontainer.json
 create mode 100644 .github/dependabot.yml
 create mode 100644 performance_test.py
 create mode 100644 test_data/README.md

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 0000000..9bfa4c5
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,30 @@
+{
+  "name": "Python 3",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye",
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {}
+  },
+  //"forwardPorts": [],
+  "postCreateCommand": "pip3 install --user -r requirements.txt",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "ms-toolsai.jupyter",
+        "stkb.rewrap",
+        "DavidAnson.vscode-markdownlint",
+        "charliermarsh.ruff",
+        "GitHub.vscode-pull-request-github"
+      ]
+    }
+  },
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+  ],
+  "remoteUser": "vscode", // Use a non-root user
+  "containerEnv": {
+    "PYTHONUNBUFFERED": "1",
+    "PYTHONPATH": "/workspace/src"
+  }
+}
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..f33a02c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly
diff --git a/.gitignore b/.gitignore
index c793664..06b1ab7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -169,3 +169,8 @@ cython_debug/
 
 # Testing artifacts
 end_to_end_pipeline_artifacts
+reports
+test_outputs
+
+
+logs
diff --git a/performance_test.py b/performance_test.py
new file mode 100644
index 0000000..425baad
--- /dev/null
+++ b/performance_test.py
@@ -0,0 +1,194 @@
+import os
+import time
+import json
+import shutil
+import datetime
+from typing import Dict, List
+
+from dotenv import load_dotenv
+from pipeline import analyze, LabelStorage, OCR, GPT
+from tests import levenshtein_similarity
+
+
+def validate_env_vars() -> None:
+    """Ensure all required environment variables are set.
+
+    This function checks for the presence of the environment variables needed for 
+    the pipeline to function correctly. If any required variables are missing, 
+    it raises an EnvironmentError.
+    """
+    required_vars = [
+        "AZURE_API_ENDPOINT",
+        "AZURE_API_KEY",
+        "AZURE_OPENAI_ENDPOINT",
+        "AZURE_OPENAI_KEY",
+        "AZURE_OPENAI_DEPLOYMENT",
+    ]
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    if missing_vars:
+        raise EnvironmentError(f"Missing required environment variables: {
+                               ', '.join(missing_vars)}")
+
+
+# Load environment variables
+load_dotenv()
+
+DOCUMENT_INTELLIGENCE_API_ENDPOINT = os.getenv("AZURE_API_ENDPOINT")
+DOCUMENT_INTELLIGENCE_API_KEY = os.getenv("AZURE_API_KEY")
+OPENAI_API_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
+OPENAI_API_KEY = os.getenv("AZURE_OPENAI_KEY")
+OPENAI_DEPLOYMENT_ID = os.getenv("AZURE_OPENAI_DEPLOYMENT")
+
+
+class TestCase:
+    def __init__(self, image_paths: List[str], expected_json_path: str):
+        """
+        Initialize a test case with image paths and the expected JSON output path.
+        """
+        self.original_image_paths = image_paths
+
+        # Note: The pipeline's `LabelStorage.clear()` method removes added images.
+        # To preserve the original test data for future runs, copies are created and used instead.
+        # This is clearly a workaround that will need to be addressed properly eventually.
+        self.image_paths = self._create_image_copies(image_paths)
+
+        self.expected_json_path = expected_json_path
+
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        os.makedirs("test_outputs", exist_ok=True)
+        self.actual_json_path = os.path.join(
+            "test_outputs", f'actual_output_{timestamp}.json')
+
+        self.results: Dict[str, float] = {}
+        self.label_storage = LabelStorage()
+        for image_path in self.image_paths:
+            self.label_storage.add_image(image_path)
+        self.ocr = OCR(DOCUMENT_INTELLIGENCE_API_ENDPOINT,
+                       DOCUMENT_INTELLIGENCE_API_KEY)
+        self.gpt = GPT(OPENAI_API_ENDPOINT, OPENAI_API_KEY,
+                       OPENAI_DEPLOYMENT_ID)
+
+    def _create_image_copies(self, image_paths: List[str]) -> List[str]:
+        """
+        Create copies of the input images to prevent deletion of original files 
+        done by the clear method of LabelStorage.
+        """
+        copied_paths = []
+        for image_path in image_paths:
+            # Create a copy of the image with a '_copy' suffix
+            base, ext = os.path.splitext(image_path)
+            copy_path = f"{base}_copy{ext}"
+            shutil.copy2(image_path, copy_path)
+            copied_paths.append(copy_path)
+        return copied_paths
+
+    def run_tests(self) -> None:
+        """
+        Run performance and accuracy tests for the pipeline.
+        """
+        self.run_performance_test()
+        self.run_accuracy_test()
+
+    def run_performance_test(self) -> None:
+        """Measure the time taken to run the pipeline analysis."""
+        start_time = time.time()
+        actual_output = analyze(self.label_storage, self.ocr, self.gpt)
+        end_time = time.time()
+        self.results['performance'] = end_time - start_time
+
+        # Save actual output to JSON file
+        self.save_json_output(actual_output.model_dump_json(indent=2))
+
+    def run_accuracy_test(self) -> None:
+        """Calculate and store the accuracy of the pipeline's output."""
+        self.results['accuracy'] = self.calculate_levenshtein_accuracy()
+
+    def calculate_levenshtein_accuracy(self) -> float:
+        """Calculate Levenshtein accuracy between expected and actual output."""
+        expected_json = self.load_json(self.expected_json_path)
+        actual_json = self.load_json(self.actual_json_path)
+
+        expected_str = json.dumps(expected_json, sort_keys=True)
+        actual_str = json.dumps(actual_json, sort_keys=True)
+
+        return levenshtein_similarity(expected_str, actual_str)
+
+    def save_json_output(self, output: str) -> None:
+        """Save the actual output JSON to a file."""
+        with open(self.actual_json_path, 'w') as f:
+            f.write(output)
+
+    @staticmethod
+    def load_json(path: str) -> Dict:
+        """Load and return JSON content from a file."""
+        with open(path, 'r') as f:
+            return json.load(f)
+
+
+class TestRunner:
+    def __init__(self, test_cases: List[TestCase]):
+        """
+        Initialize the test runner with a list of test cases.
+        """
+        self.test_cases = test_cases
+
+    def run_tests(self) -> None:
+        """Run all test cases."""
+        for current_test_case in self.test_cases:
+            current_test_case.run_tests()
+
+    def generate_report(self) -> None:
+        """Generate a report of the test results and save it as a timestamped .md file."""
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        os.makedirs("reports", exist_ok=True)
+        report_path = os.path.join(
+            "reports", f"performance_report_{timestamp}.md")
+
+        with open(report_path, 'w') as f:
+            f.write("# Performance Test Report\n\n")
+            f.write(f"Generated on: {
+                    datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
+
+            for i, test_case in enumerate(self.test_cases):
+                f.write(f"## Test Case {i+1}\n\n")
+                f.write(
+                    f"- Performance: {test_case.results['performance']:.4f} seconds\n")
+                f.write(f"- Accuracy: {test_case.results['accuracy']:.2f}\n")
+                f.write(
+                    f"- Actual output saved to: {test_case.actual_json_path}\n")
+                f.write(
+                    f"- Expected output path: {test_case.expected_json_path}\n")
+                f.write(
+                    f"- Original image paths: {', '.join(test_case.original_image_paths)}\n\n")
+
+        print(f"Report generated and saved to: {report_path}")
+
+
+if __name__ == "__main__":
+    validate_env_vars()
+
+    test_cases = []
+    labels_folder = "test_data/labels"
+
+    # Recursively analyze the labels folder
+    for root, dirs, files in os.walk(labels_folder):
+        for dir_name in dirs:
+            label_folder = os.path.join(root, dir_name)
+            image_paths = []
+            expected_json_path = ""
+
+            # Find image paths and expected output for each label folder
+            for file_name in os.listdir(label_folder):
+                file_path = os.path.join(label_folder, file_name)
+                if file_name.endswith((".png", ".jpg")):
+                    image_paths.append(file_path)
+                elif file_name == "expected_output.json":
+                    expected_json_path = file_path
+
+            # Create a test case if image paths and expected output are found
+            if image_paths and expected_json_path:
+                test_cases.append(TestCase(image_paths, expected_json_path))
+
+    runner = TestRunner(test_cases)
+    runner.run_tests()
+    runner.generate_report()
diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index ba60087..b8989e8 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -75,6 +75,12 @@ class FertilizerInspection(BaseModel):
     specifications_fr: List[Specification] = []
     first_aid_fr: List[str] = None
 
+    @field_validator('company_phone_number', 'manufacturer_phone_number', mode='before', check_fields=False)
+    def handle_phone_numbers(cls, v):
+        if not isinstance(v, str):
+            return ''
+        return v
+
     @field_validator('weight_kg', 'weight_lb', 'density', 'volume', mode='before', check_fields=False)
     def convert_values(cls, v):
         if isinstance(v, (int, float)):
diff --git a/test_data/README.md b/test_data/README.md
new file mode 100644
index 0000000..43e56c6
--- /dev/null
+++ b/test_data/README.md
@@ -0,0 +1,19 @@
+### Test Data
+
+This folder hosts all the data needed to perform testing on the processing pipeline (label images and their expected output).
+
+Please follow the following structure when adding new test cases:
+
+```
+├── test_data/                # Test images and related data
+│   ├── labels/               # Folders organized by test case
+│   │   ├── label_001/        # Each folder contains images and expected output JSON
+│   │   │   ├── img_001.jpg
+│   │   │   ├── img_002.jpg
+│   │   │   └── expected_output.json
+│   │   ├── label_002/
+│   │   │   ├── img_001.jpg
+│   │   │   ├── img_002.jpg
+│   │   │   └── expected_output.json
+│   │   └── ...
+```
\ No newline at end of file

From 869497e004f88cdcfae9ed5dabb9c9115c43ebbc Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Thu, 12 Sep 2024 06:38:38 +0000
Subject: [PATCH 05/17] feat: adding the feature to mesure accuracy for one
 field at a time and save the result in a csv file

---
 performance_test.py | 281 ++++++++++++++++++++++----------------------
 pipeline/gpt.py     |   5 +-
 2 files changed, 144 insertions(+), 142 deletions(-)

diff --git a/performance_test.py b/performance_test.py
index 425baad..e6023aa 100644
--- a/performance_test.py
+++ b/performance_test.py
@@ -3,20 +3,17 @@
 import json
 import shutil
 import datetime
-from typing import Dict, List
+import csv
+from typing import Dict, List, Union
 
 from dotenv import load_dotenv
 from pipeline import analyze, LabelStorage, OCR, GPT
 from tests import levenshtein_similarity
 
+ACCURACY_THRESHOLD = 80.0
 
-def validate_env_vars() -> None:
-    """Ensure all required environment variables are set.
-
-    This function checks for the presence of the environment variables needed for 
-    the pipeline to function correctly. If any required variables are missing, 
-    it raises an EnvironmentError.
-    """
+def validate_environment_variables() -> None:
+    """Ensure all required environment variables are set."""
     required_vars = [
         "AZURE_API_ENDPOINT",
         "AZURE_API_KEY",
@@ -26,169 +23,177 @@ def validate_env_vars() -> None:
     ]
     missing_vars = [var for var in required_vars if not os.getenv(var)]
     if missing_vars:
-        raise EnvironmentError(f"Missing required environment variables: {
-                               ', '.join(missing_vars)}")
-
-
-# Load environment variables
-load_dotenv()
-
-DOCUMENT_INTELLIGENCE_API_ENDPOINT = os.getenv("AZURE_API_ENDPOINT")
-DOCUMENT_INTELLIGENCE_API_KEY = os.getenv("AZURE_API_KEY")
-OPENAI_API_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
-OPENAI_API_KEY = os.getenv("AZURE_OPENAI_KEY")
-OPENAI_DEPLOYMENT_ID = os.getenv("AZURE_OPENAI_DEPLOYMENT")
-
+        raise EnvironmentError(f"Missing required environment variables: {', '.join(missing_vars)}")
+
+def classify_test_result(score: float) -> str:
+    """Classify test results as Pass or Fail based on accuracy score."""
+    return "Pass" if score >= ACCURACY_THRESHOLD else "Fail"
+
+def load_json_file(file_path: str) -> Dict:
+    """Load and return JSON content from a file."""
+    with open(file_path, 'r') as file:
+        return json.load(file)
+
+def extract_leaf_fields(data: Union[dict, list], parent_key: str = '') -> Dict[str, Union[str, int, float, bool, None]]:
+    """Extract all leaf fields from nested dictionaries and lists."""
+    leaves = {}
+
+    if isinstance(data, dict):
+        for key, value in data.items():
+            new_key = f"{parent_key}.{key}" if parent_key else key
+            if isinstance(value, (dict, list)):
+                leaves.update(extract_leaf_fields(value, new_key))
+            else:
+                leaves[new_key] = value
+    elif isinstance(data, list):
+        for index, item in enumerate(data):
+            list_key = f"{parent_key}[{index}]"
+            leaves.update(extract_leaf_fields(item, list_key))
+
+    return leaves
 
 class TestCase:
     def __init__(self, image_paths: List[str], expected_json_path: str):
-        """
-        Initialize a test case with image paths and the expected JSON output path.
-        """
+        """Initialize a test case with image paths and the expected JSON output path."""
         self.original_image_paths = image_paths
-
-        # Note: The pipeline's `LabelStorage.clear()` method removes added images.
-        # To preserve the original test data for future runs, copies are created and used instead.
-        # This is clearly a workaround that will need to be addressed properly eventually.
         self.image_paths = self._create_image_copies(image_paths)
-
         self.expected_json_path = expected_json_path
+        self.actual_json_path = self._generate_output_path()
+        self.results: Dict[str, float] = {}
+        self.label_storage = self._initialize_label_storage()
+        self.ocr = self._initialize_ocr()
+        self.gpt = self._initialize_gpt()
 
+    def _create_image_copies(self, image_paths: List[str]) -> List[str]:
+        """Create copies of the input images to prevent deletion of original files."""
+        return [self._copy_image(path) for path in image_paths]
+
+    def _copy_image(self, image_path: str) -> str:
+        """Create a copy of a single image file."""
+        base, ext = os.path.splitext(image_path)
+        copy_path = f"{base}_copy{ext}"
+        shutil.copy2(image_path, copy_path)
+        return copy_path
+
+    def _generate_output_path(self) -> str:
+        """Generate a timestamped path for the actual output JSON."""
         timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
         os.makedirs("test_outputs", exist_ok=True)
-        self.actual_json_path = os.path.join(
-            "test_outputs", f'actual_output_{timestamp}.json')
+        return os.path.join("test_outputs", f'actual_output_{timestamp}.json')
 
-        self.results: Dict[str, float] = {}
-        self.label_storage = LabelStorage()
+    def _initialize_label_storage(self) -> LabelStorage:
+        """Initialize and populate LabelStorage with image paths."""
+        storage = LabelStorage()
         for image_path in self.image_paths:
-            self.label_storage.add_image(image_path)
-        self.ocr = OCR(DOCUMENT_INTELLIGENCE_API_ENDPOINT,
-                       DOCUMENT_INTELLIGENCE_API_KEY)
-        self.gpt = GPT(OPENAI_API_ENDPOINT, OPENAI_API_KEY,
-                       OPENAI_DEPLOYMENT_ID)
+            storage.add_image(image_path)
+        return storage
 
-    def _create_image_copies(self, image_paths: List[str]) -> List[str]:
-        """
-        Create copies of the input images to prevent deletion of original files 
-        done by the clear method of LabelStorage.
-        """
-        copied_paths = []
-        for image_path in image_paths:
-            # Create a copy of the image with a '_copy' suffix
-            base, ext = os.path.splitext(image_path)
-            copy_path = f"{base}_copy{ext}"
-            shutil.copy2(image_path, copy_path)
-            copied_paths.append(copy_path)
-        return copied_paths
+    def _initialize_ocr(self) -> OCR:
+        """Initialize OCR with API credentials."""
+        return OCR(os.getenv("AZURE_API_ENDPOINT"), os.getenv("AZURE_API_KEY"))
+
+    def _initialize_gpt(self) -> GPT:
+        """Initialize GPT with API credentials."""
+        return GPT(os.getenv("AZURE_OPENAI_ENDPOINT"), os.getenv("AZURE_OPENAI_KEY"), os.getenv("AZURE_OPENAI_DEPLOYMENT"))
+
+    def save_json_output(self, output: str) -> None:
+        """Save the actual output JSON to a file."""
+        with open(self.actual_json_path, 'w') as file:
+            file.write(output)
 
     def run_tests(self) -> None:
-        """
-        Run performance and accuracy tests for the pipeline.
-        """
-        self.run_performance_test()
-        self.run_accuracy_test()
+        """Run performance and accuracy tests for the pipeline."""
+        self._run_performance_test()
+        self._run_accuracy_test()
 
-    def run_performance_test(self) -> None:
+    def _run_performance_test(self) -> None:
         """Measure the time taken to run the pipeline analysis."""
         start_time = time.time()
         actual_output = analyze(self.label_storage, self.ocr, self.gpt)
         end_time = time.time()
         self.results['performance'] = end_time - start_time
-
-        # Save actual output to JSON file
         self.save_json_output(actual_output.model_dump_json(indent=2))
 
-    def run_accuracy_test(self) -> None:
+    def _run_accuracy_test(self) -> None:
         """Calculate and store the accuracy of the pipeline's output."""
-        self.results['accuracy'] = self.calculate_levenshtein_accuracy()
-
-    def calculate_levenshtein_accuracy(self) -> float:
-        """Calculate Levenshtein accuracy between expected and actual output."""
-        expected_json = self.load_json(self.expected_json_path)
-        actual_json = self.load_json(self.actual_json_path)
+        self.results['accuracy'] = self._calculate_levenshtein_accuracy()
 
-        expected_str = json.dumps(expected_json, sort_keys=True)
-        actual_str = json.dumps(actual_json, sort_keys=True)
-
-        return levenshtein_similarity(expected_str, actual_str)
-
-    def save_json_output(self, output: str) -> None:
-        """Save the actual output JSON to a file."""
-        with open(self.actual_json_path, 'w') as f:
-            f.write(output)
-
-    @staticmethod
-    def load_json(path: str) -> Dict:
-        """Load and return JSON content from a file."""
-        with open(path, 'r') as f:
-            return json.load(f)
+    def _calculate_levenshtein_accuracy(self) -> Dict[str, float]:
+        """Calculate Levenshtein accuracy per field between expected and actual output."""
+        expected_fields = extract_leaf_fields(load_json_file(self.expected_json_path))
+        actual_fields = extract_leaf_fields(load_json_file(self.actual_json_path))
 
+        return {
+            field_name: levenshtein_similarity(str(field_value), str(actual_fields.get(field_name)))
+            for field_name, field_value in expected_fields.items()
+        }
 
 class TestRunner:
     def __init__(self, test_cases: List[TestCase]):
-        """
-        Initialize the test runner with a list of test cases.
-        """
+        """Initialize the test runner with a list of test cases."""
         self.test_cases = test_cases
 
     def run_tests(self) -> None:
         """Run all test cases."""
-        for current_test_case in self.test_cases:
-            current_test_case.run_tests()
-
-    def generate_report(self) -> None:
-        """Generate a report of the test results and save it as a timestamped .md file."""
-        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        for test_case in self.test_cases:
+            test_case.run_tests()
+
+    def generate_csv_report(self) -> None:
+        """Generate a CSV report of the test results and save it as a timestamped .csv file."""
+        report_path = self._generate_report_path()
+        with open(report_path, mode='w', newline='') as file:
+            writer = csv.writer(file)
+            writer.writerow(self._get_csv_header())
+            self._write_test_results(writer)
+        print(f"CSV report generated and saved to: {report_path}")
+
+    def _generate_report_path(self) -> str:
+        """Generate a timestamped path for the CSV report."""
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M")
         os.makedirs("reports", exist_ok=True)
-        report_path = os.path.join(
-            "reports", f"performance_report_{timestamp}.md")
-
-        with open(report_path, 'w') as f:
-            f.write("# Performance Test Report\n\n")
-            f.write(f"Generated on: {
-                    datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
-
-            for i, test_case in enumerate(self.test_cases):
-                f.write(f"## Test Case {i+1}\n\n")
-                f.write(
-                    f"- Performance: {test_case.results['performance']:.4f} seconds\n")
-                f.write(f"- Accuracy: {test_case.results['accuracy']:.2f}\n")
-                f.write(
-                    f"- Actual output saved to: {test_case.actual_json_path}\n")
-                f.write(
-                    f"- Expected output path: {test_case.expected_json_path}\n")
-                f.write(
-                    f"- Original image paths: {', '.join(test_case.original_image_paths)}\n\n")
-
-        print(f"Report generated and saved to: {report_path}")
-
-
-if __name__ == "__main__":
-    validate_env_vars()
-
+        return os.path.join("reports", f"test_results_{timestamp}.csv")
+
+    def _get_csv_header(self) -> List[str]:
+        """Return the CSV header row."""
+        return ["Test Case", "Field Name", "Accuracy Score", "Expected Value", "Actual Value", "Pass/Fail", "Pipeline Speed (seconds)"]
+
+    def _write_test_results(self, writer: csv.writer) -> None:
+        """Write the test results for each test case to the CSV file."""
+        for i, test_case in enumerate(self.test_cases, 1):
+            expected_fields = extract_leaf_fields(load_json_file(test_case.expected_json_path))
+            actual_fields = extract_leaf_fields(load_json_file(test_case.actual_json_path))
+            performance = test_case.results['performance']
+
+            for field_name, score in test_case.results['accuracy'].items():
+                writer.writerow([
+                    f"{i}",
+                    field_name,
+                    f"{score:.2f}",
+                    expected_fields.get(field_name, ""),
+                    actual_fields.get(field_name, ""),
+                    classify_test_result(score),
+                    f"{performance:.4f}"
+                ])
+
+def find_test_cases(labels_folder: str) -> List[TestCase]:
+    """Find and create test cases from the labels folder."""
     test_cases = []
-    labels_folder = "test_data/labels"
-
-    # Recursively analyze the labels folder
-    for root, dirs, files in os.walk(labels_folder):
-        for dir_name in dirs:
-            label_folder = os.path.join(root, dir_name)
-            image_paths = []
-            expected_json_path = ""
-
-            # Find image paths and expected output for each label folder
-            for file_name in os.listdir(label_folder):
-                file_path = os.path.join(label_folder, file_name)
-                if file_name.endswith((".png", ".jpg")):
-                    image_paths.append(file_path)
-                elif file_name == "expected_output.json":
-                    expected_json_path = file_path
-
-            # Create a test case if image paths and expected output are found
-            if image_paths and expected_json_path:
-                test_cases.append(TestCase(image_paths, expected_json_path))
-
+    for root, _, files in os.walk(labels_folder):
+        image_paths = [os.path.join(root, f) for f in files if f.lower().endswith((".png", ".jpg"))]
+        expected_json_path = os.path.join(root, "expected_output.json")
+        if image_paths and os.path.exists(expected_json_path):
+            test_cases.append(TestCase(image_paths, expected_json_path))
+    return test_cases
+
+def main():
+    """Main function to run the performance tests."""
+    load_dotenv()
+    validate_environment_variables()
+
+    test_cases = find_test_cases("test_data/labels")
     runner = TestRunner(test_cases)
     runner.run_tests()
-    runner.generate_report()
+    runner.generate_csv_report()
+
+if __name__ == "__main__":
+    main()
diff --git a/pipeline/gpt.py b/pipeline/gpt.py
index b33f67e..1725223 100644
--- a/pipeline/gpt.py
+++ b/pipeline/gpt.py
@@ -6,8 +6,7 @@
 from pipeline.inspection import FertilizerInspection
 
 MODELS_WITH_RESPONSE_FORMAT = [
-    "ailab-llm",
-    "ailab-llm-gpt-4o"
+    "gpt-4o"
 ]  # List of models that support the response_format option
 
 REQUIREMENTS = """
@@ -42,8 +41,6 @@ def __init__(self, api_endpoint, api_key, deployment_id):
         max_token = 12000
         api_version = "2024-02-01"
         if deployment_id == MODELS_WITH_RESPONSE_FORMAT[0]:
-            max_token = 3500
-        elif deployment_id == MODELS_WITH_RESPONSE_FORMAT[1]:
             max_token = 4096
             api_version="2024-02-15-preview"
 

From 6638a0f704213ae94a63b16ca573617a3741f9f7 Mon Sep 17 00:00:00 2001
From: Bryan Ndjeutcha <49378990+snakedye@users.noreply.github.com>
Date: Mon, 16 Sep 2024 09:42:38 -0400
Subject: [PATCH 06/17] Issue #22: JSON Schema (#23)

* feat: add a new input field for the form signature for the json_schema

* feat: Update max_token value for gpt-4o model

The `max_token` value for the `gpt-4o` model in the `gpt.py` file was changed to `None`. This change allows for unlimited token length when making API calls with the `gpt-4o` model.

* feat: Update company information in expected.json

The company information in the `expected.json` file was updated to reflect the new details of GreenGrow Inc. This change includes the company name, address, website, and phone number.

* fix: remove newline at end of file in test_inspection.py

* chore: Update test_gpt.py with translated warranty information and nutrient values

* feat: Add translated nutrient values for ingredients in expected.json

The code changes include adding nutrient values for ingredients in the expected.json file. This enhancement improves the accuracy and completeness of the data. The commit message follows the established convention of using a "feat" prefix to indicate a new feature or enhancement.

* feat: Update nutrient values in expected.json

The code changes involve updating the nutrient values in the expected.json file. This improves the accuracy and completeness of the data. The commit message follows the established convention of using a "feat" prefix for new features or enhancements.

* feat: Update nutrient values in expected.json

* refactor: Refactor field validation in inspection.py

Refactor the field validation in the `GuaranteedAnalysis` and `FertilizerInspection` classes in `inspection.py`. The `replace_none_with_empty_list` methods have been updated to use the `field_validator` decorator instead of the `model_validator` decorator. This change improves the readability and maintainability of the code.
---
 expected.json          | 200 +++++++++++++++++------------------------
 pipeline/inspection.py |  56 ++++++------
 tests/test_form.py     |  32 -------
 tests/test_gpt.py      |   4 +
 tests/test_pipeline.py |  10 +--
 5 files changed, 121 insertions(+), 181 deletions(-)
 delete mode 100644 tests/test_form.py

diff --git a/expected.json b/expected.json
index 8e629fa..7a94177 100644
--- a/expected.json
+++ b/expected.json
@@ -1,75 +1,84 @@
 {
-  "company_name":"GreenGrow Fertilizers Inc.",
-  "company_address":"123 Greenway Blvd, Springfield IL 62701 USA",
-  "company_website":"www.greengrowfertilizers.com",
-  "company_phone_number":"+1 800 555 0199",
-  "manufacturer_name":"AgroTech Industries Ltd.",
-  "manufacturer_address":"456 Industrial Park Rd, Oakville ON L6H 5V4 Canada",
-  "manufacturer_website":"www.agrotechindustries.com",
-  "manufacturer_phone_number":"+1 416 555 0123",
-  "fertiliser_name":"SuperGrow 20-20-20",
-  "registration_number":"F12345678",
-  "lot_number":"L987654321",
-  "weight":[
-    {
-      "value":"25",
-      "unit":"kg"
-    },
-    {
-      "value":"55",
-      "unit":"lb"
+  "company_name": "GreenGrow Inc.",
+  "company_address": "123 Green Road, Farmville, State, 12345",
+  "company_website": "https://www.greengrow.com",
+  "company_phone_number": "123-456-7890",
+  "manufacturer_name": "AgriSupply Co.",
+  "manufacturer_address": "456 Supply Lane, AgriTown, State, 67890",
+  "manufacturer_website": "https://www.agrisupply.com",
+  "manufacturer_phone_number": "987-654-3210",
+  "fertiliser_name": "GreenGrow Fertilizer 20-20-20",
+  "registration_number": "FG123456",
+  "lot_number": "LOT20240901",
+  "weight": [
+    {
+      "value": 50,
+      "unit": "kg"
     }
   ],
-  "density":{
-    "value":"1.2",
-    "unit":"g/cm³"
+  "density": {
+    "value": 1.5,
+    "unit": "g/cm³"
   },
-  "volume":{
-    "value":"20.8",
-    "unit":"L"
+  "volume": {
+    "value": 33.3,
+    "unit": "L"
+  },
+  "npk": "20-20-20",
+  "guaranteed_analysis_en": {
+    "title": "Guaranteed Analysis",
+    "nutrients": [
+      {
+        "nutrient": "Total Nitrogen (N)",
+        "value": 20,
+        "unit": "%"
+      },
+      {
+        "nutrient": "Available Phosphate (P2O5)",
+        "value": 20,
+        "unit": "%"
+      },
+      {
+        "nutrient": "Soluble Potash (K2O)",
+        "value": 20,
+        "unit": "%"
+      }
+    ]
+  },
+  "guaranteed_analysis_fr": {
+    "title": "Analyse Garantie",
+    "nutrients": [
+      {
+        "nutrient": "Azote total (N)",
+        "value": 20,
+        "unit": "%"
+      },
+      {
+        "nutrient": "Phosphate assimilable (P2O5)",
+        "value": 20,
+        "unit": "%"
+      },
+      {
+        "nutrient": "Potasse soluble (K2O)",
+        "value": 20,
+        "unit": "%"
+      }
+    ]
   },
-  "npk":"20-20-20",
-  "warranty":"Guaranteed analysis of nutrients.",
-  "cautions_en":[
-    "Keep out of reach of children.",
-    "Avoid contact with skin and eyes."
-  ],
-  "instructions_en":[
-    "1. Dissolve 50g in 10L of water.",
-    "2. Apply every 2 weeks.",
-    "3. Store in a cool, dry place."
-  ],
-  "micronutrients_en":[
-    {
-      "nutrient":"Iron (Fe)",
-      "value":"0.10",
-      "unit":"%"
-    },
-    {
-      "nutrient":"Zinc (Zn)",
-      "value":"0.05",
-      "unit":"%"
-    },
-    {
-      "nutrient":"Manganese (Mn)",
-      "value":"0.05",
-      "unit":"%"
-    }
-  ],
   "ingredients_en":[
     {
       "nutrient":"Bone meal",
-      "value":"5",
+      "value":5,
       "unit":"%"
     },
     {
       "nutrient":"Seaweed extract",
-      "value":"3",
+      "value":3,
       "unit":"%"
     },
     {
       "nutrient":"Humic acid",
-      "value":"2",
+      "value":2,
       "unit":"%"
     },
     {
@@ -82,56 +91,20 @@
       "nutrient":"Perlite"
     }
   ],
-  "specifications_en":[
-    {
-      "humidity":"10",
-      "ph":"6.5",
-      "solubility":"100"
-    }
-  ],
-  "first_aid_en":[
-    "In case of contact with eyes, rinse immediately with plenty of water and seek medical advice."
-  ],
-  "cautions_fr":[
-    "Tenir hors de portée des enfants.",
-    "Éviter le contact avec la peau et les yeux."
-  ],
-  "instructions_fr":[
-    "1. Dissoudre 50g dans 10L d'eau.",
-    "2. Appliquer toutes les 2 semaines.",
-    "3. Conserver dans un endroit frais et sec."
-  ],
-  "micronutrients_fr":[
-    {
-      "nutrient":"Fer (Fe)",
-      "value":"0.10",
-      "unit":"%"
-    },
-    {
-      "nutrient":"Zinc (Zn)",
-      "value":"0.05",
-      "unit":"%"
-    },
-    {
-      "nutrient":"Manganèse (Mn)",
-      "value":"0.05",
-      "unit":"%"
-    }
-  ],
   "ingredients_fr":[
     {
       "nutrient":"Farine d'os",
-      "value":"5",
+      "value":5,
       "unit":"%"
     },
     {
       "nutrient":"Extrait d'algues",
-      "value":"3",
+      "value":3,
       "unit":"%"
     },
     {
       "nutrient":"Acide humique",
-      "value":"2",
+      "value":2,
       "unit":"%"
     },
     {
@@ -144,31 +117,20 @@
       "nutrient":"Perlite"
     }
   ],
-  "specifications_fr":[
-    {
-      "humidity":"10",
-      "ph":"6.5",
-      "solubility":"100"
-    }
+  "cautions_en": [
+    "Keep out of reach of children.",
+    "Store in a cool, dry place."
   ],
-  "first_aid_fr":[
-    "En cas de contact avec les yeux, rincer immédiatement à grande eau et consulter un médecin."
+  "cautions_fr": [
+    "Garder hors de la portée des enfants.",
+    "Conserver dans un endroit frais et sec."
   ],
-  "guaranteed_analysis":[
-    {
-      "nutrient":"Total Nitrogen (N)",
-      "value":"20",
-      "unit":"%"
-    },
-    {
-      "nutrient":"Available Phosphate (P2O5)",
-      "value":"20",
-      "unit":"%"
-    },
-    {
-      "nutrient":"Soluble Potash (K2O)",
-      "value":"20",
-      "unit":"%"
-    }
+  "instructions_en": [
+    "Apply evenly across the field at a rate of 5 kg per hectare.",
+    "Water thoroughly after application."
+  ],
+  "instructions_fr": [
+    "Appliquer uniformément sur le champ à raison de 5 kg par hectare.",
+    "Arroser abondamment après l'application."
   ]
 }
diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index c64104f..a628a1c 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -1,6 +1,6 @@
 import re
 from typing import List, Optional
-from pydantic import BaseModel, Field, field_validator, model_validator
+from pydantic import BaseModel, Field, field_validator
 
 class npkError(ValueError):
     pass
@@ -40,6 +40,19 @@ def convert_value(cls, v):
         elif isinstance(v, (str)):
             return extract_first_number(v)
         return None
+    
+class GuaranteedAnalysis(BaseModel):
+    title: Optional[str] = None
+    nutrients: List[NutrientValue] = []
+
+    @field_validator(
+        "nutrients",
+        mode="before",
+    )
+    def replace_none_with_empty_list(cls, v):
+        if v is None:
+            v = []
+        return v
 
 class Specification(BaseModel):
     humidity: Optional[float] = Field(..., alias='humidity')
@@ -72,21 +85,14 @@ class FertilizerInspection(BaseModel):
     density: Optional[Value] = None
     volume: Optional[Value] = None
     npk: Optional[str] = Field(None)
-    guaranteed_analysis: List[NutrientValue] = []
-    warranty: Optional[str] = None
+    guaranteed_analysis_en: Optional[GuaranteedAnalysis] = None
+    guaranteed_analysis_fr: Optional[GuaranteedAnalysis] = None
     cautions_en: List[str] = None
-    instructions_en: List[str] = []
-    micronutrients_en: List[NutrientValue] = []
-    ingredients_en: List[NutrientValue] = []
-    specifications_en: List[Specification] = []
-    first_aid_en: List[str] = None
     cautions_fr: List[str] = None
+    instructions_en: List[str] = []
     instructions_fr: List[str] = []
-    micronutrients_fr: List[NutrientValue] = []
+    ingredients_en: List[NutrientValue] = []
     ingredients_fr: List[NutrientValue] = []
-    specifications_fr: List[Specification] = []
-    first_aid_fr: List[str] = None
-
     
     @field_validator('npk', mode='before')
     def validate_npk(cls, v):
@@ -96,19 +102,19 @@ def validate_npk(cls, v):
                 return None
         return v
 
-    @model_validator(mode='before')
-    def replace_none_with_empty_list(cls, values):
-        fields_to_check = [
-            'cautions_en', 'first_aid_en', 'cautions_fr', 'first_aid_fr',
-            'instructions_en', 'micronutrients_en', 'ingredients_en',
-            'specifications_en', 'instructions_fr',
-            'micronutrients_fr', 'ingredients_fr',
-            'specifications_fr', 'guaranteed_analysis'
-        ]
-        for field in fields_to_check:
-            if values.get(field) is None:
-                values[field] = []
-        return values
+    @field_validator(
+        "cautions_en",
+        "cautions_fr",
+        "instructions_en",
+        "instructions_fr",
+        "weight",
+        mode="before",
+    )
+    def replace_none_with_empty_list(cls, v):
+        if v is None:
+            v = []
+        return v
+
 
     class Config:
         populate_by_name = True
diff --git a/tests/test_form.py b/tests/test_form.py
deleted file mode 100644
index d3e7a44..0000000
--- a/tests/test_form.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import unittest
-from pydantic import ValidationError
-from pipeline import FertilizerInspection
-
-class TestFertiliserForm(unittest.TestCase):
-    def test_valid_fertiliser_form(self):
-        data = {
-            "company_name": "ABC Company",
-            "company_address": "123 Main St",
-            "fertiliser_name": "Super Fertiliser",
-            "npk": "10-5-5",
-            "instructions_en": ["Use as directed"],
-            "micronutrients_en": [{"nutrient": "Iron", "value": 2.0, "unit": "%"}],
-            "specifications_en": [{"humidity": 23.0, "ph": 7.0, "solubility": 4.0}],
-            "guaranteed_analysis": [{"nutrient": "Nitrogen", "value": 10.0, "unit": "%"}]
-        }
-
-        try:
-            form = FertilizerInspection(**data)
-        except ValidationError as e:
-            self.fail(f"Validation error: {e}")
-
-        raw_form = form.model_dump()
-
-        # Check if values match
-        for key, expected_value in data.items():
-            value = raw_form[key]
-            self.assertEqual(expected_value, value, f"Value for key '{key}' does not match. Expected '{expected_value}', got '{value}'")
-
-            
-if __name__ == '__main__':
-    unittest.main()
diff --git a/tests/test_gpt.py b/tests/test_gpt.py
index 53ac171..20f4961 100644
--- a/tests/test_gpt.py
+++ b/tests/test_gpt.py
@@ -39,6 +39,10 @@ def setUp(self):
         Total Nitrogen (N) 20%
         Available Phosphate (P2O5) 20%
         Soluble Potash (K2O) 20%
+        Analyse Garantie.
+        Azote total (N) 20%
+        Phosphate assimilable (P2O5) 20%
+        Potasse soluble (K2O) 20%
         Micronutrients:
         Iron (Fe) 0.10%
         Zinc (Zn) 0.05%
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index e0a671c..50d6591 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -48,13 +48,13 @@ def tearDownClass(cls):
 
     def test_analyze(self):
         # Run the analyze function
-        form = analyze(self.label_storage, self.ocr, self.gpt, log_dir_path=self.log_dir_path)
+        inspection = analyze(self.label_storage, self.ocr, self.gpt, log_dir_path=self.log_dir_path)
         
         # Perform assertions
-        self.assertIsInstance(form, FertilizerInspection)
-        self.assertIn(Value(value='25', unit='kg'), form.weight)
-        self.assertGreater(levenshtein_similarity(form.company_name, "TerraLink"), 0.95)
-        self.assertGreater(levenshtein_similarity(form.npk, "10-52-0"), 0.90)
+        self.assertIsInstance(inspection, FertilizerInspection, inspection)
+        self.assertIn(Value(value='25', unit='kg'), inspection.weight, inspection)
+        self.assertGreater(levenshtein_similarity(inspection.manufacturer_name, "TerraLink"), 0.95, inspection)
+        self.assertGreater(levenshtein_similarity(inspection.npk, "10-52-0"), 0.90, inspection)
 
         # Ensure logs are created and then deleted
         now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

From 44c2f6c528b0040fd6b342cbb62ee42bedb13660 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Tue, 27 Aug 2024 04:18:43 -0400
Subject: [PATCH 07/17] feat: setup devcontainer + implement basic performance
 testing

This commit adds a basic devcontainer configuration and implements a simple naive framework for performance testing.
---
 pipeline/inspection.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index a628a1c..4435569 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -117,4 +117,4 @@ def replace_none_with_empty_list(cls, v):
 
 
     class Config:
-        populate_by_name = True
+        populate_by_name = True
\ No newline at end of file

From 981d23e712fdeed8e37578dceab2c56c0cb12fa2 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Thu, 3 Oct 2024 06:10:39 +0000
Subject: [PATCH 08/17] bugfix: fixed most bugs in performance_assessment.py +
 partial work done on unit tests

---
 ...mance_test.py => performance_assessment.py | 108 +++---
 scripts/__init__.py                           |   0
 .../end_to_end_pipeline.py                    |   0
 tests/test_performance_assessment.py          | 332 ++++++++++++++++++
 4 files changed, 400 insertions(+), 40 deletions(-)
 rename performance_test.py => performance_assessment.py (63%)
 create mode 100644 scripts/__init__.py
 rename end_to_end_pipeline.py => scripts/end_to_end_pipeline.py (100%)
 create mode 100644 tests/test_performance_assessment.py

diff --git a/performance_test.py b/performance_assessment.py
similarity index 63%
rename from performance_test.py
rename to performance_assessment.py
index e6023aa..b675faa 100644
--- a/performance_test.py
+++ b/performance_assessment.py
@@ -4,10 +4,10 @@
 import shutil
 import datetime
 import csv
-from typing import Dict, List, Union
+import pydantic
 
 from dotenv import load_dotenv
-from pipeline import analyze, LabelStorage, OCR, GPT
+from pipeline import analyze, LabelStorage, OCR, GPT, FertilizerInspection
 from tests import levenshtein_similarity
 
 ACCURACY_THRESHOLD = 80.0
@@ -28,13 +28,19 @@ def validate_environment_variables() -> None:
 def classify_test_result(score: float) -> str:
     """Classify test results as Pass or Fail based on accuracy score."""
     return "Pass" if score >= ACCURACY_THRESHOLD else "Fail"
-
-def load_json_file(file_path: str) -> Dict:
-    """Load and return JSON content from a file."""
+    
+def load_and_validate_json_inspection_file(file_path: str) -> dict:
+    """Load JSON content from a file and validate it against the schema."""
     with open(file_path, 'r') as file:
-        return json.load(file)
-
-def extract_leaf_fields(data: Union[dict, list], parent_key: str = '') -> Dict[str, Union[str, int, float, bool, None]]:
+        data = json.load(file)
+    try:
+        # Validate against the current schema
+        FertilizerInspection.model_validate(data, strict=True)
+    except pydantic.ValidationError as e:
+        print(f"Warning: Validation error in {file_path}.: This inspection JSON does not conform to the current inspection schema.\n")
+    return data
+
+def extract_leaf_fields(data: dict| list, parent_key: str = '') -> dict[str, str | int | float | bool | None]:
     """Extract all leaf fields from nested dictionaries and lists."""
     leaves = {}
 
@@ -47,24 +53,29 @@ def extract_leaf_fields(data: Union[dict, list], parent_key: str = '') -> Dict[s
                 leaves[new_key] = value
     elif isinstance(data, list):
         for index, item in enumerate(data):
-            list_key = f"{parent_key}[{index}]"
-            leaves.update(extract_leaf_fields(item, list_key))
+            list_key = f"{parent_key}[{index}]" if parent_key else f"[{index}]"
+            if isinstance(item, (dict, list)):
+                leaves.update(extract_leaf_fields(item, list_key))
+            else:
+                leaves[list_key] = item
 
     return leaves
 
 class TestCase:
-    def __init__(self, image_paths: List[str], expected_json_path: str):
+    def __init__(self, image_paths: list[str], expected_json_path: str):
         """Initialize a test case with image paths and the expected JSON output path."""
-        self.original_image_paths = image_paths
-        self.image_paths = self._create_image_copies(image_paths)
-        self.expected_json_path = expected_json_path
-        self.actual_json_path = self._generate_output_path()
-        self.results: Dict[str, float] = {}
-        self.label_storage = self._initialize_label_storage()
-        self.ocr = self._initialize_ocr()
-        self.gpt = self._initialize_gpt()
-
-    def _create_image_copies(self, image_paths: List[str]) -> List[str]:
+        self.original_image_paths : list[str] = image_paths
+        self.image_paths: list[str] = self._create_image_copies(image_paths) # because the pipeline automatically deletes images when processing them
+        self.expected_json_path: str = expected_json_path
+        self.expected_fields: dict[str, str | int | float | bool | None] = {}
+        self.actual_json_path : str = self._generate_output_path()
+        self.actual_fields: dict[str, str | int | float | bool | None] = {}
+        self.results: dict[str, float] = {}
+        self.label_storage: LabelStorage = self._initialize_label_storage()
+        self.ocr: OCR = self._initialize_ocr()
+        self.gpt: GPT = self._initialize_gpt()
+
+    def _create_image_copies(self, image_paths: list[str]) -> list[str]:
         """Create copies of the input images to prevent deletion of original files."""
         return [self._copy_image(path) for path in image_paths]
 
@@ -77,9 +88,7 @@ def _copy_image(self, image_path: str) -> str:
 
     def _generate_output_path(self) -> str:
         """Generate a timestamped path for the actual output JSON."""
-        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-        os.makedirs("test_outputs", exist_ok=True)
-        return os.path.join("test_outputs", f'actual_output_{timestamp}.json')
+        return self.expected_json_path.replace("expected", "actual")
 
     def _initialize_label_storage(self) -> LabelStorage:
         """Initialize and populate LabelStorage with image paths."""
@@ -105,6 +114,10 @@ def run_tests(self) -> None:
         """Run performance and accuracy tests for the pipeline."""
         self._run_performance_test()
         self._run_accuracy_test()
+        
+        # clean up the actual output .json files
+        if os.path.exists(self.actual_json_path):
+            os.remove(self.actual_json_path)
 
     def _run_performance_test(self) -> None:
         """Measure the time taken to run the pipeline analysis."""
@@ -118,18 +131,18 @@ def _run_accuracy_test(self) -> None:
         """Calculate and store the accuracy of the pipeline's output."""
         self.results['accuracy'] = self._calculate_levenshtein_accuracy()
 
-    def _calculate_levenshtein_accuracy(self) -> Dict[str, float]:
+    def _calculate_levenshtein_accuracy(self) -> dict[str, float]:
         """Calculate Levenshtein accuracy per field between expected and actual output."""
-        expected_fields = extract_leaf_fields(load_json_file(self.expected_json_path))
-        actual_fields = extract_leaf_fields(load_json_file(self.actual_json_path))
+        self.expected_fields = extract_leaf_fields(load_and_validate_json_inspection_file(self.expected_json_path))
+        self.actual_fields = extract_leaf_fields(load_and_validate_json_inspection_file(self.actual_json_path))
 
         return {
-            field_name: levenshtein_similarity(str(field_value), str(actual_fields.get(field_name)))
-            for field_name, field_value in expected_fields.items()
+            field_name: levenshtein_similarity(str(field_value), str(self.actual_fields.get(field_name)))
+            for field_name, field_value in self.expected_fields.items()
         }
 
 class TestRunner:
-    def __init__(self, test_cases: List[TestCase]):
+    def __init__(self, test_cases: list[TestCase]):
         """Initialize the test runner with a list of test cases."""
         self.test_cases = test_cases
 
@@ -153,15 +166,13 @@ def _generate_report_path(self) -> str:
         os.makedirs("reports", exist_ok=True)
         return os.path.join("reports", f"test_results_{timestamp}.csv")
 
-    def _get_csv_header(self) -> List[str]:
+    def _get_csv_header(self) -> list[str]:
         """Return the CSV header row."""
         return ["Test Case", "Field Name", "Accuracy Score", "Expected Value", "Actual Value", "Pass/Fail", "Pipeline Speed (seconds)"]
 
     def _write_test_results(self, writer: csv.writer) -> None:
         """Write the test results for each test case to the CSV file."""
         for i, test_case in enumerate(self.test_cases, 1):
-            expected_fields = extract_leaf_fields(load_json_file(test_case.expected_json_path))
-            actual_fields = extract_leaf_fields(load_json_file(test_case.actual_json_path))
             performance = test_case.results['performance']
 
             for field_name, score in test_case.results['accuracy'].items():
@@ -169,22 +180,39 @@ def _write_test_results(self, writer: csv.writer) -> None:
                     f"{i}",
                     field_name,
                     f"{score:.2f}",
-                    expected_fields.get(field_name, ""),
-                    actual_fields.get(field_name, ""),
+                    test_case.expected_fields.get(field_name, ""),
+                    test_case.actual_fields.get(field_name, ""),
                     classify_test_result(score),
                     f"{performance:.4f}"
                 ])
 
-def find_test_cases(labels_folder: str) -> List[TestCase]:
-    """Find and create test cases from the labels folder."""
+def find_test_cases(labels_folder: str) -> list[TestCase]:
+    """Find and create test cases from the labels folder in an ordered manner."""
     test_cases = []
-    for root, _, files in os.walk(labels_folder):
-        image_paths = [os.path.join(root, f) for f in files if f.lower().endswith((".png", ".jpg"))]
-        expected_json_path = os.path.join(root, "expected_output.json")
+    # List all entries in the labels_folder
+    label_entries = os.listdir(labels_folder)
+    # Filter out directories that start with 'label_'
+    label_dirs = [
+        os.path.join(labels_folder, d)
+        for d in label_entries
+        if os.path.isdir(os.path.join(labels_folder, d)) and d.startswith("label_")
+    ]
+    # Sort the label directories
+    label_dirs.sort()
+    # Process each label directory
+    for label_dir in label_dirs:
+        files = os.listdir(label_dir)
+        image_paths = [
+            os.path.join(label_dir, f)
+            for f in files
+            if f.lower().endswith((".png", ".jpg"))
+        ]
+        expected_json_path = os.path.join(label_dir, "expected_output.json")
         if image_paths and os.path.exists(expected_json_path):
             test_cases.append(TestCase(image_paths, expected_json_path))
     return test_cases
 
+
 def main():
     """Main function to run the performance tests."""
     load_dotenv()
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/end_to_end_pipeline.py b/scripts/end_to_end_pipeline.py
similarity index 100%
rename from end_to_end_pipeline.py
rename to scripts/end_to_end_pipeline.py
diff --git a/tests/test_performance_assessment.py b/tests/test_performance_assessment.py
new file mode 100644
index 0000000..7a66723
--- /dev/null
+++ b/tests/test_performance_assessment.py
@@ -0,0 +1,332 @@
+# test_script.py
+
+import unittest
+import os
+import json
+from unittest.mock import patch, mock_open
+import pydantic
+from typing import Optional
+from pydantic import BaseModel, field_validator
+
+from performance_assessment import (
+    validate_environment_variables,
+    classify_test_result,
+    load_and_validate_json_inspection_file,
+    extract_leaf_fields,
+    TestCase,
+    TestRunner,
+    find_test_cases
+)
+
+from pipeline.inspection import FertilizerInspection, extract_first_number
+
+class TestValidateEnvironmentVariables(unittest.TestCase):
+    @patch.dict(os.environ, {
+        "AZURE_API_ENDPOINT": "endpoint",
+        "AZURE_API_KEY": "key",
+        "AZURE_OPENAI_ENDPOINT": "endpoint",
+        "AZURE_OPENAI_KEY": "key",
+        "AZURE_OPENAI_DEPLOYMENT": "deployment"
+    })
+    def test_validate_environment_variables_all_set(self):
+        try:
+            validate_environment_variables()
+        except EnvironmentError:
+            self.fail("validate_environment_variables() raised EnvironmentError unexpectedly!")
+
+    @patch.dict(os.environ, {}, clear=True)
+    def test_validate_environment_variables_missing(self):
+        with self.assertRaises(EnvironmentError) as context:
+            validate_environment_variables()
+        self.assertIn("Missing required environment variables", str(context.exception))
+
+    @patch.dict(os.environ, {
+        "AZURE_API_ENDPOINT": "endpoint",
+        "AZURE_API_KEY": "key",
+        "AZURE_OPENAI_ENDPOINT": "endpoint"
+    }, clear=True)
+    def test_validate_some_environment_variables_missing(self):
+        with self.assertRaises(EnvironmentError) as context:
+            validate_environment_variables()
+        self.assertIn("Missing required environment variables", str(context.exception))
+        self.assertIn("AZURE_OPENAI_KEY", str(context.exception))
+        self.assertIn("AZURE_OPENAI_DEPLOYMENT", str(context.exception))
+
+
+class TestClassifyTestResult(unittest.TestCase):
+    @patch('performance_assessment.ACCURACY_THRESHOLD', 80.0)
+    def test_classify_test_result_pass(self):
+        self.assertEqual(classify_test_result(80.0), "Pass")
+        self.assertEqual(classify_test_result(90.0), "Pass")
+        self.assertEqual(classify_test_result(100.0), "Pass")
+
+    @patch('performance_assessment.ACCURACY_THRESHOLD', 80.0)
+    def test_classify_test_result_fail(self):
+        self.assertEqual(classify_test_result(79.9), "Fail")
+        self.assertEqual(classify_test_result(50.0), "Fail")
+        self.assertEqual(classify_test_result(0.0), "Fail")
+        self.assertEqual(classify_test_result(-10.0), "Fail")
+
+
+"""
+class MockNutrientValue(BaseModel):
+    nutrient: str
+    value: Optional[float] = None
+    unit: Optional[str] = None
+    
+    @field_validator('value', mode='before', check_fields=False)
+    def convert_value(cls, v):
+        if isinstance(v, bool):
+            return None
+        elif isinstance(v, (int, float)):
+            return str(v)
+        elif isinstance(v, (str)):
+            return extract_first_number(v)
+        return None
+
+class MockGuaranteedAnalysis(BaseModel):
+    title: Optional[str] = None
+    nutrients: list[MockNutrientValue] = []
+
+    @field_validator(
+        "nutrients",
+        mode="before",
+    )
+    def replace_none_with_empty_list(cls, v):
+        if v is None:
+            v = []
+        return v
+    
+class MockFertilizerInspection(FertilizerInspection):
+    company_name: Optional[str] = None
+    guaranteed_analysis: Optional[MockGuaranteedAnalysis] = None
+
+
+class TestLoadAndValidateJsonInspectionFile(unittest.TestCase):    
+
+    def test_load_json_inspection_file_valid(self):
+        test_data = {'key': 'value'}
+        json_content = json.dumps(test_data)
+        
+        # Use mock_open to simulate file operations
+        with patch('builtins.open', mock_open(read_data=json_content)) as mocked_file:
+            result = load_and_validate_json_inspection_file('dummy_path.json')
+            self.assertEqual(result, test_data)
+            mocked_file.assert_called_once_with('dummy_path.json', 'r')
+
+    def test_load_json_inspection_file_invalid_json(self):
+        invalid_json_content = '{"key": "value"'  # Missing closing brace
+        
+        with patch('builtins.open', mock_open(read_data=invalid_json_content)):
+            with self.assertRaises(json.JSONDecodeError):
+                load_and_validate_json_inspection_file('dummy_path.json')
+
+    def test_load_json_inspection_file_file_not_found(self):
+        # Simulate FileNotFoundError when attempting to open the file
+        with patch('builtins.open', side_effect=FileNotFoundError):
+            with self.assertRaises(FileNotFoundError):
+                load_and_validate_json_inspection_file('nonexistent_file.json')
+
+    def test_load_json_inspection_file_permission_error(self):
+        # Simulate PermissionError when attempting to open the file
+        with patch('builtins.open', side_effect=PermissionError):
+            with self.assertRaises(PermissionError):
+                load_and_validate_json_inspection_file('protected_file.json')
+
+    @patch('pipeline.inspection.FertilizerInspection', MockFertilizerInspection)
+    def test_load_json_inspection_file_validation_is_valid(self):
+        mock_data = {
+            "company_name": "Nature's aid",
+            "guaranteed_analysis": {
+                "title": "Analyse Garantie",
+                "nutrients": [
+                    {
+                        "nutrient": "extraits d'algues (ascophylle noueuse)",
+                        "value": 8.5,
+                        "unit": "%"
+                    },
+                    {
+                        "nutrient": "acide humique",
+                        "value": 0.6,
+                        "unit": "%"
+                    }
+                ]
+            }
+        }
+        json_content = json.dumps(mock_data)
+
+        with patch('builtins.open', mock_open(read_data=json_content)) as mocked_file:
+            result = load_and_validate_json_inspection_file('dummy_path.json')
+            self.assertEqual(result.company_name, "Nature's aid")
+            self.assertEqual(result.guaranteed_analysis.title, "Analyse Garantie")
+            self.assertEqual(result.guaranteed_analysis.nutrients[0].nutrient, "extraits d'algues (ascophylle noueuse)")
+            self.assertEqual(result.guaranteed_analysis.nutrients[0].value, 8.5)
+            self.assertEqual(result.guaranteed_analysis.nutrients[0].unit, "%")
+            self.assertEqual(result.guaranteed_analysis.nutrients[1].nutrient, "acide humique")
+            self.assertEqual(result.guaranteed_analysis.nutrients[1].value, 0.6)
+            self.assertEqual(result.guaranteed_analysis.nutrients[1].unit, "%")
+
+            mocked_file.assert_called_once_with('dummy_path.json', 'r')
+
+    @patch('pipeline.inspection.FertilizerInspection', MockFertilizerInspection)
+    def test_load_json_inspection_file_validation_is_invalid(self):
+        mock_data = {
+            "company_name": "Nature's aid",
+            "guaranteed_analysis": [
+                {
+                    "nutrients": "extraits d'algues (ascophylle noueuse)",
+                    "value": 8.5,
+                    "unit": "%"
+                },
+                {
+                    "nutrient": "acide humique",
+                    "value": 0.6,
+                    "unit": "%"
+                }
+            ]
+        }
+
+        json_content = json.dumps(mock_data)
+
+        with patch('builtins.open', mock_open(read_data=json_content)):
+            self.assertRaises(pydantic.ValidationError, load_and_validate_json_inspection_file('dummy_path.json'))
+"""
+
+class TestExtractLeafFields(unittest.TestCase):
+    def test_extract_leaf_fields_simple_root_dict(self):
+        mock_input = {
+            "company_name": "Nature's aid",
+            "company_address": None,
+            "company_website": "http://www.SOIL-AID.com",
+            "company_phone_number": None,
+            "manufacturer_name": "Diamond Fertilizers Inc.",
+            "manufacturer_address": "PO Box 5508 stn Main Hight River, AB CANADA T1V 1M6",
+            "manufacturer_website": None,
+        }
+        
+        expected_output = {
+            "company_name": "Nature's aid",
+            "company_address": None,
+            "company_website": "http://www.SOIL-AID.com",
+            "company_phone_number": None,
+            "manufacturer_name": "Diamond Fertilizers Inc.",
+            "manufacturer_address": "PO Box 5508 stn Main Hight River, AB CANADA T1V 1M6",
+            "manufacturer_website": None,
+        }  
+
+        actual_output = extract_leaf_fields(mock_input)
+        self.assertEqual(actual_output, expected_output)
+
+    def test_extract_leaf_fields_simple_child_dict(self):
+        mock_input = {
+            "density": {
+                "value": None,
+                "unit": None
+            },
+            "volume": {
+                "value": 10,
+                "unit": "liter"
+            },
+        }
+
+        expected_output = {
+            "density.value": None,
+            "density.unit": None,
+            "volume.value": 10,
+            "volume.unit": "liter"
+        }
+
+        actual_output = extract_leaf_fields(mock_input)
+        self.assertEqual(actual_output, expected_output)
+
+    def test_extract_leaf_fields_simple_root_list(self):
+        mock_input = [
+            "step 1: prepare the soil",
+            "step 2: plant the seeds",
+            "step 3: water the plants",
+        ]
+
+        expected_output = {
+            "[0]": "step 1: prepare the soil",
+            "[1]": "step 2: plant the seeds",
+            "[2]": "step 3: water the plants",
+        }
+
+        actual_output = extract_leaf_fields(mock_input)
+        self.assertEqual(actual_output, expected_output)
+
+    def test_extract_leaf_fields_simple_child_list(self):
+        mock_input = {
+            "steps": [
+                "step 1: prepare the soil",
+                "step 2: plant the seeds",
+                "step 3: water the plants",
+            ],
+            "materials": [
+                "soil",
+                "seeds",
+                "water",
+            ]
+        }
+
+        expected_output = {
+            "steps[0]": "step 1: prepare the soil",
+            "steps[1]": "step 2: plant the seeds",
+            "steps[2]": "step 3: water the plants",
+            "materials[0]": "soil",
+            "materials[1]": "seeds",
+            "materials[2]": "water",
+        }
+
+        actual_output = extract_leaf_fields(mock_input)
+        self.assertEqual(actual_output, expected_output)
+
+    def test_extract_leaf_fields_mixed_structure(self):
+        mock_input = {
+            "guaranteed_analysis_fr": {
+                "title": "Analyse Garantie",
+                "nutrients": [
+                    {
+                    "nutrient": "extraits d'algues (ascophylle noueuse)",
+                    "value": 8.5,
+                    "unit": "%"
+                    },
+                    {
+                    "nutrient": "acide humique",
+                    "value": 0.6,
+                    "unit": "%"
+                    }
+                ]
+            },
+            "cautions_en": None,
+            "cautions_fr": None,
+            "instructions_en": [
+                "step 1: prepare the soil",
+                "step 2: plant the seeds",
+                "step 3: water the plants",
+            ],
+        }
+
+        expected_output = {
+            "guaranteed_analysis_fr.title": "Analyse Garantie",
+            "guaranteed_analysis_fr.nutrients[0].nutrient": "extraits d'algues (ascophylle noueuse)",
+            "guaranteed_analysis_fr.nutrients[0].value": 8.5,
+            "guaranteed_analysis_fr.nutrients[0].unit": "%",
+            "guaranteed_analysis_fr.nutrients[1].nutrient": "acide humique",
+            "guaranteed_analysis_fr.nutrients[1].value": 0.6,
+            "guaranteed_analysis_fr.nutrients[1].unit": "%",
+            "cautions_en": None,
+            "cautions_fr": None,
+            "instructions_en[0]": "step 1: prepare the soil",
+            "instructions_en[1]": "step 2: plant the seeds",
+            "instructions_en[2]": "step 3: water the plants",
+        }
+
+        actual_output = extract_leaf_fields(mock_input)
+        self.assertEqual(actual_output, expected_output)
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+

From 3b90655b88f9c6bfdce1ea0166ff6b402ef045e6 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Thu, 3 Oct 2024 07:19:25 +0000
Subject: [PATCH 09/17] Thinks start breaking if I use model_validator instead
 of field_validator and I have no idea why...

---
 pipeline/inspection.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index 4435569..f67a928 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -1,6 +1,6 @@
 import re
 from typing import List, Optional
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, field_validator, model_validator
 
 class npkError(ValueError):
     pass
@@ -102,18 +102,19 @@ def validate_npk(cls, v):
                 return None
         return v
 
-    @field_validator(
-        "cautions_en",
-        "cautions_fr",
-        "instructions_en",
-        "instructions_fr",
-        "weight",
-        mode="before",
-    )
-    def replace_none_with_empty_list(cls, v):
-        if v is None:
-            v = []
-        return v
+    @model_validator(mode='before')
+    def replace_none_with_empty_list(cls, values):
+        fields_to_check = [
+            'cautions_en', 'first_aid_en', 'cautions_fr', 'first_aid_fr',
+            'instructions_en', 'micronutrients_en', 'ingredients_en',
+            'specifications_en', 'instructions_fr',
+            'micronutrients_fr', 'ingredients_fr',
+            'specifications_fr', 'guaranteed_analysis'
+        ]
+        for field in fields_to_check:
+            if values.get(field) is None:
+                values[field] = []
+        return values
 
 
     class Config:

From 19d5166607f838231ba1be1c96977374c07f451b Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Mon, 7 Oct 2024 15:13:09 +0000
Subject: [PATCH 10/17] refactor: simplify performance_assessment.py by
 removing unnecessary classes

Simplified the script by replacing classes with functions to reduce complexity and improve readability. The script now:

1. Loads environment variables.
2. Loads test cases (images and expected outputs) from the `test_data` folder.
3. Iterates through the test cases to run the pipeline and assess performance.
4. Compiles the results into a CSV file.

Consolidated trivial functions into larger ones with single responsibilities to make the code more maintainable. Updated type hints to use the latest built-in types.
---
 performance_assessment.py | 337 +++++++++++++++++---------------------
 1 file changed, 149 insertions(+), 188 deletions(-)

diff --git a/performance_assessment.py b/performance_assessment.py
index b675faa..ce75bd2 100644
--- a/performance_assessment.py
+++ b/performance_assessment.py
@@ -1,48 +1,23 @@
+# Requires Python 3.10 or newer
+
 import os
 import time
 import json
 import shutil
 import datetime
 import csv
-import pydantic
-
+import tempfile
 from dotenv import load_dotenv
-from pipeline import analyze, LabelStorage, OCR, GPT, FertilizerInspection
+from pipeline import analyze, LabelStorage, OCR, GPT
 from tests import levenshtein_similarity
 
 ACCURACY_THRESHOLD = 80.0
 
-def validate_environment_variables() -> None:
-    """Ensure all required environment variables are set."""
-    required_vars = [
-        "AZURE_API_ENDPOINT",
-        "AZURE_API_KEY",
-        "AZURE_OPENAI_ENDPOINT",
-        "AZURE_OPENAI_KEY",
-        "AZURE_OPENAI_DEPLOYMENT",
-    ]
-    missing_vars = [var for var in required_vars if not os.getenv(var)]
-    if missing_vars:
-        raise EnvironmentError(f"Missing required environment variables: {', '.join(missing_vars)}")
-
-def classify_test_result(score: float) -> str:
-    """Classify test results as Pass or Fail based on accuracy score."""
-    return "Pass" if score >= ACCURACY_THRESHOLD else "Fail"
-    
-def load_and_validate_json_inspection_file(file_path: str) -> dict:
-    """Load JSON content from a file and validate it against the schema."""
-    with open(file_path, 'r') as file:
-        data = json.load(file)
-    try:
-        # Validate against the current schema
-        FertilizerInspection.model_validate(data, strict=True)
-    except pydantic.ValidationError as e:
-        print(f"Warning: Validation error in {file_path}.: This inspection JSON does not conform to the current inspection schema.\n")
-    return data
-
-def extract_leaf_fields(data: dict| list, parent_key: str = '') -> dict[str, str | int | float | bool | None]:
-    """Extract all leaf fields from nested dictionaries and lists."""
-    leaves = {}
+
+def extract_leaf_fields(
+    data: dict | list, parent_key: str = ''
+) -> dict[str, str | int | float | bool | None]:
+    leaves: dict[str, str | int | float | bool | None] = {}
 
     if isinstance(data, dict):
         for key, value in data.items():
@@ -53,175 +28,161 @@ def extract_leaf_fields(data: dict| list, parent_key: str = '') -> dict[str, str
                 leaves[new_key] = value
     elif isinstance(data, list):
         for index, item in enumerate(data):
-            list_key = f"{parent_key}[{index}]" if parent_key else f"[{index}]"
+            new_key = f"{parent_key}[{index}]" if parent_key else f"[{index}]"
             if isinstance(item, (dict, list)):
-                leaves.update(extract_leaf_fields(item, list_key))
+                leaves.update(extract_leaf_fields(item, new_key))
             else:
-                leaves[list_key] = item
+                leaves[new_key] = item
 
     return leaves
 
-class TestCase:
-    def __init__(self, image_paths: list[str], expected_json_path: str):
-        """Initialize a test case with image paths and the expected JSON output path."""
-        self.original_image_paths : list[str] = image_paths
-        self.image_paths: list[str] = self._create_image_copies(image_paths) # because the pipeline automatically deletes images when processing them
-        self.expected_json_path: str = expected_json_path
-        self.expected_fields: dict[str, str | int | float | bool | None] = {}
-        self.actual_json_path : str = self._generate_output_path()
-        self.actual_fields: dict[str, str | int | float | bool | None] = {}
-        self.results: dict[str, float] = {}
-        self.label_storage: LabelStorage = self._initialize_label_storage()
-        self.ocr: OCR = self._initialize_ocr()
-        self.gpt: GPT = self._initialize_gpt()
-
-    def _create_image_copies(self, image_paths: list[str]) -> list[str]:
-        """Create copies of the input images to prevent deletion of original files."""
-        return [self._copy_image(path) for path in image_paths]
-
-    def _copy_image(self, image_path: str) -> str:
-        """Create a copy of a single image file."""
-        base, ext = os.path.splitext(image_path)
-        copy_path = f"{base}_copy{ext}"
-        shutil.copy2(image_path, copy_path)
-        return copy_path
-
-    def _generate_output_path(self) -> str:
-        """Generate a timestamped path for the actual output JSON."""
-        return self.expected_json_path.replace("expected", "actual")
-
-    def _initialize_label_storage(self) -> LabelStorage:
-        """Initialize and populate LabelStorage with image paths."""
-        storage = LabelStorage()
-        for image_path in self.image_paths:
-            storage.add_image(image_path)
-        return storage
-
-    def _initialize_ocr(self) -> OCR:
-        """Initialize OCR with API credentials."""
-        return OCR(os.getenv("AZURE_API_ENDPOINT"), os.getenv("AZURE_API_KEY"))
-
-    def _initialize_gpt(self) -> GPT:
-        """Initialize GPT with API credentials."""
-        return GPT(os.getenv("AZURE_OPENAI_ENDPOINT"), os.getenv("AZURE_OPENAI_KEY"), os.getenv("AZURE_OPENAI_DEPLOYMENT"))
-
-    def save_json_output(self, output: str) -> None:
-        """Save the actual output JSON to a file."""
-        with open(self.actual_json_path, 'w') as file:
-            file.write(output)
-
-    def run_tests(self) -> None:
-        """Run performance and accuracy tests for the pipeline."""
-        self._run_performance_test()
-        self._run_accuracy_test()
-        
-        # clean up the actual output .json files
-        if os.path.exists(self.actual_json_path):
-            os.remove(self.actual_json_path)
-
-    def _run_performance_test(self) -> None:
-        """Measure the time taken to run the pipeline analysis."""
-        start_time = time.time()
-        actual_output = analyze(self.label_storage, self.ocr, self.gpt)
-        end_time = time.time()
-        self.results['performance'] = end_time - start_time
-        self.save_json_output(actual_output.model_dump_json(indent=2))
-
-    def _run_accuracy_test(self) -> None:
-        """Calculate and store the accuracy of the pipeline's output."""
-        self.results['accuracy'] = self._calculate_levenshtein_accuracy()
-
-    def _calculate_levenshtein_accuracy(self) -> dict[str, float]:
-        """Calculate Levenshtein accuracy per field between expected and actual output."""
-        self.expected_fields = extract_leaf_fields(load_and_validate_json_inspection_file(self.expected_json_path))
-        self.actual_fields = extract_leaf_fields(load_and_validate_json_inspection_file(self.actual_json_path))
-
-        return {
-            field_name: levenshtein_similarity(str(field_value), str(self.actual_fields.get(field_name)))
-            for field_name, field_value in self.expected_fields.items()
-        }
 
-class TestRunner:
-    def __init__(self, test_cases: list[TestCase]):
-        """Initialize the test runner with a list of test cases."""
-        self.test_cases = test_cases
-
-    def run_tests(self) -> None:
-        """Run all test cases."""
-        for test_case in self.test_cases:
-            test_case.run_tests()
-
-    def generate_csv_report(self) -> None:
-        """Generate a CSV report of the test results and save it as a timestamped .csv file."""
-        report_path = self._generate_report_path()
-        with open(report_path, mode='w', newline='') as file:
-            writer = csv.writer(file)
-            writer.writerow(self._get_csv_header())
-            self._write_test_results(writer)
-        print(f"CSV report generated and saved to: {report_path}")
-
-    def _generate_report_path(self) -> str:
-        """Generate a timestamped path for the CSV report."""
-        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M")
-        os.makedirs("reports", exist_ok=True)
-        return os.path.join("reports", f"test_results_{timestamp}.csv")
-
-    def _get_csv_header(self) -> list[str]:
-        """Return the CSV header row."""
-        return ["Test Case", "Field Name", "Accuracy Score", "Expected Value", "Actual Value", "Pass/Fail", "Pipeline Speed (seconds)"]
-
-    def _write_test_results(self, writer: csv.writer) -> None:
-        """Write the test results for each test case to the CSV file."""
-        for i, test_case in enumerate(self.test_cases, 1):
-            performance = test_case.results['performance']
-
-            for field_name, score in test_case.results['accuracy'].items():
-                writer.writerow([
-                    f"{i}",
-                    field_name,
-                    f"{score:.2f}",
-                    test_case.expected_fields.get(field_name, ""),
-                    test_case.actual_fields.get(field_name, ""),
-                    classify_test_result(score),
-                    f"{performance:.4f}"
-                ])
-
-def find_test_cases(labels_folder: str) -> list[TestCase]:
-    """Find and create test cases from the labels folder in an ordered manner."""
+def find_test_cases(labels_folder: str) -> list[tuple[list[str], str]]:
     test_cases = []
-    # List all entries in the labels_folder
-    label_entries = os.listdir(labels_folder)
-    # Filter out directories that start with 'label_'
-    label_dirs = [
-        os.path.join(labels_folder, d)
-        for d in label_entries
-        if os.path.isdir(os.path.join(labels_folder, d)) and d.startswith("label_")
-    ]
-    # Sort the label directories
-    label_dirs.sort()
-    # Process each label directory
-    for label_dir in label_dirs:
-        files = os.listdir(label_dir)
+    label_directories = sorted(
+        os.path.join(labels_folder, directory)
+        for directory in os.listdir(labels_folder)
+        if os.path.isdir(os.path.join(labels_folder, directory)) and directory.startswith("label_")
+    )
+    if len(label_directories) == 0:
+        raise FileNotFoundError(f"No label directories found in {labels_folder}")
+
+    for label_directory in label_directories:
+        files = os.listdir(label_directory)
         image_paths = [
-            os.path.join(label_dir, f)
-            for f in files
-            if f.lower().endswith((".png", ".jpg"))
+            os.path.join(label_directory, file)
+            for file in files
+            if file.lower().endswith((".png", ".jpg"))
         ]
-        expected_json_path = os.path.join(label_dir, "expected_output.json")
-        if image_paths and os.path.exists(expected_json_path):
-            test_cases.append(TestCase(image_paths, expected_json_path))
+        expected_json_path = os.path.join(label_directory, "expected_output.json")
+
+        if not image_paths:
+            raise FileNotFoundError(f"No image files found in {label_directory}")
+        if not os.path.exists(expected_json_path):
+            raise FileNotFoundError(f"Expected output JSON not found in {label_directory}")
+        test_cases.append((image_paths, expected_json_path))
+
     return test_cases
 
 
-def main():
-    """Main function to run the performance tests."""
+def calculate_accuracy(
+    expected_fields: dict[str, str],
+    actual_fields: dict[str, str]
+) -> dict[str, dict[str, str | float]]:
+    accuracy_results = {}
+    for field_name, expected_value in expected_fields.items():
+        actual_value = actual_fields.get(field_name, "FIELD_NOT_FOUND")
+        score = levenshtein_similarity(str(expected_value), str(actual_value))
+        pass_fail = "Pass" if score >= ACCURACY_THRESHOLD else "Fail"
+        accuracy_results[field_name] = {
+            'score': score,
+            'expected_value': expected_value,
+            'actual_value': actual_value,
+            'pass_fail': pass_fail,
+        }
+    return accuracy_results
+
+
+def run_test_case(
+    test_case_number: int, image_paths: list[str], expected_json_path: str
+) -> dict[str, any]:
+    # Copy images to temporary files to prevent deletion due to LabelStorage behavior
+    copied_image_paths = []
+    for image_path in image_paths:
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(image_path)[1])
+        shutil.copy2(image_path, temp_file.name)
+        copied_image_paths.append(temp_file.name)
+
+    # Initialize LabelStorage, OCR, GPT
+    storage = LabelStorage()
+    for image_path in copied_image_paths:
+        storage.add_image(image_path)
+
+    ocr = OCR(os.getenv("AZURE_API_ENDPOINT"), os.getenv("AZURE_API_KEY"))
+    gpt = GPT(
+        os.getenv("AZURE_OPENAI_ENDPOINT"),
+        os.getenv("AZURE_OPENAI_KEY"),
+        os.getenv("AZURE_OPENAI_DEPLOYMENT"),
+    )
+
+    # Run performance test
+    start_time = time.time()
+    actual_output = analyze(storage, ocr, gpt) # <-- the `analyse` function deletes the images it processes so we don't need to clean up our image copies
+    performance = time.time() - start_time
+
+    # Process actual output
+    actual_fields = extract_leaf_fields(json.loads(actual_output.model_dump_json()))
+
+    # Load expected output
+    with open(expected_json_path, 'r') as file:
+        expected_fields = extract_leaf_fields(json.load(file))
+
+    # Calculate accuracy
+    accuracy_results = calculate_accuracy(expected_fields, actual_fields)
+
+    # Return results
+    return {
+        'test_case_number': test_case_number,
+        'performance': performance,
+        'accuracy_results': accuracy_results,
+    }
+
+
+def generate_csv_report(results: list[dict[str, any]]) -> None:
+    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M")
+    os.makedirs("reports", exist_ok=True)
+    report_path = os.path.join("reports", f"test_results_{timestamp}.csv")
+
+    with open(report_path, mode='w', newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow([
+            "Test Case",
+            "Field Name",
+            "Pass/Fail",
+            "Accuracy Score",
+            "Pipeline Speed (seconds)",
+            "Expected Value",
+            "Actual Value",
+        ])
+
+        for result in results:
+            test_case_number = result['test_case_number']
+            performance = result['performance']
+            for field_name, data in result['accuracy_results'].items():
+                writer.writerow([
+                    test_case_number,
+                    field_name,
+                    data['pass_fail'],
+                    f"{data['score']:.2f}",
+                    f"{performance:.4f}",
+                    data['expected_value'],
+                    data['actual_value'],
+                ])
+    print(f"CSV report generated and saved to: {report_path}")
+
+
+def main() -> None:
     load_dotenv()
-    validate_environment_variables()
+
+    # Validate required environment variables
+    required_vars = [
+        "AZURE_API_ENDPOINT",
+        "AZURE_API_KEY",
+        "AZURE_OPENAI_ENDPOINT",
+        "AZURE_OPENAI_KEY",
+        "AZURE_OPENAI_DEPLOYMENT",
+    ]
+    missing_vars = [var for var in required_vars if not os.getenv(var)]
+    if missing_vars:
+        raise RuntimeError(f"Missing required environment variables: {', '.join(missing_vars)}")
 
     test_cases = find_test_cases("test_data/labels")
-    runner = TestRunner(test_cases)
-    runner.run_tests()
-    runner.generate_csv_report()
+    results = []
+    for idx, (image_paths, expected_json_path) in enumerate(test_cases, 1):
+        result = run_test_case(idx, image_paths, expected_json_path)
+        results.append(result)
+    generate_csv_report(results)
+
 
 if __name__ == "__main__":
     main()

From 19b15b89606d1f48c215e53430bbaf5b0b7b1af1 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Mon, 7 Oct 2024 22:55:50 +0000
Subject: [PATCH 11/17] feat: Add unit tests for `performance_assessment.py`
 and improve edge case handling for missing fields in `calculate_accuracy()`

---
 performance_assessment.py            |   7 +-
 tests/test_performance_assessment.py | 426 +++++++++++++++------------
 2 files changed, 241 insertions(+), 192 deletions(-)

diff --git a/performance_assessment.py b/performance_assessment.py
index ce75bd2..9b81c73 100644
--- a/performance_assessment.py
+++ b/performance_assessment.py
@@ -1,5 +1,3 @@
-# Requires Python 3.10 or newer
-
 import os
 import time
 import json
@@ -72,7 +70,10 @@ def calculate_accuracy(
     accuracy_results = {}
     for field_name, expected_value in expected_fields.items():
         actual_value = actual_fields.get(field_name, "FIELD_NOT_FOUND")
-        score = levenshtein_similarity(str(expected_value), str(actual_value))
+        if actual_value == "FIELD_NOT_FOUND":
+            score = 0.0
+        else:
+            score = levenshtein_similarity(str(expected_value), str(actual_value))
         pass_fail = "Pass" if score >= ACCURACY_THRESHOLD else "Fail"
         accuracy_results[field_name] = {
             'score': score,
diff --git a/tests/test_performance_assessment.py b/tests/test_performance_assessment.py
index 7a66723..e36583e 100644
--- a/tests/test_performance_assessment.py
+++ b/tests/test_performance_assessment.py
@@ -1,197 +1,19 @@
-# test_script.py
-
+import json
 import unittest
 import os
-import json
-from unittest.mock import patch, mock_open
-import pydantic
-from typing import Optional
-from pydantic import BaseModel, field_validator
-
+import shutil
+import csv
+import tempfile
+from unittest.mock import patch, MagicMock
 from performance_assessment import (
-    validate_environment_variables,
-    classify_test_result,
-    load_and_validate_json_inspection_file,
     extract_leaf_fields,
-    TestCase,
-    TestRunner,
-    find_test_cases
+    find_test_cases,
+    calculate_accuracy,
+    run_test_case,
+    generate_csv_report,
+    main
 )
 
-from pipeline.inspection import FertilizerInspection, extract_first_number
-
-class TestValidateEnvironmentVariables(unittest.TestCase):
-    @patch.dict(os.environ, {
-        "AZURE_API_ENDPOINT": "endpoint",
-        "AZURE_API_KEY": "key",
-        "AZURE_OPENAI_ENDPOINT": "endpoint",
-        "AZURE_OPENAI_KEY": "key",
-        "AZURE_OPENAI_DEPLOYMENT": "deployment"
-    })
-    def test_validate_environment_variables_all_set(self):
-        try:
-            validate_environment_variables()
-        except EnvironmentError:
-            self.fail("validate_environment_variables() raised EnvironmentError unexpectedly!")
-
-    @patch.dict(os.environ, {}, clear=True)
-    def test_validate_environment_variables_missing(self):
-        with self.assertRaises(EnvironmentError) as context:
-            validate_environment_variables()
-        self.assertIn("Missing required environment variables", str(context.exception))
-
-    @patch.dict(os.environ, {
-        "AZURE_API_ENDPOINT": "endpoint",
-        "AZURE_API_KEY": "key",
-        "AZURE_OPENAI_ENDPOINT": "endpoint"
-    }, clear=True)
-    def test_validate_some_environment_variables_missing(self):
-        with self.assertRaises(EnvironmentError) as context:
-            validate_environment_variables()
-        self.assertIn("Missing required environment variables", str(context.exception))
-        self.assertIn("AZURE_OPENAI_KEY", str(context.exception))
-        self.assertIn("AZURE_OPENAI_DEPLOYMENT", str(context.exception))
-
-
-class TestClassifyTestResult(unittest.TestCase):
-    @patch('performance_assessment.ACCURACY_THRESHOLD', 80.0)
-    def test_classify_test_result_pass(self):
-        self.assertEqual(classify_test_result(80.0), "Pass")
-        self.assertEqual(classify_test_result(90.0), "Pass")
-        self.assertEqual(classify_test_result(100.0), "Pass")
-
-    @patch('performance_assessment.ACCURACY_THRESHOLD', 80.0)
-    def test_classify_test_result_fail(self):
-        self.assertEqual(classify_test_result(79.9), "Fail")
-        self.assertEqual(classify_test_result(50.0), "Fail")
-        self.assertEqual(classify_test_result(0.0), "Fail")
-        self.assertEqual(classify_test_result(-10.0), "Fail")
-
-
-"""
-class MockNutrientValue(BaseModel):
-    nutrient: str
-    value: Optional[float] = None
-    unit: Optional[str] = None
-    
-    @field_validator('value', mode='before', check_fields=False)
-    def convert_value(cls, v):
-        if isinstance(v, bool):
-            return None
-        elif isinstance(v, (int, float)):
-            return str(v)
-        elif isinstance(v, (str)):
-            return extract_first_number(v)
-        return None
-
-class MockGuaranteedAnalysis(BaseModel):
-    title: Optional[str] = None
-    nutrients: list[MockNutrientValue] = []
-
-    @field_validator(
-        "nutrients",
-        mode="before",
-    )
-    def replace_none_with_empty_list(cls, v):
-        if v is None:
-            v = []
-        return v
-    
-class MockFertilizerInspection(FertilizerInspection):
-    company_name: Optional[str] = None
-    guaranteed_analysis: Optional[MockGuaranteedAnalysis] = None
-
-
-class TestLoadAndValidateJsonInspectionFile(unittest.TestCase):    
-
-    def test_load_json_inspection_file_valid(self):
-        test_data = {'key': 'value'}
-        json_content = json.dumps(test_data)
-        
-        # Use mock_open to simulate file operations
-        with patch('builtins.open', mock_open(read_data=json_content)) as mocked_file:
-            result = load_and_validate_json_inspection_file('dummy_path.json')
-            self.assertEqual(result, test_data)
-            mocked_file.assert_called_once_with('dummy_path.json', 'r')
-
-    def test_load_json_inspection_file_invalid_json(self):
-        invalid_json_content = '{"key": "value"'  # Missing closing brace
-        
-        with patch('builtins.open', mock_open(read_data=invalid_json_content)):
-            with self.assertRaises(json.JSONDecodeError):
-                load_and_validate_json_inspection_file('dummy_path.json')
-
-    def test_load_json_inspection_file_file_not_found(self):
-        # Simulate FileNotFoundError when attempting to open the file
-        with patch('builtins.open', side_effect=FileNotFoundError):
-            with self.assertRaises(FileNotFoundError):
-                load_and_validate_json_inspection_file('nonexistent_file.json')
-
-    def test_load_json_inspection_file_permission_error(self):
-        # Simulate PermissionError when attempting to open the file
-        with patch('builtins.open', side_effect=PermissionError):
-            with self.assertRaises(PermissionError):
-                load_and_validate_json_inspection_file('protected_file.json')
-
-    @patch('pipeline.inspection.FertilizerInspection', MockFertilizerInspection)
-    def test_load_json_inspection_file_validation_is_valid(self):
-        mock_data = {
-            "company_name": "Nature's aid",
-            "guaranteed_analysis": {
-                "title": "Analyse Garantie",
-                "nutrients": [
-                    {
-                        "nutrient": "extraits d'algues (ascophylle noueuse)",
-                        "value": 8.5,
-                        "unit": "%"
-                    },
-                    {
-                        "nutrient": "acide humique",
-                        "value": 0.6,
-                        "unit": "%"
-                    }
-                ]
-            }
-        }
-        json_content = json.dumps(mock_data)
-
-        with patch('builtins.open', mock_open(read_data=json_content)) as mocked_file:
-            result = load_and_validate_json_inspection_file('dummy_path.json')
-            self.assertEqual(result.company_name, "Nature's aid")
-            self.assertEqual(result.guaranteed_analysis.title, "Analyse Garantie")
-            self.assertEqual(result.guaranteed_analysis.nutrients[0].nutrient, "extraits d'algues (ascophylle noueuse)")
-            self.assertEqual(result.guaranteed_analysis.nutrients[0].value, 8.5)
-            self.assertEqual(result.guaranteed_analysis.nutrients[0].unit, "%")
-            self.assertEqual(result.guaranteed_analysis.nutrients[1].nutrient, "acide humique")
-            self.assertEqual(result.guaranteed_analysis.nutrients[1].value, 0.6)
-            self.assertEqual(result.guaranteed_analysis.nutrients[1].unit, "%")
-
-            mocked_file.assert_called_once_with('dummy_path.json', 'r')
-
-    @patch('pipeline.inspection.FertilizerInspection', MockFertilizerInspection)
-    def test_load_json_inspection_file_validation_is_invalid(self):
-        mock_data = {
-            "company_name": "Nature's aid",
-            "guaranteed_analysis": [
-                {
-                    "nutrients": "extraits d'algues (ascophylle noueuse)",
-                    "value": 8.5,
-                    "unit": "%"
-                },
-                {
-                    "nutrient": "acide humique",
-                    "value": 0.6,
-                    "unit": "%"
-                }
-            ]
-        }
-
-        json_content = json.dumps(mock_data)
-
-        with patch('builtins.open', mock_open(read_data=json_content)):
-            self.assertRaises(pydantic.ValidationError, load_and_validate_json_inspection_file('dummy_path.json'))
-"""
-
 class TestExtractLeafFields(unittest.TestCase):
     def test_extract_leaf_fields_simple_root_dict(self):
         mock_input = {
@@ -325,7 +147,233 @@ def test_extract_leaf_fields_mixed_structure(self):
         actual_output = extract_leaf_fields(mock_input)
         self.assertEqual(actual_output, expected_output)
 
-if __name__ == '__main__':
+class TestFindTestCases(unittest.TestCase):
+
+    def setUp(self):
+        self.test_dir = "test_labels_folder"
+        os.makedirs(self.test_dir, exist_ok=True)
+
+    def tearDown(self):
+        shutil.rmtree(self.test_dir)
+
+    def create_test_structure(self, structure):
+        for path, files in structure.items():
+            dir_path = os.path.join(self.test_dir, path)
+            os.makedirs(dir_path, exist_ok=True)
+            for file_name, content in files.items():
+                with open(os.path.join(dir_path, file_name), 'w') as f:
+                    f.write(content)
+
+    def test_valid_structure(self):
+        structure = {
+            "label_001": {
+                "image001.png": "",
+                "image002.jpg": "",
+                "expected_output.json": "{}"
+            },
+            "label_002": {
+                "image001.png": "",
+                "expected_output.json": "{}"
+            },
+            "label_003": {
+                "image001.png": "",
+                "image002.jpg": "",
+                "expected_output.json": "{}"
+            }
+        }
+        self.create_test_structure(structure)
+        expected = [
+            (
+                [
+                    os.path.join(self.test_dir, "label_001", "image001.png"),
+                    os.path.join(self.test_dir, "label_001", "image002.jpg")
+                ],
+                os.path.join(self.test_dir, "label_001", "expected_output.json")
+            ),
+            (
+                [
+                    os.path.join(self.test_dir, "label_002", "image001.png")
+                ],
+                os.path.join(self.test_dir, "label_002", "expected_output.json")
+            ),
+            (
+                [
+                    os.path.join(self.test_dir, "label_003", "image001.png"),
+                    os.path.join(self.test_dir, "label_003", "image002.jpg")
+                ],
+                os.path.join(self.test_dir, "label_003", "expected_output.json")
+            )
+        ]
+        result = find_test_cases(self.test_dir)
+        self.assertEqual(result, expected)
+
+    def test_missing_expected_output(self):
+        structure = {
+            "label_001": {
+                "image001.png": "",
+                "image002.jpg": ""
+            }
+        }
+        self.create_test_structure(structure)
+        with self.assertRaises(FileNotFoundError):
+            find_test_cases(self.test_dir)
+
+    def test_no_image_files(self):
+        structure = {
+            "label_001": {
+                "expected_output.json": "{}"
+            }
+        }
+        self.create_test_structure(structure)
+        with self.assertRaises(FileNotFoundError):
+            find_test_cases(self.test_dir)
+
+    def test_empty_labels_folder(self):
+        with self.assertRaises(FileNotFoundError):
+            find_test_cases(self.test_dir)
+
+
+class TestCalculateAccuracy(unittest.TestCase):
+    def test_calculate_accuracy_perfect_score(self):
+        expected = {
+            "field_1": "value_1",
+            "field_2": "value_2"
+        }
+        actual = {
+            "field_1": "value_1",
+            "field_2": "value_2"
+        }
+        result = calculate_accuracy(expected, actual)
+        for field in result.values():
+            self.assertEqual(field['pass_fail'], "Pass")
+            self.assertEqual(field['score'], 100.0)
+
+    def test_calculate_accuracy_all_fail(self):
+        expected = {
+            "field_1": "value_1",
+            "field_2": "value_2"
+        }
+        actual = {
+            "field_1": "wrong_value",
+            "field_2": "another_wrong_value"
+        }
+        result = calculate_accuracy(expected, actual)
+        self.assertEqual(int(result['field_1']['score']), 27)
+        self.assertEqual(int(result['field_2']['score']), 15)
+
+
+    def test_calculate_accuracy_with_missing_field(self):
+        expected = {
+            "field_1": "value_1",
+            "field_2": "value_2"
+        }
+        actual = {
+            "field_1": "value_1",
+        }
+        result = calculate_accuracy(expected, actual)
+        self.assertEqual(int(result['field_1']['score']), 100)
+        self.assertEqual(int(result['field_2']['score']), 0)
+
+class TestRunTestCase(unittest.TestCase):
+    @patch("performance_assessment.LabelStorage")
+    @patch("performance_assessment.OCR")
+    @patch("performance_assessment.GPT")
+    @patch("performance_assessment.analyze")
+    def test_run_test_case(self, mock_analyze, MockGPT, MockOCR, MockLabelStorage):
+        # Setting up mock returns
+        mock_analyze.return_value.model_dump_json.return_value = json.dumps({
+            "field_1": "value_1",
+            "field_2": "value_2"
+        })
+        MockOCR.return_value = MagicMock()
+        MockGPT.return_value = MagicMock()
+        MockLabelStorage.return_value = MagicMock()
+        
+        # Create temporary image and expected JSON files
+        temp_image = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
+        temp_json = tempfile.NamedTemporaryFile(delete=False, suffix=".json")
+        with open(temp_json.name, 'w') as f:
+            json.dump({"field_1": "value_1", "field_2": "value_2"}, f)
+        
+        result = run_test_case(1, [temp_image.name], temp_json.name)
+        
+        self.assertEqual(result['test_case_number'], 1)
+        self.assertIn('accuracy_results', result)
+        self.assertIn('performance', result)
+        for field in result['accuracy_results'].values():
+            self.assertEqual(field['pass_fail'], "Pass")
+
+        # Clean up temporary files
+        os.unlink(temp_image.name)
+        os.unlink(temp_json.name)
+
+
+class TestGenerateCSVReport(unittest.TestCase):
+    def setUp(self):
+        self.results = [
+            {
+                'test_case_number': 1,
+                'performance': 5.44,
+                'accuracy_results': {
+                    'field_1': {
+                        'score': 100.0,
+                        'expected_value': 'value_1',
+                        'actual_value': 'value_1',
+                        'pass_fail': 'Pass'
+                    },
+                    'field_2': {
+                        'score': 100.0,
+                        'expected_value': 'value_2',
+                        'actual_value': 'value_2',
+                        'pass_fail': 'Pass'
+                    }
+                }
+            }
+        ]
+        self.report_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.report_dir)
+
+    @patch("os.makedirs")
+    @patch("os.path.join", return_value=os.path.join(tempfile.gettempdir(), "test_results.csv"))
+    def test_generate_csv_report(self, mock_join, mock_makedirs):
+        # Generate the CSV report
+        generate_csv_report(self.results) # <- since we patched make_dirs, we don't need to clean up the directory
+        report_path = mock_join.return_value
+
+        # Check if the file was created
+        self.assertTrue(os.path.exists(report_path))
+
+        # Verify the contents of the CSV
+        with open(report_path, 'r') as f:
+            reader = csv.reader(f)
+            rows = list(reader)
+
+        # Check header
+        self.assertEqual(rows[0], [
+            "Test Case",
+            "Field Name",
+            "Pass/Fail",
+            "Accuracy Score",
+            "Pipeline Speed (seconds)",
+            "Expected Value",
+            "Actual Value"
+        ])
+
+        # Check data row
+        self.assertEqual(rows[1], [
+            '1',
+            'field_1',
+            'Pass',
+            '100.00',
+            '5.4400',
+            'value_1',
+            'value_1'
+        ])
+
+
+if __name__ == "__main__":
     unittest.main()
 
 

From 7a1c837e6561575790ad97c474817526872b8bd8 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Mon, 7 Oct 2024 23:10:14 +0000
Subject: [PATCH 12/17] fix: lint and markdown lint

---
 test_data/README.md                  | 13 +++++++------
 tests/test_performance_assessment.py |  1 -
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/test_data/README.md b/test_data/README.md
index 43e56c6..df9da87 100644
--- a/test_data/README.md
+++ b/test_data/README.md
@@ -1,14 +1,15 @@
-### Test Data
+# Test Data
 
-This folder hosts all the data needed to perform testing on the processing pipeline (label images and their expected output).
+This folder hosts all the data needed to perform testing on the processing
+pipeline (label images and their expected output).
 
 Please follow the following structure when adding new test cases:
 
-```
+```text
 ├── test_data/                # Test images and related data
 │   ├── labels/               # Folders organized by test case
-│   │   ├── label_001/        # Each folder contains images and expected output JSON
-│   │   │   ├── img_001.jpg
+│   │   ├── label_001/        # Each folder contains images and expected 
+│   │   │   ├── img_001.jpg   # output JSON
 │   │   │   ├── img_002.jpg
 │   │   │   └── expected_output.json
 │   │   ├── label_002/
@@ -16,4 +17,4 @@ Please follow the following structure when adding new test cases:
 │   │   │   ├── img_002.jpg
 │   │   │   └── expected_output.json
 │   │   └── ...
-```
\ No newline at end of file
+```
diff --git a/tests/test_performance_assessment.py b/tests/test_performance_assessment.py
index e36583e..5005764 100644
--- a/tests/test_performance_assessment.py
+++ b/tests/test_performance_assessment.py
@@ -11,7 +11,6 @@
     calculate_accuracy,
     run_test_case,
     generate_csv_report,
-    main
 )
 
 class TestExtractLeafFields(unittest.TestCase):

From 5078634e001608639531ac1e3a3aac4a35b8960a Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Tue, 8 Oct 2024 17:28:14 +0000
Subject: [PATCH 13/17] fix: updated the gitignore file + fixed the conflicts
 with main

---
 .gitignore             |  3 ---
 pipeline/inspection.py | 60 ++++++++++++++++++++++++------------------
 2 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/.gitignore b/.gitignore
index 06b1ab7..4861de4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -170,7 +170,4 @@ cython_debug/
 # Testing artifacts
 end_to_end_pipeline_artifacts
 reports
-test_outputs
-
-
 logs
diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index f67a928..dc41b44 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -1,23 +1,27 @@
 import re
 from typing import List, Optional
-from pydantic import BaseModel, Field, field_validator, model_validator
+
+from pydantic import BaseModel, Field, field_validator
+
 
 class npkError(ValueError):
     pass
 
+
 def extract_first_number(string: str) -> Optional[str]:
     if string is not None:
-        match = re.search(r'\d+(\.\d+)?', string)
+        match = re.search(r"\d+(\.\d+)?", string)
         if match:
             return match.group()
     return None
 
+
 class NutrientValue(BaseModel):
     nutrient: str
     value: Optional[float] = None
     unit: Optional[str] = None
 
-    @field_validator('value', mode='before', check_fields=False)
+    @field_validator("value", mode="before", check_fields=False)
     def convert_value(cls, v):
         if isinstance(v, bool):
             return None
@@ -26,12 +30,13 @@ def convert_value(cls, v):
         elif isinstance(v, (str)):
             return extract_first_number(v)
         return None
-    
+
+
 class Value(BaseModel):
     value: Optional[float]
     unit: Optional[str]
 
-    @field_validator('value', mode='before', check_fields=False)
+    @field_validator("value", mode="before", check_fields=False)
     def convert_value(cls, v):
         if isinstance(v, bool):
             return None
@@ -40,7 +45,8 @@ def convert_value(cls, v):
         elif isinstance(v, (str)):
             return extract_first_number(v)
         return None
-    
+
+
 class GuaranteedAnalysis(BaseModel):
     title: Optional[str] = None
     nutrients: List[NutrientValue] = []
@@ -54,12 +60,13 @@ def replace_none_with_empty_list(cls, v):
             v = []
         return v
 
+
 class Specification(BaseModel):
-    humidity: Optional[float] = Field(..., alias='humidity')
-    ph: Optional[float] = Field(..., alias='ph')
+    humidity: Optional[float] = Field(..., alias="humidity")
+    ph: Optional[float] = Field(..., alias="ph")
     solubility: Optional[float]
 
-    @field_validator('humidity', 'ph', 'solubility', mode='before', check_fields=False)
+    @field_validator("humidity", "ph", "solubility", mode="before", check_fields=False)
     def convert_specification_values(cls, v):
         if isinstance(v, bool):
             return None
@@ -69,6 +76,7 @@ def convert_specification_values(cls, v):
             return extract_first_number(v)
         return None
 
+
 class FertilizerInspection(BaseModel):
     company_name: Optional[str] = None
     company_address: Optional[str] = None
@@ -93,29 +101,29 @@ class FertilizerInspection(BaseModel):
     instructions_fr: List[str] = []
     ingredients_en: List[NutrientValue] = []
     ingredients_fr: List[NutrientValue] = []
-    
-    @field_validator('npk', mode='before')
+
+    @field_validator("npk", mode="before")
     def validate_npk(cls, v):
         if v is not None:
-            pattern = re.compile(r'^\d+(\.\d+)?-\d+(\.\d+)?-\d+(\.\d+)?$')
+            pattern = re.compile(r"^\d+(\.\d+)?-\d+(\.\d+)?-\d+(\.\d+)?$")
             if not pattern.match(v):
                 return None
         return v
 
-    @model_validator(mode='before')
-    def replace_none_with_empty_list(cls, values):
-        fields_to_check = [
-            'cautions_en', 'first_aid_en', 'cautions_fr', 'first_aid_fr',
-            'instructions_en', 'micronutrients_en', 'ingredients_en',
-            'specifications_en', 'instructions_fr',
-            'micronutrients_fr', 'ingredients_fr',
-            'specifications_fr', 'guaranteed_analysis'
-        ]
-        for field in fields_to_check:
-            if values.get(field) is None:
-                values[field] = []
-        return values
-
+    @field_validator(
+        "cautions_en",
+        "cautions_fr",
+        "instructions_en",
+        "instructions_fr",
+        "ingredients_en",
+        "ingredients_fr",
+        "weight",
+        mode="before",
+    )
+    def replace_none_with_empty_list(cls, v):
+        if v is None:
+            v = []
+        return v
 
     class Config:
         populate_by_name = True
\ No newline at end of file

From 37b7764b13b9392fce810e649e4f4afe5902b295 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Tue, 8 Oct 2024 21:11:02 +0000
Subject: [PATCH 14/17] fix: adding EOL

---
 pipeline/inspection.py               | 2 +-
 tests/test_performance_assessment.py | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index dc41b44..5fdfc1b 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -126,4 +126,4 @@ def replace_none_with_empty_list(cls, v):
         return v
 
     class Config:
-        populate_by_name = True
\ No newline at end of file
+        populate_by_name = True
diff --git a/tests/test_performance_assessment.py b/tests/test_performance_assessment.py
index 5005764..4ee904d 100644
--- a/tests/test_performance_assessment.py
+++ b/tests/test_performance_assessment.py
@@ -374,6 +374,3 @@ def test_generate_csv_report(self, mock_join, mock_makedirs):
 
 if __name__ == "__main__":
     unittest.main()
-
-
-

From cd29f93b1586cc8a505f845293f17737b77777e6 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Wed, 16 Oct 2024 15:36:09 +0000
Subject: [PATCH 15/17] feat: adding simple progress logging.

---
 performance_assessment.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/performance_assessment.py b/performance_assessment.py
index 9b81c73..e5bfaba 100644
--- a/performance_assessment.py
+++ b/performance_assessment.py
@@ -11,7 +11,6 @@
 
 ACCURACY_THRESHOLD = 80.0
 
-
 def extract_leaf_fields(
     data: dict | list, parent_key: str = ''
 ) -> dict[str, str | int | float | bool | None]:
@@ -43,6 +42,7 @@ def find_test_cases(labels_folder: str) -> list[tuple[list[str], str]]:
         if os.path.isdir(os.path.join(labels_folder, directory)) and directory.startswith("label_")
     )
     if len(label_directories) == 0:
+        print(f"No label directories found in {labels_folder}")
         raise FileNotFoundError(f"No label directories found in {labels_folder}")
 
     for label_directory in label_directories:
@@ -107,9 +107,11 @@ def run_test_case(
     )
 
     # Run performance test
+    print(f"\tRunning analysis for test case...")
     start_time = time.time()
     actual_output = analyze(storage, ocr, gpt) # <-- the `analyse` function deletes the images it processes so we don't need to clean up our image copies
     performance = time.time() - start_time
+    print(f"\tAnalysis completed in {performance:.2f} seconds.")
 
     # Process actual output
     actual_fields = extract_leaf_fields(json.loads(actual_output.model_dump_json()))
@@ -119,6 +121,7 @@ def run_test_case(
         expected_fields = extract_leaf_fields(json.load(file))
 
     # Calculate accuracy
+    print(f"\tCalculating accuracy of results...")
     accuracy_results = calculate_accuracy(expected_fields, actual_fields)
 
     # Return results
@@ -163,8 +166,10 @@ def generate_csv_report(results: list[dict[str, any]]) -> None:
 
 
 def main() -> None:
-    load_dotenv()
+    print("Script execution started.")
 
+    load_dotenv()
+    
     # Validate required environment variables
     required_vars = [
         "AZURE_API_ENDPOINT",
@@ -178,12 +183,20 @@ def main() -> None:
         raise RuntimeError(f"Missing required environment variables: {', '.join(missing_vars)}")
 
     test_cases = find_test_cases("test_data/labels")
+    print(f"Found {len(test_cases)} test case(s) to process.")
+
     results = []
     for idx, (image_paths, expected_json_path) in enumerate(test_cases, 1):
-        result = run_test_case(idx, image_paths, expected_json_path)
-        results.append(result)
+        print(f"Processing test case {idx}...")
+        try:
+            result = run_test_case(idx, image_paths, expected_json_path)
+            results.append(result)
+        except Exception as e:
+            print(f"Error processing test case {idx}: {e}")
+            continue # I'd rather continue processing the other test cases than stop the script for now
+    
     generate_csv_report(results)
-
+    print("Script execution completed.")
 
 if __name__ == "__main__":
     main()

From 4179c58e41dc84fe0cf9e171c423c75146e45c32 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Wed, 16 Oct 2024 12:07:44 -0400
Subject: [PATCH 16/17] fix: fixed the model_validator missing import

model_validator import got lost during the merge conflict resolution
---
 pipeline/inspection.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pipeline/inspection.py b/pipeline/inspection.py
index 0a7e4ce..237784f 100644
--- a/pipeline/inspection.py
+++ b/pipeline/inspection.py
@@ -1,8 +1,6 @@
 import re
 from typing import List, Optional
-
-from pydantic import BaseModel, Field, field_validator
-
+from pydantic import BaseModel, Field, field_validator, model_validator
 
 class npkError(ValueError):
     pass

From b8a45810d13fe1ecfdb6316559b7fe3e1d66f571 Mon Sep 17 00:00:00 2001
From: James <60270865+Endlessflow@users.noreply.github.com>
Date: Wed, 16 Oct 2024 12:11:51 -0400
Subject: [PATCH 17/17] fix: removed f-string in places they are not necessary

---
 performance_assessment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/performance_assessment.py b/performance_assessment.py
index e5bfaba..a62d54d 100644
--- a/performance_assessment.py
+++ b/performance_assessment.py
@@ -107,7 +107,7 @@ def run_test_case(
     )
 
     # Run performance test
-    print(f"\tRunning analysis for test case...")
+    print("\tRunning analysis for test case...")
     start_time = time.time()
     actual_output = analyze(storage, ocr, gpt) # <-- the `analyse` function deletes the images it processes so we don't need to clean up our image copies
     performance = time.time() - start_time
@@ -121,7 +121,7 @@ def run_test_case(
         expected_fields = extract_leaf_fields(json.load(file))
 
     # Calculate accuracy
-    print(f"\tCalculating accuracy of results...")
+    print("\tCalculating accuracy of results...")
     accuracy_results = calculate_accuracy(expected_fields, actual_fields)
 
     # Return results