Add support for partial config files

docs/configuration.md

@@ -19,6 +19,35 @@ create-config --help` for further details).
 topostats create-config
 ```
 
+## Partial configurations
+
+TopoStats supports using a partial configuration, where you specify only the fields you wish to override. This is
+useful if you only want to change a few parameters from the default configuration or would like to use a configuration
+file that is smaller and easier to read.
+
+To create a partial configuration file, simply create a new config file and delete anything you don't want to override.
+
+TopoStats will take the partial configuration file and merge it with the default configuration file, with the partial
+configuration taking precedence. This means that any fields you specify in the partial configuration will override the
+default configuration, while any fields you don't specify will be taken from the default configuration. Command-line
+arguments will override both the default and partial configurations.
+
+For example, you could use a configuration as simple as:
+
+```yaml
+base_dir: ./mydata/
+output_dir: ./myoutput/
+filter:
+  remove_scars:
+    run: true
+grains:
+  threshold_method: absolute
+  threshold_absolute:
+    above: 1.2
+  absolute_area_threshold:
+    above: [400, 1000]
+```
+
 ## Using a custom configuration
 
 If you have generated a configuration file you can modify and edit a configuration it to change the parameters (see

tests/resources/test_partial_config.yaml

@@ -0,0 +1,45 @@
+# # For more information on configuration and how to use it:
+# https://afm-spm.github.io/TopoStats/main/configuration.html
+base_dir: ./ # Directory in which to search for data files
+output_dir: ./output # Directory to output results to
+log_level: info # Verbosity of output. Options: warning, error, info, debug
+cores: 2 # Number of CPU cores to utilise for processing multiple files simultaneously.
+file_ext: .spm # File extension of the data files.
+loading:
+  channel: Height # Channel to pull data from in the data files.
+filter:
+  run: true # Options : true, false
+  row_alignment_quantile: 0.5 # lower values may improve flattening of larger features
+  threshold_method: absolute # Options : otsu, std_dev, absolute
+  otsu_threshold_multiplier: 1.0
+  threshold_std_dev:
+    below: 10.0 # Threshold for data below the image background
+    above: 1.0 # Threshold for data above the image background
+  threshold_absolute:
+    below: -1.0 # Threshold for data below the image background
+    above: 1.0 # Threshold for data above the image background
+  gaussian_size: 1.0121397464510862 # Gaussian blur intensity in px
+  gaussian_mode: nearest
+grains:
+  run: true # Options : true, false
+  # Thresholding by height
+  threshold_method: std_dev # Options : std_dev, otsu, absolute, unet
+  otsu_threshold_multiplier: 1.0
+  threshold_std_dev:
+    below: 10.0 # Threshold for grains below the image background
+    above: 1.0 # Threshold for grains above the image background
+  threshold_absolute:
+    below: -1.0 # Threshold for grains below the image background
+    above: 1.0 # Threshold for grains above the image background
+  direction: above # Options: above, below, both (defines whether to look for grains above or below thresholds or both)
+  # Thresholding by area
+  smallest_grain_size_nm2: 50 # Size in nm^2 of tiny grains/blobs (noise) to remove, must be > 0.0
+  absolute_area_threshold:
+    above: [300, 3000] # above surface [Low, High] in nm^2 (also takes null)
+    below: [null, null] # below surface [Low, High] in nm^2 (also takes null)
+  remove_edge_intersecting_grains: true # Whether or not to remove grains that touch the image border
+  unet_config:
+    model_path: null # Path to a trained U-Net model
+    grain_crop_padding: 2 # Padding to apply to the grain crop bounding box
+    upper_norm_bound: 5.0 # Upper bound for normalisation of input data. This should be slightly higher than the maximum desired / expected height of grains.
+    lower_norm_bound: -1.0 # Lower bound for normalisation of input data. This should be slightly lower than the minimum desired / expected height of the background.

tests/test_io.py

@@ -26,6 +26,7 @@
     hdf5_to_dict,
     load_array,
     load_pkl,
+    merge_mappings,
     path_to_str,
     read_64d,
     read_char,
@@ -62,6 +63,36 @@
 # pylint: disable=too-many-lines
 
 
+@pytest.mark.parametrize(
+    ("dict1", "dict2", "expected_merged_dict"),
+    [
+        pytest.param(
+            {"a": 1, "b": 2},
+            {"c": 3, "d": 4},
+            {"a": 1, "b": 2, "c": 3, "d": 4},
+            id="two dicts, no common keys",
+        ),
+        pytest.param(
+            {"a": 1, "b": 2},
+            {"b": 3, "c": 4},
+            {"a": 1, "b": 3, "c": 4},
+            id="two dicts, one common key, testing priority of second dict",
+        ),
+        # Nested dictionaries
+        pytest.param(
+            {"a": 1, "b": {"c": 2, "d": 3}},
+            {"b": {"c": 4, "e": 5}},
+            {"a": 1, "b": {"c": 4, "d": 3, "e": 5}},
+            id="nested dictionaries, one common key in nested dict, testing priority of second dict",
+        ),
+    ],
+)
+def test_merge_mappings(dict1: dict, dict2: dict, expected_merged_dict: dict) -> None:
+    """Test merging of mappings."""
+    merged_dict = merge_mappings(dict1, dict2)
+    assert merged_dict == expected_merged_dict
+
+
 def test_get_date_time() -> None:
     """Test the fetching of a formatted date and time string."""
     assert datetime.strptime(get_date_time(), "%Y-%m-%d %H:%M:%S")

tests/test_run_topostats.py

@@ -1,16 +1,86 @@
 """Test end-to-end running of topostats."""
 
+import argparse
 import logging
 from pathlib import Path
 
 import pytest
 
 from topostats.entry_point import entry_point
 from topostats.logs.logs import LOGGER_NAME
+from topostats.run_topostats import reconcile_config_args
+from topostats.validation import DEFAULT_CONFIG_SCHEMA, validate_config
 
 BASE_DIR = Path.cwd()
 
 
+def test_reconcile_config_args_no_config() -> None:
+    """Test the handle config file function with no config."""
+    args = argparse.Namespace(
+        program="process",
+        config_file=None,
+    )
+    config = reconcile_config_args(args=args)
+
+    # Check that the config passes the schema
+    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
+
+
+def test_reconcile_config_args_no_config_with_overrides() -> None:
+    """Test the handle config file function with no config and overrides."""
+    args = argparse.Namespace(
+        program="process",
+        config_file=None,
+        output_dir="./dummy_output_dir",
+    )
+    config = reconcile_config_args(args=args)
+
+    # Check that the overrides have been applied
+    assert config["output_dir"] == Path("./dummy_output_dir")
+    # Check that the config still passes the schema
+    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
+
+
+def test_reconcile_config_args_full_config() -> None:
+    """Test the handle config file function with a full config."""
+    args = argparse.Namespace(program="process", config_file=f"{BASE_DIR / 'topostats' / 'default_config.yaml'}")
+
+    config = reconcile_config_args(args=args)
+
+    # Check that the config passes the schema
+    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
+
+
+def test_reconcile_config_args_partial_config() -> None:
+    """Test the reconcile_config_args function with a partial config."""
+    args = argparse.Namespace(
+        program="process", config_file=f"{BASE_DIR / 'tests' / 'resources' / 'test_partial_config.yaml'}"
+    )
+    config = reconcile_config_args(args=args)
+
+    # Check that the partial config has overridden the default config
+    assert config["filter"]["threshold_method"] == "absolute"
+    # Check that the config still passes the schema
+    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
+
+
+def test_reconcile_config_args_partial_config_with_overrides() -> None:
+    """Test the reconcile_config_args function with a partial config and overrides."""
+    args = argparse.Namespace(
+        program="process",
+        config_file=f"{BASE_DIR / 'tests' / 'resources' / 'test_partial_config.yaml'}",
+        output_dir="./dummy_output_dir",
+    )
+    config = reconcile_config_args(args=args)
+
+    # Check that the partial config has overridden the default config
+    assert config["filter"]["threshold_method"] == "absolute"
+    # Check that the overrides have been applied
+    assert config["output_dir"] == Path("./dummy_output_dir")
+    # Check that the config still passes the schema
+    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
+
+
 @pytest.mark.parametrize("option", [("-h"), ("--help")])
 def test_run_topostats_main_help(capsys, option) -> None:
     """Test the -h/--help flag to run_topostats."""

topostats/io.py

@@ -9,10 +9,11 @@
 import pickle as pkl
 import re
 import struct
+from collections.abc import MutableMapping
 from datetime import datetime
 from importlib import resources
 from pathlib import Path
-from typing import Any
+from typing import Any, TypeVar
 
 import h5py
 import numpy as np
@@ -36,6 +37,39 @@
 # pylint: disable=broad-except
 # pylint: disable=too-many-lines
 
+MutableMappingType = TypeVar("MutableMappingType", bound="MutableMapping")
+
+
+def merge_mappings(map1: MutableMappingType, map2: MutableMappingType) -> MutableMappingType:
+    """
+    Merge two mappings (dictionaries), with priority given to the second mapping.
+
+    Note: Using a Mapping should make this robust to any mapping type, not just dictionaries. MutableMapping was needed
+    as Mapping is not a mutable type, and this function needs to be able to change the dictionaries.
+
+    Parameters
+    ----------
+    map1 : MutableMapping
+        First mapping to merge, with secondary priority.
+    map2 : MutableMapping
+        Second mapping to merge, with primary priority.
+
+    Returns
+    -------
+    dict
+        Merged dictionary.
+    """
+    # Iterate over the second mapping
+    for key, value in map2.items():
+        # If the value is another mapping, then recurse
+        if isinstance(value, MutableMapping):
+            # If the key is not in the first mapping, add it as an empty dictionary before recursing
+            map1[key] = merge_mappings(map1.get(key, {}), value)
+        else:
+            # Else simply add / override the key value pair
+            map1[key] = value
+    return map1
+
 
 # Sylvia: Ruff says too complex but I think breaking this out would be more complex.
 def dict_almost_equal(dict1: dict, dict2: dict, abs_tol: float = 1e-9):  # noqa: C901

topostats/run_topostats.py

@@ -4,6 +4,9 @@
 This provides an entry point for running TopoStats as a command line programme.
 """
 
+from __future__ import annotations
+
+import argparse
 import logging
 import sys
 from collections import defaultdict
@@ -16,7 +19,15 @@
 import yaml
 from tqdm import tqdm
 
-from topostats.io import LoadScans, dict_to_json, find_files, read_yaml, save_folder_grainstats, write_yaml
+from topostats.io import (
+    LoadScans,
+    dict_to_json,
+    find_files,
+    merge_mappings,
+    read_yaml,
+    save_folder_grainstats,
+    write_yaml,
+)
 from topostats.logs.logs import LOGGER_NAME
 from topostats.plotting import toposum
 from topostats.processing import check_run_steps, completion_message, process_scan
@@ -37,7 +48,54 @@
 # pylint: disable=too-many-nested-blocks
 
 
-def run_topostats(args: None = None) -> None:  # noqa: C901
+def reconcile_config_args(args: argparse.Namespace | None) -> dict:
+    """
+    Reconcile command line arguments with the default configuration.
+
+    Command line arguments take precedence over the default configuration. If a partial configuration file is specified
+    (with '-c' or '--config-file') the defaults are over-ridden by these values (internally the configuration
+    dictionary is updated with these values). Any other command line arguments take precedence over both the default
+    and those supplied in a configuration file (again the dictionary is updated).
+
+    The final configuration is validated before processing begins.
+
+    Parameters
+    ----------
+    args : argparse.Namespace
+        Command line arguments passed into TopoStats.
+
+    Returns
+    -------
+    dict
+        The configuration dictionary.
+    """
+    default_config_raw = (resources.files(__package__) / "default_config.yaml").read_text()
+    default_config = yaml.safe_load(default_config_raw)
+    if args is not None:
+        config_file_arg: str | None = args.config_file
+        if config_file_arg is not None:
+            config = read_yaml(config_file_arg)
+            # Merge the loaded config with the default config to fill in any defaults that are missing
+            # Make sure to prioritise the loaded config, so it overrides the default
+            config = merge_mappings(map1=default_config, map2=config)
+        else:
+            # If no config file is provided, use the default config
+            config = default_config
+    else:
+        # If no args are provided, use the default config
+        config = default_config
+
+    # Override the config with command line arguments passed in, eg --output_dir ./output/
+    if args is not None:
+        config = update_config(config, args)
+
+    # Validate configuration
+    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
+
+    return config
+
+
+def run_topostats(args: argparse.Namespace | None = None) -> None:  # noqa: C901
     """
     Find and process all files.
 
@@ -47,13 +105,7 @@ def run_topostats(args: None = None) -> None:  # noqa: C901
         Arguments.
     """
     # Parse command line options, load config (or default) and update with command line options
-    if args.config_file is not None:
-        config = read_yaml(args.config_file)
-    else:
-        default_config = (resources.files(__package__) / "default_config.yaml").read_text()
-        config = yaml.safe_load(default_config)
-    # Override the config with command line arguments passed in, eg --output_dir ./output/
-    config = update_config(config, args)
+    config = reconcile_config_args(args=args)
 
     # Set logging level
     if config["log_level"] == "warning":
@@ -64,8 +116,6 @@ def run_topostats(args: None = None) -> None:  # noqa: C901
         LOGGER.setLevel("DEBUG")
     else:
         LOGGER.setLevel("INFO")
-    # Validate configuration
-    validate_config(config, schema=DEFAULT_CONFIG_SCHEMA, config_type="YAML configuration file")
 
     # Create base output directory
     config["output_dir"].mkdir(parents=True, exist_ok=True)