diff --git a/dev-requirements.txt b/dev-requirements.txt index 066ce393..b189db62 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -55,6 +55,7 @@ colorama==0.4.6 # via # build # click + # loguru # pytest # tqdm contourpy==1.2.0 @@ -116,6 +117,8 @@ lazy-loader==0.3 # via scikit-image llvmlite==0.41.1 # via numba +loguru==0.7.2 + # via swmmanywhere (pyproject.toml) matplotlib==3.8.2 # via # salib @@ -300,6 +303,8 @@ virtualenv==20.24.5 # via pre-commit wheel==0.41.3 # via pip-tools +win32-setctime==1.1.0 + # via loguru xarray==2023.12.0 # via # rioxarray diff --git a/pyproject.toml b/pyproject.toml index 673acf7d..acbcd2f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "geopandas", "geopy", "GitPython", + "loguru", "matplotlib", "netcdf4", "networkx", diff --git a/requirements.txt b/requirements.txt index cde240ba..20bdd514 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ cligj==0.7.2 colorama==0.4.6 # via # click + # loguru # tqdm contourpy==1.2.0 # via matplotlib @@ -94,6 +95,8 @@ lazy-loader==0.3 # via scikit-image llvmlite==0.41.1 # via numba +loguru==0.7.2 + # via swmmanywhere (pyproject.toml) matplotlib==3.8.2 # via # salib @@ -237,6 +240,8 @@ tzdata==2024.1 # via pandas urllib3==2.1.0 # via requests +win32-setctime==1.1.0 + # via loguru xarray==2023.12.0 # via # rioxarray diff --git a/swmmanywhere/graph_utilities.py b/swmmanywhere/graph_utilities.py index e771c009..d63ab08c 100644 --- a/swmmanywhere/graph_utilities.py +++ b/swmmanywhere/graph_utilities.py @@ -22,6 +22,7 @@ from swmmanywhere import geospatial_utilities as go from swmmanywhere import parameters +from swmmanywhere.logging import logger def load_graph(fid: Path) -> nx.Graph: @@ -933,7 +934,7 @@ def process_successors(G: nx.Graph, edge_diams[(node,ds_node,0)] = diam chamber_floor[ds_node] = surface_elevations[ds_node] - depth if ix > 0: - print('''a node has multiple successors, + logger.warning('''a node has multiple successors, not sure how that can happen if using shortest path to derive topology''') diff --git a/swmmanywhere/logging.py b/swmmanywhere/logging.py new file mode 100644 index 00000000..4eff38e0 --- /dev/null +++ b/swmmanywhere/logging.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +"""Created on 2024-03-04. + +@author: Barney +""" +import os +import sys + +import loguru + + +def dynamic_filter(record): + """A dynamic filter.""" + return os.getenv("SWMMANYWHERE_VERBOSE", "false").lower() == "true" + +def get_logger() -> loguru.logger: + """Get a logger.""" + logger = loguru.logger + logger.configure( + handlers=[ + { + "sink": sys.stdout, + "filter" : dynamic_filter, + "colorize": True, + "format": " | ".join( + [ + "{time:YYYY/MM/DD HH:mm:ss}", + "{message}", + ] + ), + } + ] + ) + return logger + +# Get the logger +logger = get_logger() + +# Add a test_logger method to the logger +logger.test_logger = lambda : logger.info("This is a test message.") + +# Store the original add method +original_add = logger.add + +# Define a new function that wraps the original add method +def new_add(sink, **kwargs): + """A new add method to wrap existing one but with the filter.""" + # Include the dynamic filter in the kwargs if not already specified + if 'filter' not in kwargs: + kwargs['filter'] = dynamic_filter + # Call the original add method with the updated kwargs + return original_add(sink, **kwargs) + +# Replace the logger's add method with new_add +logger.add = new_add \ No newline at end of file diff --git a/swmmanywhere/prepare_data.py b/swmmanywhere/prepare_data.py index 0551702b..ab2cef1d 100644 --- a/swmmanywhere/prepare_data.py +++ b/swmmanywhere/prepare_data.py @@ -17,7 +17,8 @@ import yaml from geopy.geocoders import Nominatim -# Some minor comment (to remove) +from swmmanywhere.logging import logger + def get_country(x: float, y: float) -> dict[int, str]: @@ -83,9 +84,9 @@ def download_buildings(file_address: Path, # Save data to the specified file address with file_address.open("wb") as file: file.write(response.content) - print(f"Data downloaded and saved to {file_address}") + logger.info(f"Data downloaded and saved to {file_address}") else: - print(f"Error downloading data. Status code: {response.status_code}") + logger.error(f"Error downloading data. Status code: {response.status_code}") return response.status_code def download_street(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph: @@ -174,10 +175,10 @@ def download_elevation(fid: Path, with fid.open('wb') as rast_file: shutil.copyfileobj(r.raw, rast_file) - print('Elevation data downloaded successfully.') + logger.info('Elevation data downloaded successfully.') except requests.exceptions.RequestException as e: - print(f'Error downloading elevation data: {e}') + logger.error(f'Error downloading elevation data: {e}') return r.status_code diff --git a/swmmanywhere/preprocessing.py b/swmmanywhere/preprocessing.py index ca9c6c0b..527413bc 100644 --- a/swmmanywhere/preprocessing.py +++ b/swmmanywhere/preprocessing.py @@ -16,6 +16,7 @@ from swmmanywhere import geospatial_utilities as go from swmmanywhere import graph_utilities as gu from swmmanywhere import parameters, prepare_data +from swmmanywhere.logging import logger def next_directory(keyword: str, directory: Path) -> int: @@ -160,7 +161,7 @@ def prepare_precipitation(bbox: tuple[float, float, float, float], """Download and reproject precipitation data.""" if addresses.precipitation.exists(): return - print(f'downloading precipitation to {addresses.precipitation}') + logger.info(f'downloading precipitation to {addresses.precipitation}') precip = prepare_data.download_precipitation(bbox, api_keys['cds_username'], api_keys['cds_api_key']) @@ -175,7 +176,7 @@ def prepare_elvation(bbox: tuple[float, float, float, float], """Download and reproject elevation data.""" if addresses.elevation.exists(): return - print(f'downloading elevation to {addresses.elevation}') + logger.info(f'downloading elevation to {addresses.elevation}') with tempfile.TemporaryDirectory() as temp_dir: fid = Path(temp_dir) / 'elevation.tif' prepare_data.download_elevation(fid, @@ -194,12 +195,12 @@ def prepare_building(bbox: tuple[float, float, float, float], return if not addresses.national_building.exists(): - print(f'downloading buildings to {addresses.national_building}') + logger.info(f'downloading buildings to {addresses.national_building}') prepare_data.download_buildings(addresses.national_building, bbox[0], bbox[1]) - print(f'trimming buildings to {addresses.building}') + logger.info(f'trimming buildings to {addresses.building}') national_buildings = gpd.read_parquet(addresses.national_building) buildings = national_buildings.cx[bbox[0]:bbox[2], bbox[1]:bbox[3]] # type: ignore @@ -213,7 +214,7 @@ def prepare_street(bbox: tuple[float, float, float, float], """Download and reproject street graph.""" if addresses.street.exists(): return - print(f'downloading street network to {addresses.street}') + logger.info(f'downloading street network to {addresses.street}') street_network = prepare_data.download_street(bbox) street_network = go.reproject_graph(street_network, source_crs, @@ -227,7 +228,7 @@ def prepare_river(bbox: tuple[float, float, float, float], """Download and reproject river graph.""" if addresses.river.exists(): return - print(f'downloading river network to {addresses.river}') + logger.info(f'downloading river network to {addresses.river}') river_network = prepare_data.download_river(bbox) river_network = go.reproject_graph(river_network, source_crs, diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 00000000..f228ebfc --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +"""Created on 2024-01-26. + +@author: Barney +""" +import os +from pathlib import Path +from tempfile import NamedTemporaryFile + +from swmmanywhere.logging import logger + + +def test_logger(): + """Test logger.""" + os.environ["SWMMANYWHERE_VERBOSE"] = "true" + assert logger is not None + logger.test_logger() + logger.debug("This is a debug message.") + logger.info("This is an info message.") + logger.warning("This is a warning message.") + logger.error("This is an error message.") + logger.critical("This is a critical message.") + with NamedTemporaryFile(suffix='.log', + mode = 'w+b', + delete=False) as temp_file: + fid = Path(temp_file.name) + logger.add(fid) + logger.test_logger() + assert temp_file.read() != b"" + logger.remove() + fid.unlink() + +def test_logger_disable(): + """Test the disable function.""" + with NamedTemporaryFile(suffix='.log', + mode = 'w+b', + delete=False) as temp_file: + fid = Path(temp_file.name) + os.environ["SWMMANYWHERE_VERBOSE"] = "false" + logger.add(fid) + logger.test_logger() + assert temp_file.read() == b"" + logger.remove() + fid.unlink() + +def test_logger_reimport(): + """Reimport logger to check that changes from disable are persistent.""" + from swmmanywhere.logging import logger + with NamedTemporaryFile(suffix='.log', + mode = 'w+b', + delete=False) as temp_file: + fid = Path(temp_file.name) + logger.add(fid) + logger.test_logger() + assert temp_file.read() == b"" + logger.remove() + fid.unlink() + +def test_logger_again(): + """Test the logger after these changes to make sure still works.""" + os.environ["SWMMANYWHERE_VERBOSE"] = "true" + with NamedTemporaryFile(suffix='.log', + mode = 'w+b', + delete=False) as temp_file: + fid = Path(temp_file.name) + logger.add(fid) + logger.test_logger() + assert temp_file.read() != b"" + logger.remove() + fid.unlink() \ No newline at end of file