Skip to content

Commit

Permalink
Merge branch '47-implement-loggers'
Browse files Browse the repository at this point in the history
  • Loading branch information
barneydobson committed Mar 7, 2024
2 parents ad9f039 + f736ee7 commit c972b32
Show file tree
Hide file tree
Showing 8 changed files with 151 additions and 12 deletions.
5 changes: 5 additions & 0 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ colorama==0.4.6
# via
# build
# click
# loguru
# pytest
# tqdm
contourpy==1.2.0
Expand Down Expand Up @@ -116,6 +117,8 @@ lazy-loader==0.3
# via scikit-image
llvmlite==0.41.1
# via numba
loguru==0.7.2
# via swmmanywhere (pyproject.toml)
matplotlib==3.8.2
# via
# salib
Expand Down Expand Up @@ -300,6 +303,8 @@ virtualenv==20.24.5
# via pre-commit
wheel==0.41.3
# via pip-tools
win32-setctime==1.1.0
# via loguru
xarray==2023.12.0
# via
# rioxarray
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ dependencies = [
"geopandas",
"geopy",
"GitPython",
"loguru",
"matplotlib",
"netcdf4",
"networkx",
Expand Down
5 changes: 5 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ cligj==0.7.2
colorama==0.4.6
# via
# click
# loguru
# tqdm
contourpy==1.2.0
# via matplotlib
Expand Down Expand Up @@ -94,6 +95,8 @@ lazy-loader==0.3
# via scikit-image
llvmlite==0.41.1
# via numba
loguru==0.7.2
# via swmmanywhere (pyproject.toml)
matplotlib==3.8.2
# via
# salib
Expand Down Expand Up @@ -237,6 +240,8 @@ tzdata==2024.1
# via pandas
urllib3==2.1.0
# via requests
win32-setctime==1.1.0
# via loguru
xarray==2023.12.0
# via
# rioxarray
Expand Down
3 changes: 2 additions & 1 deletion swmmanywhere/graph_utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

from swmmanywhere import geospatial_utilities as go
from swmmanywhere import parameters
from swmmanywhere.logging import logger


def load_graph(fid: Path) -> nx.Graph:
Expand Down Expand Up @@ -933,7 +934,7 @@ def process_successors(G: nx.Graph,
edge_diams[(node,ds_node,0)] = diam
chamber_floor[ds_node] = surface_elevations[ds_node] - depth
if ix > 0:
print('''a node has multiple successors,
logger.warning('''a node has multiple successors,
not sure how that can happen if using shortest path
to derive topology''')

Expand Down
55 changes: 55 additions & 0 deletions swmmanywhere/logging.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
"""Created on 2024-03-04.
@author: Barney
"""
import os
import sys

import loguru


def dynamic_filter(record):
"""A dynamic filter."""
return os.getenv("SWMMANYWHERE_VERBOSE", "false").lower() == "true"

def get_logger() -> loguru.logger:
"""Get a logger."""
logger = loguru.logger
logger.configure(
handlers=[
{
"sink": sys.stdout,
"filter" : dynamic_filter,
"colorize": True,
"format": " | ".join(
[
"<cyan>{time:YYYY/MM/DD HH:mm:ss}</>",
"{message}",
]
),
}
]
)
return logger

# Get the logger
logger = get_logger()

# Add a test_logger method to the logger
logger.test_logger = lambda : logger.info("This is a test message.")

# Store the original add method
original_add = logger.add

# Define a new function that wraps the original add method
def new_add(sink, **kwargs):
"""A new add method to wrap existing one but with the filter."""
# Include the dynamic filter in the kwargs if not already specified
if 'filter' not in kwargs:
kwargs['filter'] = dynamic_filter
# Call the original add method with the updated kwargs
return original_add(sink, **kwargs)

# Replace the logger's add method with new_add
logger.add = new_add
11 changes: 6 additions & 5 deletions swmmanywhere/prepare_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import yaml
from geopy.geocoders import Nominatim

# Some minor comment (to remove)
from swmmanywhere.logging import logger


def get_country(x: float,
y: float) -> dict[int, str]:
Expand Down Expand Up @@ -83,9 +84,9 @@ def download_buildings(file_address: Path,
# Save data to the specified file address
with file_address.open("wb") as file:
file.write(response.content)
print(f"Data downloaded and saved to {file_address}")
logger.info(f"Data downloaded and saved to {file_address}")
else:
print(f"Error downloading data. Status code: {response.status_code}")
logger.error(f"Error downloading data. Status code: {response.status_code}")
return response.status_code

def download_street(bbox: tuple[float, float, float, float]) -> nx.MultiDiGraph:
Expand Down Expand Up @@ -174,10 +175,10 @@ def download_elevation(fid: Path,
with fid.open('wb') as rast_file:
shutil.copyfileobj(r.raw, rast_file)

print('Elevation data downloaded successfully.')
logger.info('Elevation data downloaded successfully.')

except requests.exceptions.RequestException as e:
print(f'Error downloading elevation data: {e}')
logger.error(f'Error downloading elevation data: {e}')

return r.status_code

Expand Down
13 changes: 7 additions & 6 deletions swmmanywhere/preprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from swmmanywhere import geospatial_utilities as go
from swmmanywhere import graph_utilities as gu
from swmmanywhere import parameters, prepare_data
from swmmanywhere.logging import logger


def next_directory(keyword: str, directory: Path) -> int:
Expand Down Expand Up @@ -160,7 +161,7 @@ def prepare_precipitation(bbox: tuple[float, float, float, float],
"""Download and reproject precipitation data."""
if addresses.precipitation.exists():
return
print(f'downloading precipitation to {addresses.precipitation}')
logger.info(f'downloading precipitation to {addresses.precipitation}')
precip = prepare_data.download_precipitation(bbox,
api_keys['cds_username'],
api_keys['cds_api_key'])
Expand All @@ -175,7 +176,7 @@ def prepare_elvation(bbox: tuple[float, float, float, float],
"""Download and reproject elevation data."""
if addresses.elevation.exists():
return
print(f'downloading elevation to {addresses.elevation}')
logger.info(f'downloading elevation to {addresses.elevation}')
with tempfile.TemporaryDirectory() as temp_dir:
fid = Path(temp_dir) / 'elevation.tif'
prepare_data.download_elevation(fid,
Expand All @@ -194,12 +195,12 @@ def prepare_building(bbox: tuple[float, float, float, float],
return

if not addresses.national_building.exists():
print(f'downloading buildings to {addresses.national_building}')
logger.info(f'downloading buildings to {addresses.national_building}')
prepare_data.download_buildings(addresses.national_building,
bbox[0],
bbox[1])

print(f'trimming buildings to {addresses.building}')
logger.info(f'trimming buildings to {addresses.building}')
national_buildings = gpd.read_parquet(addresses.national_building)
buildings = national_buildings.cx[bbox[0]:bbox[2], bbox[1]:bbox[3]] # type: ignore

Expand All @@ -213,7 +214,7 @@ def prepare_street(bbox: tuple[float, float, float, float],
"""Download and reproject street graph."""
if addresses.street.exists():
return
print(f'downloading street network to {addresses.street}')
logger.info(f'downloading street network to {addresses.street}')
street_network = prepare_data.download_street(bbox)
street_network = go.reproject_graph(street_network,
source_crs,
Expand All @@ -227,7 +228,7 @@ def prepare_river(bbox: tuple[float, float, float, float],
"""Download and reproject river graph."""
if addresses.river.exists():
return
print(f'downloading river network to {addresses.river}')
logger.info(f'downloading river network to {addresses.river}')
river_network = prepare_data.download_river(bbox)
river_network = go.reproject_graph(river_network,
source_crs,
Expand Down
70 changes: 70 additions & 0 deletions tests/test_logging.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
"""Created on 2024-01-26.
@author: Barney
"""
import os
from pathlib import Path
from tempfile import NamedTemporaryFile

from swmmanywhere.logging import logger


def test_logger():
"""Test logger."""
os.environ["SWMMANYWHERE_VERBOSE"] = "true"
assert logger is not None
logger.test_logger()
logger.debug("This is a debug message.")
logger.info("This is an info message.")
logger.warning("This is a warning message.")
logger.error("This is an error message.")
logger.critical("This is a critical message.")
with NamedTemporaryFile(suffix='.log',
mode = 'w+b',
delete=False) as temp_file:
fid = Path(temp_file.name)
logger.add(fid)
logger.test_logger()
assert temp_file.read() != b""
logger.remove()
fid.unlink()

def test_logger_disable():
"""Test the disable function."""
with NamedTemporaryFile(suffix='.log',
mode = 'w+b',
delete=False) as temp_file:
fid = Path(temp_file.name)
os.environ["SWMMANYWHERE_VERBOSE"] = "false"
logger.add(fid)
logger.test_logger()
assert temp_file.read() == b""
logger.remove()
fid.unlink()

def test_logger_reimport():
"""Reimport logger to check that changes from disable are persistent."""
from swmmanywhere.logging import logger
with NamedTemporaryFile(suffix='.log',
mode = 'w+b',
delete=False) as temp_file:
fid = Path(temp_file.name)
logger.add(fid)
logger.test_logger()
assert temp_file.read() == b""
logger.remove()
fid.unlink()

def test_logger_again():
"""Test the logger after these changes to make sure still works."""
os.environ["SWMMANYWHERE_VERBOSE"] = "true"
with NamedTemporaryFile(suffix='.log',
mode = 'w+b',
delete=False) as temp_file:
fid = Path(temp_file.name)
logger.add(fid)
logger.test_logger()
assert temp_file.read() != b""
logger.remove()
fid.unlink()

0 comments on commit c972b32

Please sign in to comment.