Skip to content

Commit

Permalink
relax progressive checks for categorical dtypes
Browse files Browse the repository at this point in the history
  • Loading branch information
jpn-- committed Feb 2, 2024
1 parent a9dd383 commit ccc195c
Showing 1 changed file with 29 additions and 4 deletions.
33 changes: 29 additions & 4 deletions activitysim/core/workflow/checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -958,14 +958,21 @@ def restore_from(self, location: Path, checkpoint_name: str = LAST_CHECKPOINT):
self.load(checkpoint_name, store=from_store)
logger.debug(f"checkpoint.restore_from of {checkpoint_name} complete")

def check_against(self, location: Path, checkpoint_name: str):
def check_against(
self, location: Path, checkpoint_name: str, strict_categoricals: bool = False
):
"""
Check that the tables in this State match those in an archived pipeline.
Parameters
----------
location : Path-like
checkpoint_name : str
strict_categoricals : bool, default False
If True, check that categorical columns have the same categories
in both the current state and the checkpoint. Otherwise, the dtypes
of categorical columns are ignored, and only the values themselves are
checked to confirm they match.
Raises
------
Expand Down Expand Up @@ -1033,9 +1040,27 @@ def check_against(self, location: Path, checkpoint_name: str):
local_table[ref_table.columns], ref_table, check_dtype=False
)
except Exception as err:
raise AssertionError(
f"checkpoint {checkpoint_name!r} table {table_name!r}, {str(err)}"
)
if not strict_categoricals:
try:
pd.testing.assert_frame_equal(
local_table[ref_table.columns],
ref_table,
check_dtype=False,
check_categorical=False,
)
except Exception as err2:
raise AssertionError(
f"checkpoint {checkpoint_name!r} table {table_name!r}, {str(err)}"
)
else:
warnings.warn(
f"checkpoint {checkpoint_name!r} table {table_name!r}, "
f"values match but categorical dtype does not"
)
else:
raise AssertionError(
f"checkpoint {checkpoint_name!r} table {table_name!r}, {str(err)}"
)
else:
logger.info(f"table {table_name!r}: ok")

Expand Down

0 comments on commit ccc195c

Please sign in to comment.