diff --git a/altair/utils/core.py b/altair/utils/core.py
index 45033572a..f365ce962 100644
--- a/altair/utils/core.py
+++ b/altair/utils/core.py
@@ -588,7 +588,7 @@ def parse_shorthand(
                     column = dfi.get_column_by_name(unescaped_field)
                     try:
                         attrs["type"] = infer_vegalite_type_for_dfi_column(column)
-                    except (NotImplementedError, AttributeError):
+                    except (NotImplementedError, AttributeError, ValueError):
                         # Fall back to pandas-based inference.
                         # Note: The AttributeError catch is a workaround for
                         # https://github.com/pandas-dev/pandas/issues/55332