diff --git a/torchode/step_size_controllers.py b/torchode/step_size_controllers.py
index 06fa66c..b6b7a2e 100644
--- a/torchode/step_size_controllers.py
+++ b/torchode/step_size_controllers.py
@@ -478,7 +478,7 @@ def _select_initial_step(
             args,
         )
 
-        d2 = norm((f1 - f0) * inv_scale) / dt0
+        d2 = torch.where(dt0 == 0, torch.inf, norm((f1 - f0) * inv_scale) / dt0)
 
         maxd1d2 = torch.maximum(d1, d2)
         dt1 = torch.where(