diff --git a/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py
index 1423920191d7..46c9f25b6eb6 100644
--- a/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py
+++ b/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py
@@ -898,9 +898,9 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N
             )
 
             num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()
-            if self.scheduler.order == 2:
-                # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1
-                # because `num_inference_steps` will always be even given that every timestep
+            if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
+                # if the scheduler is a 2nd order scheduler we might have to do +1
+                # because `num_inference_steps` might be even given that every timestep
                 # (except the highest one) is duplicated. If `num_inference_steps` is even it would
                 # mean that we cut the timesteps in the middle of the denoising step
                 # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1
diff --git a/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py
index ff9d669a802b..57d00af82106 100644
--- a/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py
+++ b/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py
@@ -555,9 +555,9 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N
             )
 
             num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()
-            if self.scheduler.order == 2:
-                # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1
-                # because `num_inference_steps` will always be even given that every timestep
+            if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
+                # if the scheduler is a 2nd order scheduler we might have to do +1
+                # because `num_inference_steps` might be even given that every timestep
                 # (except the highest one) is duplicated. If `num_inference_steps` is even it would
                 # mean that we cut the timesteps in the middle of the denoising step
                 # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1
diff --git a/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py
index 200f5a7bf45a..11ae0a0d85f0 100644
--- a/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py
+++ b/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py
@@ -840,9 +840,9 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N
             )
 
             num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()
-            if self.scheduler.order == 2:
-                # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1
-                # because `num_inference_steps` will always be even given that every timestep
+            if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
+                # if the scheduler is a 2nd order scheduler we might have to do +1
+                # because `num_inference_steps` might be even given that every timestep
                 # (except the highest one) is duplicated. If `num_inference_steps` is even it would
                 # mean that we cut the timesteps in the middle of the denoising step
                 # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1