[Schedulers] Fix 2nd order other than heun (huggingface#5526)

* [Schedulers] Fix 2nd order other than heun

* Apply suggestions from code review

pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py

@@ -898,9 +898,9 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N
             )
 
             num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()
-            if self.scheduler.order == 2:
-                # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1
-                # because `num_inference_steps` will always be even given that every timestep
+            if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
+                # if the scheduler is a 2nd order scheduler we might have to do +1
+                # because `num_inference_steps` might be even given that every timestep
                 # (except the highest one) is duplicated. If `num_inference_steps` is even it would
                 # mean that we cut the timesteps in the middle of the denoising step
                 # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1

pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py

@@ -555,9 +555,9 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N
             )
 
             num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()
-            if self.scheduler.order == 2:
-                # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1
-                # because `num_inference_steps` will always be even given that every timestep
+            if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
+                # if the scheduler is a 2nd order scheduler we might have to do +1
+                # because `num_inference_steps` might be even given that every timestep
                 # (except the highest one) is duplicated. If `num_inference_steps` is even it would
                 # mean that we cut the timesteps in the middle of the denoising step
                 # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1

pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py

@@ -840,9 +840,9 @@ def get_timesteps(self, num_inference_steps, strength, device, denoising_start=N
             )
 
             num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item()
-            if self.scheduler.order == 2:
-                # if the scheduler is a 2nd order scheduler we ALWAYS have to do +1
-                # because `num_inference_steps` will always be even given that every timestep
+            if self.scheduler.order == 2 and num_inference_steps % 2 == 0:
+                # if the scheduler is a 2nd order scheduler we might have to do +1
+                # because `num_inference_steps` might be even given that every timestep
                 # (except the highest one) is duplicated. If `num_inference_steps` is even it would
                 # mean that we cut the timesteps in the middle of the denoising step
                 # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1