diff --git a/crates/egui/src/containers/scroll_area.rs b/crates/egui/src/containers/scroll_area.rs
index d48e5950866..9b3eddf7db2 100644
--- a/crates/egui/src/containers/scroll_area.rs
+++ b/crates/egui/src/containers/scroll_area.rs
@@ -871,7 +871,10 @@ impl Prepared {
         let max_offset = content_size - inner_rect.size();
         let is_hovering_outer_rect = ui.rect_contains_pointer(outer_rect);
 
-        if scrolling_enabled {
+        if scrolling_enabled
+            && (is_hovering_outer_rect
+                || scroll_bar_visibility == ScrollBarVisibility::AlwaysVisible)
+        {
             let always_scroll_enabled_direction = ui.style().always_scroll_the_only_direction
                 && scroll_enabled[0] != scroll_enabled[1];
             for d in 0..2 {