diff --git a/CHANGELOG.md b/CHANGELOG.md
index efd717a73c..27bd4f7989 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 * Fixed incompatible rendering of empty hunks in git/unified diffs.
   [#5049](https://github.com/martinvonz/jj/issues/5049)
 
+* Fixed performance of progress bar rendering when fetching from Git remote.
+  [#5057](https://github.com/martinvonz/jj/issues/5057)
+
 ## [0.24.0] - 2024-12-04
 
 ### Release highlights
diff --git a/cli/src/progress.rs b/cli/src/progress.rs
index 55b4dc0be1..991d2ea1a5 100644
--- a/cli/src/progress.rs
+++ b/cli/src/progress.rs
@@ -53,6 +53,7 @@ impl Progress {
         if now < self.next_print {
             return Ok(());
         }
+        self.next_print = now + Duration::from_secs(1) / UPDATE_HZ;
         if self.guard.is_none() {
             let guard = output.output_guard(crossterm::cursor::Show.to_string());
             let guard = CleanupGuard::new(move || {
@@ -61,7 +62,6 @@ impl Progress {
             _ = write!(output, "{}", crossterm::cursor::Hide);
             self.guard = Some(guard);
         }
-        self.next_print = now.min(self.next_print + Duration::from_secs(1) / UPDATE_HZ);
 
         self.buffer.clear();
         write!(self.buffer, "\r").unwrap();
@@ -260,14 +260,14 @@ mod tests {
         // First output is after the initial delay
         assert_snapshot!(update(INITIAL_DELAY - Duration::from_millis(1), 0.1), @"");
         assert_snapshot!(update(Duration::from_millis(1), 0.10), @"[?25l\r 10% [█▊                ]");
-        // TODO: No updates for the next 30 milliseconds
-        assert_snapshot!(update(Duration::from_millis(10), 0.11), @" 11% [██                ]");
-        assert_snapshot!(update(Duration::from_millis(10), 0.12), @" 12% [██▏               ]");
-        assert_snapshot!(update(Duration::from_millis(10), 0.13), @" 13% [██▍               ]");
+        // No updates for the next 30 milliseconds
+        assert_snapshot!(update(Duration::from_millis(10), 0.11), @"");
+        assert_snapshot!(update(Duration::from_millis(10), 0.12), @"");
+        assert_snapshot!(update(Duration::from_millis(10), 0.13), @"");
         // We get an update now that we go over the threshold
         assert_snapshot!(update(Duration::from_millis(100), 0.30), @" 30% [█████▍            ]");
-        // TODO: Even though we went over by quite a bit, the new threshold is relative
-        // to the previous output, so we don't get an update here
-        assert_snapshot!(update(Duration::from_millis(30), 0.40), @" 40% [███████▎          ]");
+        // Even though we went over by quite a bit, the new threshold is relative to the
+        // previous output, so we don't get an update here
+        assert_snapshot!(update(Duration::from_millis(30), 0.40), @"");
     }
 }