diff --git a/src/input_streams.rs b/src/input_streams.rs
index 28afafe4..aad7a0e9 100644
--- a/src/input_streams.rs
+++ b/src/input_streams.rs
@@ -30,6 +30,14 @@ pub(crate) fn read_logical_key_changes<'a>(
             caches.insert(logical_key.clone());
         } else {
             caches.remove(logical_key);
+
+            // TODO: These magic numbers are just intuitive guesses, not tested or validated.
+            // Avoid memory leaking
+            let len = caches.len();
+            let capacity = caches.capacity();
+            if capacity > 8 * len {
+                caches.shrink_to(2 * len);
+            }
         }
     }
 }