[ngrams] Speed up bz2 decoding

Decode at once.
behdad · Nov 10, 2023 · 5e14c60 · 5e14c60
1 parent 7605b0d
commit 5e14c60
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/ngrams.py b/ngrams.py
@@ -61,8 +61,8 @@ def extract_ngrams_from_file(filename, *kargs, **kwargs):
         import bz2
 
         # Assume harfbuzz-testing-wikipedia format
-        txtfile = bz2.open(filename + ".txt.bz2")
-        frqfile = bz2.open(filename + ".frq.bz2")
+        txtfile = bz2.open(filename + ".txt.bz2").read().splitlines()
+        frqfile = bz2.open(filename + ".frq.bz2").read().splitlines()
 
     return extract_ngrams(txtfile, *kargs, frequencies=frqfile, **kwargs)