Finish up tokens

scripts/data_prep/convert_text_to_mds.py

@@ -94,6 +94,12 @@ def __iter__(self) -> Iterable[Dict[str, bytes]]:
                 # Add the EOS token to the buffer to separate files.
                 buffer += self.eos_tokens
 
+        # Finish up the last of the tokens.
+        while len(buffer) >= self.max_length:
+            concat_sample = buffer[:self.max_length]
+            buffer = buffer[self.max_length:] if self.should_wrap else []
+            yield {'tokens': np.asarray(concat_sample).tobytes()}
+
 
 def parse_args() -> Namespace:
     """Parse commandline arguments."""