3.1.1

docs/source/changelog/changelog_3.0.rst

@@ -5,6 +5,15 @@
 3.0 Changelog
 *************
 
+3.1.1
+-----
+
+- Fixed a bug where hidden files and folders would be parsed as corpus data
+- Fixed a bug where validation would not respect :code:`--no_final_clean`
+- Fixed a rare crash in training when a job would not have utterances assigned to it
+- Fixed a bug where MFA would mistakenly report a dictionary and acoustic model phones did not match for older versions
+
+
 3.1.0
 -----
 

montreal_forced_aligner/abc.py

@@ -699,7 +699,7 @@ def cleanup(self) -> None:
                 logger.info(f"Done! Everything took {time.time() - self.start_time:.3f} seconds")
                 if config.FINAL_CLEAN:
                     logger.debug(
-                        "Cleaning up temporary files, use the --debug flag to keep temporary files."
+                        "Cleaning up temporary files, use the --no_final_clean flag to keep temporary files."
                     )
                     if hasattr(self, "delete_database"):
                         if config.USE_POSTGRES:

montreal_forced_aligner/acoustic_modeling/monophone.py

@@ -133,9 +133,10 @@ def _run(self):
                 writer.Close()
                 self.callback((accumulator.transition_accs, accumulator.gmm_accs))
                 train_logger.info(f"Done {num_done} utterances, errors on {num_error} utterances.")
-                train_logger.info(
-                    f"Overall avg like per frame (Gaussian only) = {tot_like/tot_t} over {tot_t} frames."
-                )
+                if tot_t:
+                    train_logger.info(
+                        f"Overall avg like per frame (Gaussian only) = {tot_like/tot_t} over {tot_t} frames."
+                    )
 
 
 class MonophoneTrainer(AcousticModelTrainingMixin):

montreal_forced_aligner/acoustic_modeling/triphone.py

@@ -411,11 +411,22 @@ def _setup_tree(self, init_from_previous=False, initial_mix_up=True) -> None:
             silence_sets = [
                 x for x in questions if silence_phone_id in x and x != [silence_phone_id]
             ]
+            filtered = []
+            existing_sets = {tuple(x) for x in questions}
             for q_set in silence_sets:
                 train_logger.debug(", ".join([self.reversed_phone_mapping[x] for x in q_set]))
-            questions = [
-                x for x in questions if silence_phone_id not in x or x == [silence_phone_id]
-            ]
+
+            for q_set in questions:
+                if silence_phone_id not in q_set or q_set == [silence_phone_id]:
+                    filtered.append(q_set)
+                    continue
+                q_set = [x for x in q_set if x != silence_phone_id]
+                if not q_set:
+                    continue
+                if tuple(q_set) in existing_sets:
+                    continue
+                filtered.append(q_set)
+            questions = filtered
 
             extra_questions = self.worker.extra_questions_mapping
             if extra_questions:

montreal_forced_aligner/alignment/base.py

@@ -1449,6 +1449,7 @@ def evaluate_alignments(
                 align_phones,
                 silence_phone=self.optional_silence_phone,
                 custom_mapping=mapping,
+                debug=config.DEBUG,
             )
             unaligned_utts = []
             utterances: typing.List[Utterance] = session.query(Utterance).options(

montreal_forced_aligner/command_line/align.py

@@ -4,7 +4,6 @@
 from pathlib import Path
 
 import rich_click as click
-import yaml
 
 from montreal_forced_aligner import config
 from montreal_forced_aligner.alignment import PretrainedAligner
@@ -15,7 +14,7 @@
     validate_g2p_model,
 )
 from montreal_forced_aligner.data import WorkflowType
-from montreal_forced_aligner.helper import mfa_open
+from montreal_forced_aligner.helper import load_evaluation_mapping
 
 __all__ = ["align_corpus_cli"]
 
@@ -137,8 +136,7 @@ def align_corpus_cli(context, **kwargs) -> None:
             aligner.load_reference_alignments(reference_directory)
             mapping = None
             if custom_mapping_path:
-                with mfa_open(custom_mapping_path, "r") as f:
-                    mapping = yaml.load(f, Loader=yaml.Loader)
+                mapping = load_evaluation_mapping(custom_mapping_path)
                 aligner.validate_mapping(mapping)
             reference_alignments = WorkflowType.reference
         else:

montreal_forced_aligner/command_line/validate.py

@@ -104,6 +104,7 @@ def validate_corpus_cli(context, **kwargs) -> None:
     """
     if kwargs.get("profile", None) is not None:
         config.profile = kwargs.pop("profile")
+    config.FINAL_CLEAN = True
     config.update_configuration(kwargs)
     kwargs["USE_THREADING"] = False
 
@@ -139,7 +140,6 @@ def validate_corpus_cli(context, **kwargs) -> None:
         validator.dirty = True
         raise
     finally:
-        config.FINAL_CLEAN = True
         validator.cleanup()
 
 

montreal_forced_aligner/corpus/acoustic_corpus.py

@@ -234,8 +234,12 @@ def load_reference_alignments(self, reference_directory: Path) -> None:
                     max_id = p_id
             new_phones = []
             for root, _, files in os.walk(reference_directory, followlinks=True):
+                if root.startswith("."):  # Ignore hidden directories
+                    continue
                 root_speaker = os.path.basename(root)
                 for f in files:
+                    if f.startswith("."):  # Ignore hidden files
+                        continue
                     if f.endswith(".TextGrid"):
                         file_name = f.replace(".TextGrid", "")
                         file_id = session.query(File.id).filter_by(name=file_name).scalar()
@@ -988,6 +992,8 @@ def _load_corpus_from_source(self) -> None:
             for root, _, files in os.walk(self.audio_directory, followlinks=True):
                 if self.stopped.is_set():
                     return
+                if root.startswith("."):  # Ignore hidden directories
+                    continue
                 exts = find_exts(files)
                 exts.wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}
                 exts.other_audio_files = {
@@ -999,12 +1005,14 @@ def _load_corpus_from_source(self) -> None:
         with self.session() as session:
             import_data = DatabaseImportData()
             for root, _, files in os.walk(self.corpus_directory, followlinks=True):
+                if self.stopped.is_set():
+                    return
+                if root.startswith("."):  # Ignore hidden directories
+                    continue
                 exts = find_exts(files)
                 relative_path = (
                     root.replace(str(self.corpus_directory), "").lstrip("/").lstrip("\\")
                 )
-                if self.stopped.is_set():
-                    return
                 if not use_audio_directory:
                     all_sound_files = {}
                     wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}

montreal_forced_aligner/corpus/helper.py

@@ -62,6 +62,8 @@ def find_exts(files: typing.List[str]) -> FileExtensions:
     """
     exts = FileExtensions(set(), {}, {}, {}, {})
     for full_filename in files:
+        if full_filename.startswith("."):  # Ignore hidden files
+            continue
         try:
             filename, fext = full_filename.rsplit(".", maxsplit=1)
         except ValueError:

montreal_forced_aligner/corpus/multiprocessing.py

@@ -106,6 +106,8 @@ def run(self) -> None:
         if self.audio_directory and os.path.exists(self.audio_directory):
             use_audio_directory = True
             for root, _, files in os.walk(self.audio_directory, followlinks=True):
+                if root.startswith("."):  # Ignore hidden directories
+                    continue
                 exts = find_exts(files)
                 wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}
                 other_audio_files = {
@@ -114,11 +116,12 @@ def run(self) -> None:
                 all_sound_files.update(other_audio_files)
                 all_sound_files.update(wav_files)
         for root, _, files in os.walk(self.corpus_directory, followlinks=True):
-            exts = find_exts(files)
-            relative_path = root.replace(str(self.corpus_directory), "").lstrip("/").lstrip("\\")
-
             if self.stopped.is_set():
                 break
+            if root.startswith("."):  # Ignore hidden directories
+                continue
+            exts = find_exts(files)
+            relative_path = root.replace(str(self.corpus_directory), "").lstrip("/").lstrip("\\")
             if not use_audio_directory:
                 all_sound_files = {}
                 exts.wav_files = {k: os.path.join(root, v) for k, v in exts.wav_files.items()}

montreal_forced_aligner/corpus/text_corpus.py

@@ -65,13 +65,14 @@ def _load_corpus_from_source_mp(self) -> None:
             file_count = 0
             with tqdm(total=1, disable=config.QUIET) as pbar, self.session() as session:
                 for root, _, files in os.walk(self.corpus_directory, followlinks=True):
+                    if self.stopped.is_set():
+                        break
+                    if root.startswith("."):  # Ignore hidden directories
+                        continue
                     exts = find_exts(files)
                     relative_path = (
                         root.replace(str(self.corpus_directory), "").lstrip("/").lstrip("\\")
                     )
-
-                    if self.stopped.is_set():
-                        break
                     for file_name in exts.identifiers:
                         if self.stopped.is_set():
                             break
@@ -181,12 +182,14 @@ def _load_corpus_from_source(self) -> None:
         sanitize_function = getattr(self, "sanitize_function", None)
         with self.session() as session:
             for root, _, files in os.walk(self.corpus_directory, followlinks=True):
+                if self.stopped:
+                    return
+                if root.startswith("."):  # Ignore hidden directories
+                    continue
                 exts = find_exts(files)
                 relative_path = (
                     root.replace(str(self.corpus_directory), "").lstrip("/").lstrip("\\")
                 )
-                if self.stopped:
-                    return
                 for file_name in exts.identifiers:
                     wav_path = None
                     if file_name in exts.lab_files:

montreal_forced_aligner/dictionary/mixins.py

@@ -260,11 +260,7 @@ def get_base_phone(self, phone: str) -> str:
     @property
     def extra_questions_mapping(self) -> Dict[str, List[str]]:
         """Mapping of extra questions for the given phone set type"""
-        mapping = {"silence_question": []}
-        for p in sorted(self.silence_phones):
-            mapping["silence_question"].append(p)
-            if self.position_dependent_phones:
-                mapping["silence_question"].extend([p + x for x in self.positions])
+        mapping = {}
         for k, v in self.phone_set_type.extra_questions.items():
             if k not in mapping:
                 mapping[k] = []
@@ -427,7 +423,8 @@ def _generate_positional_list(self, phones: Set[str]) -> List[str]:
             List of positional phones, sorted by base phone
         """
         positional_phones = []
-        phones |= {self.get_base_phone(p) for p in phones}
+        if not hasattr(self, "acoustic_model"):
+            phones |= {self.get_base_phone(p) for p in phones}
         for p in sorted(phones):
             if p not in self.non_silence_phones:
                 continue

montreal_forced_aligner/dictionary/multispeaker.py

@@ -152,7 +152,7 @@ def tokenizers(self):
                         bracketed_word=self.bracketed_word,
                         cutoff_word=self.cutoff_word,
                         ignore_case=self.ignore_case,
-                        use_g2p=self.use_g2p,
+                        use_g2p=self.use_g2p or getattr(self, "g2p_model", None) is not None,
                         clitic_set=clitic_set,
                         grapheme_set=grapheme_set,
                     )