Voice Cloning Support for fairseq

[tap222]

Hi,

Thank you so much for creating such a wonderful repository. I have a small question regarding voice cloning with Fairseq. Even though I am providing the speaker_wav, it doesn’t seem to be working.

I understand that voice conversion might be an alternative solution here. Could you confirm if voice cloning is currently supported or suggest how it can be achieved?

api = TTS("tts_models/ory/fairseq/vits")
api.tts_to_file(
text="ନମସ୍କାର",
file_path="output_ory.wav",
speaker_wav='./example_1.wav',
)

[eginhard]

The Fairseq models don't support voice cloning. You can only do TTS followed by voice conversion like this:

from TTS.api import TTS
api = TTS("tts_models/ory/fairseq/vits")
api.tts_with_vc_to_file(
  text="ନମସ୍କାର",
  file_path="output_ory.wav",
  speaker_wav='./example_1.wav',
)

[tap222]

Thanks a lot for the response, I am getting this error

---

RuntimeError Traceback (most recent call last)
Cell In[1], line 3
1 from TTS.api import TTS
2 api = TTS("tts_models/ory/fairseq/vits")
----> 3 api.tts_with_vc_to_file(
4 text="ନମସ୍କାର",
5 file_path="output_ory.wav",
6 speaker_wav='./example_1.wav',
7 )

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/api.py:455, in TTS.tts_with_vc_to_file(self, text, language, speaker_wav, file_path, speaker, split_sentences)
423 def tts_with_vc_to_file(
424 self,
425 text: str,
(...)
430 split_sentences: bool = True,
431 ):
432 """Convert text to speech with voice conversion and save to file.
433
434 Check tts_with_vc for more details.
(...)
453 applicable to the 🐸TTS models. Defaults to True.
454 """
--> 455 wav = self.tts_with_vc(
456 text=text, language=language, speaker_wav=speaker_wav, speaker=speaker, split_sentences=split_sentences
457 )
458 save_wav(wav=wav, path=file_path, sample_rate=self.voice_converter.vc_config.audio.output_sample_rate)

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/api.py:419, in TTS.tts_with_vc(self, text, language, speaker_wav, speaker, split_sentences)
415 self.tts_to_file(
416 text=text, speaker=speaker, language=language, file_path=fp.name, split_sentences=split_sentences
417 )
418 if self.voice_converter is None:
--> 419 self.load_vc_model_by_name("voice_conversion_models/multilingual/vctk/freevc24")
420 wav = self.voice_converter.voice_conversion(source_wav=fp.name, target_wav=speaker_wav)
421 return wav

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/api.py:159, in TTS.load_vc_model_by_name(self, model_name, gpu)
157 self.model_name = model_name
158 model_path, config_path, _, _, _ = self.download_model_by_name(model_name)
--> 159 self.voice_converter = Synthesizer(vc_checkpoint=model_path, vc_config=config_path, use_cuda=gpu)

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/utils/synthesizer.py:101, in Synthesizer.init(self, tts_checkpoint, tts_config_path, tts_speakers_file, tts_languages_file, vocoder_checkpoint, vocoder_config, encoder_checkpoint, encoder_config, vc_checkpoint, vc_config, model_dir, voice_dir, use_cuda)
98 self.output_sample_rate = self.vocoder_config.audio["sample_rate"]
100 if vc_checkpoint:
--> 101 self._load_vc(vc_checkpoint, vc_config, use_cuda)
102 self.output_sample_rate = self.vc_config.audio["output_sample_rate"]
104 if model_dir:

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/utils/synthesizer.py:139, in Synthesizer._load_vc(self, vc_checkpoint, vc_config_path, use_cuda)
137 # pylint: disable=global-statement
138 self.vc_config = load_config(vc_config_path)
--> 139 self.vc_model = setup_vc_model(config=self.vc_config)
140 self.vc_model.load_checkpoint(self.vc_config, vc_checkpoint)
141 if use_cuda:

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/vc/models/init.py:19, in setup_model(config, samples)
17 if "model" in config and config["model"].lower() == "freevc":
18 MyModel = importlib.import_module("TTS.vc.models.freevc").FreeVC
---> 19 model = MyModel.init_from_config(config, samples)
20 return model

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/vc/models/freevc.py:556, in FreeVC.init_from_config(config, samples)
554 @staticmethod
555 def init_from_config(config: FreeVCConfig, samples: Union[List[List], List[Dict]] = None):
--> 556 model = FreeVC(config)
557 return model

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/vc/models/freevc.py:377, in FreeVC.init(self, config, speaker_manager)
374 else:
375 self.load_pretrained_speaker_encoder()
--> 377 self.wavlm = get_wavlm()

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/TTS/vc/modules/freevc/wavlm/init.py:30, in get_wavlm(device)
27 logger.info("Downloading WavLM model to %s ...", output_path)
28 urllib.request.urlretrieve(model_uri, output_path)
---> 30 checkpoint = torch.load(output_path, map_location=torch.device(device), weights_only=is_pytorch_at_least_2_4())
31 cfg = WavLMConfig(checkpoint["cfg"])
32 wavlm = WavLM(cfg).to(device)

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/torch/serialization.py:1326, in load(f, map_location, pickle_module, weights_only, mmap, **pickle_load_args)
1324 orig_position = opened_file.tell()
1325 overall_storage = None
-> 1326 with _open_zipfile_reader(opened_file) as opened_zipfile:
1327 if _is_torchscript_zip(opened_zipfile):
1328 warnings.warn(
1329 "'torch.load' received a zip file that looks like a TorchScript archive"
1330 " dispatching to 'torch.jit.load' (call 'torch.jit.load' directly to"
1331 " silence this warning)",
1332 UserWarning,
1333 )

File ~/Downloads/voxmorph/Rhubarb-Lip-Sync/avtr/lib/python3.12/site-packages/torch/serialization.py:671, in _open_zipfile_reader.init(self, name_or_buffer)
670 def init(self, name_or_buffer) -> None:
--> 671 super().init(torch._C.PyTorchFileReader(name_or_buffer))

RuntimeError: PytorchStreamReader failed reading zip archive: failed finding central directory

[eginhard]

Looks like a download error, just try again. If it still doesn't work, you might have to delete the partially downloaded files first.

[tap222]

Really sorry to ask naive question but I am not able to locate the downloaded file

[eginhard]

You can find out the folder with:

import trainer
print(trainer.io.get_user_data_dir("tts"))

You can remove the freevc folder in there and try again.

[tap222]

Thanks a lot it works