From 860d11ff7c4235e0baaeee50d96cf1686781bdd3 Mon Sep 17 00:00:00 2001 From: Wissam Antoun Date: Wed, 14 Jun 2023 18:24:53 +0200 Subject: [PATCH 01/15] Fix Debertav2 embed_proj (#24205) * MLM prediction head output size from embed_size Take the output size of the dense projection layer from embedding_size instead of hidden_size since there could be a projection of the input embedding into hidden_size if they are different * project TFDebertaV2 mlm output to embedding size embedding size can be different that hidden_size, so the final layer needs to project back to embedding size. like in ELECTRA or DeBERTaV3 style pertaining. This should solve an error that occurs when loading models like "almanach/camemberta-base-generator". * fix the same issue for reshaping after projection * fix layernorm size * add self.embedding_size to scope * fix embed_proj scope name * apply the same changes to TF Deberta * add the changes to deberta * added self.embedding_size instead of config.embedding_size * added the same change to debertav2 * added coppied from deberta to deberta2 model * config.embedding_size fix * black * fix deberta config name --- .../models/deberta/modeling_deberta.py | 11 ++++++----- .../models/deberta/modeling_tf_deberta.py | 15 +++++++++++---- .../models/deberta_v2/modeling_deberta_v2.py | 13 ++++++++----- .../models/deberta_v2/modeling_tf_deberta_v2.py | 15 +++++++++++---- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/transformers/models/deberta/modeling_deberta.py b/src/transformers/models/deberta/modeling_deberta.py index b2a4e8152234fd..9a0d43db3a0aec 100644 --- a/src/transformers/models/deberta/modeling_deberta.py +++ b/src/transformers/models/deberta/modeling_deberta.py @@ -1100,16 +1100,17 @@ def forward( ) -# copied from transformers.models.bert.BertPredictionHeadTransform with bert -> deberta class DebertaPredictionHeadTransform(nn.Module): def __init__(self, config): super().__init__() - self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) + + self.dense = nn.Linear(config.hidden_size, self.embedding_size) if isinstance(config.hidden_act, str): self.transform_act_fn = ACT2FN[config.hidden_act] else: self.transform_act_fn = config.hidden_act - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.LayerNorm = nn.LayerNorm(self.embedding_size, eps=config.layer_norm_eps) def forward(self, hidden_states): hidden_states = self.dense(hidden_states) @@ -1118,15 +1119,15 @@ def forward(self, hidden_states): return hidden_states -# copied from transformers.models.bert.BertLMPredictionHead with bert -> deberta class DebertaLMPredictionHead(nn.Module): def __init__(self, config): super().__init__() self.transform = DebertaPredictionHeadTransform(config) + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. - self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.decoder = nn.Linear(self.embedding_size, config.vocab_size, bias=False) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) diff --git a/src/transformers/models/deberta/modeling_tf_deberta.py b/src/transformers/models/deberta/modeling_tf_deberta.py index 5fc4ce783ccbe6..29c5a256d30599 100644 --- a/src/transformers/models/deberta/modeling_tf_deberta.py +++ b/src/transformers/models/deberta/modeling_tf_deberta.py @@ -726,7 +726,12 @@ def __init__(self, config, **kwargs): self.position_biased_input = getattr(config, "position_biased_input", True) self.initializer_range = config.initializer_range if self.embedding_size != config.hidden_size: - self.embed_proj = tf.keras.layers.Dense(config.hidden_size, use_bias=False) + self.embed_proj = tf.keras.layers.Dense( + config.hidden_size, + kernel_initializer=get_initializer(config.initializer_range), + name="embed_proj", + use_bias=False, + ) self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm") self.dropout = TFDebertaStableDropout(config.hidden_dropout_prob, name="dropout") @@ -820,8 +825,10 @@ class TFDebertaPredictionHeadTransform(tf.keras.layers.Layer): def __init__(self, config: DebertaConfig, **kwargs): super().__init__(**kwargs) + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) + self.dense = tf.keras.layers.Dense( - units=config.hidden_size, + units=self.embedding_size, kernel_initializer=get_initializer(config.initializer_range), name="dense", ) @@ -845,7 +852,7 @@ def __init__(self, config: DebertaConfig, input_embeddings: tf.keras.layers.Laye super().__init__(**kwargs) self.config = config - self.hidden_size = config.hidden_size + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) self.transform = TFDebertaPredictionHeadTransform(config, name="transform") @@ -875,7 +882,7 @@ def set_bias(self, value: tf.Variable): def call(self, hidden_states: tf.Tensor) -> tf.Tensor: hidden_states = self.transform(hidden_states=hidden_states) seq_length = shape_list(hidden_states)[1] - hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, self.hidden_size]) + hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, self.embedding_size]) hidden_states = tf.matmul(a=hidden_states, b=self.input_embeddings.weight, transpose_b=True) hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, seq_length, self.config.vocab_size]) hidden_states = tf.nn.bias_add(value=hidden_states, bias=self.bias) diff --git a/src/transformers/models/deberta_v2/modeling_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_deberta_v2.py index 434f3d208aecdd..1596ad4ffad42e 100644 --- a/src/transformers/models/deberta_v2/modeling_deberta_v2.py +++ b/src/transformers/models/deberta_v2/modeling_deberta_v2.py @@ -1199,16 +1199,18 @@ def forward( ) -# copied from transformers.models.bert.BertPredictionHeadTransform with bert -> deberta +# Copied from transformers.models.deberta.modeling_deberta.DebertaPredictionHeadTransform with Deberta->DebertaV2 class DebertaV2PredictionHeadTransform(nn.Module): def __init__(self, config): super().__init__() - self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) + + self.dense = nn.Linear(config.hidden_size, self.embedding_size) if isinstance(config.hidden_act, str): self.transform_act_fn = ACT2FN[config.hidden_act] else: self.transform_act_fn = config.hidden_act - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.LayerNorm = nn.LayerNorm(self.embedding_size, eps=config.layer_norm_eps) def forward(self, hidden_states): hidden_states = self.dense(hidden_states) @@ -1217,15 +1219,16 @@ def forward(self, hidden_states): return hidden_states -# copied from transformers.models.bert.BertLMPredictionHead with bert -> deberta +# Copied from transformers.models.deberta.modeling_deberta.DebertaLMPredictionHead with Deberta->DebertaV2 class DebertaV2LMPredictionHead(nn.Module): def __init__(self, config): super().__init__() self.transform = DebertaV2PredictionHeadTransform(config) + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. - self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.decoder = nn.Linear(self.embedding_size, config.vocab_size, bias=False) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) diff --git a/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py index d3c254f7ce6887..e53d86dcb83076 100644 --- a/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py +++ b/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py @@ -816,7 +816,12 @@ def __init__(self, config, **kwargs): self.position_biased_input = getattr(config, "position_biased_input", True) self.initializer_range = config.initializer_range if self.embedding_size != config.hidden_size: - self.embed_proj = tf.keras.layers.Dense(config.hidden_size, use_bias=False) + self.embed_proj = tf.keras.layers.Dense( + config.hidden_size, + kernel_initializer=get_initializer(config.initializer_range), + name="embed_proj", + use_bias=False, + ) self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm") self.dropout = TFDebertaV2StableDropout(config.hidden_dropout_prob, name="dropout") @@ -911,8 +916,10 @@ class TFDebertaV2PredictionHeadTransform(tf.keras.layers.Layer): def __init__(self, config: DebertaV2Config, **kwargs): super().__init__(**kwargs) + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) + self.dense = tf.keras.layers.Dense( - units=config.hidden_size, + units=self.embedding_size, kernel_initializer=get_initializer(config.initializer_range), name="dense", ) @@ -937,7 +944,7 @@ def __init__(self, config: DebertaV2Config, input_embeddings: tf.keras.layers.La super().__init__(**kwargs) self.config = config - self.hidden_size = config.hidden_size + self.embedding_size = getattr(config, "embedding_size", config.hidden_size) self.transform = TFDebertaV2PredictionHeadTransform(config, name="transform") @@ -967,7 +974,7 @@ def set_bias(self, value: tf.Variable): def call(self, hidden_states: tf.Tensor) -> tf.Tensor: hidden_states = self.transform(hidden_states=hidden_states) seq_length = shape_list(hidden_states)[1] - hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, self.hidden_size]) + hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, self.embedding_size]) hidden_states = tf.matmul(a=hidden_states, b=self.input_embeddings.weight, transpose_b=True) hidden_states = tf.reshape(tensor=hidden_states, shape=[-1, seq_length, self.config.vocab_size]) hidden_states = tf.nn.bias_add(value=hidden_states, bias=self.bias) From 26a2ec56d743bb232f93c75ec47164974064a20a Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 14 Jun 2023 12:44:09 -0400 Subject: [PATCH 02/15] Clean up old Accelerate checks (#24279) * Clean up old Accelerate checks * Put back imports --- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- src/transformers/modeling_utils.py | 23 +++---------- src/transformers/trainer.py | 32 ++++--------------- 4 files changed, 14 insertions(+), 45 deletions(-) diff --git a/setup.py b/setup.py index b395adf6f586e6..d6e213798dbb98 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ # 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py _deps = [ "Pillow", - "accelerate>=0.20.2", + "accelerate>=0.20.3", "av==9.2.0", # Latest version of PyAV (10.0.0) has issues with audio stream. "beautifulsoup4", "black~=23.1", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 64ec3c2a54008b..39d8d0cb30ca78 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -3,7 +3,7 @@ # 2. run `make deps_table_update`` deps = { "Pillow": "Pillow", - "accelerate": "accelerate>=0.20.2", + "accelerate": "accelerate>=0.20.3", "av": "av==9.2.0", "beautifulsoup4": "beautifulsoup4", "black": "black~=23.1", diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 08bf79a3ff7fc8..b0d52f10d93814 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -82,27 +82,17 @@ XLA_DOWNCAST_BF16 = os.environ.get("XLA_DOWNCAST_BF16", "0").upper() if is_accelerate_available(): - from accelerate import __version__ as accelerate_version from accelerate import dispatch_model, infer_auto_device_map, init_empty_weights from accelerate.utils import ( + check_tied_parameters_on_same_device, find_tied_parameters, + get_balanced_memory, load_offloaded_weights, offload_weight, save_offload_index, set_module_tensor_to_device, ) - if version.parse(accelerate_version) > version.parse("0.11.0"): - from accelerate.utils import get_balanced_memory - else: - get_balanced_memory = None - if version.parse(accelerate_version) > version.parse("0.19.0"): - from accelerate.utils import check_tied_parameters_on_same_device - else: - check_tied_parameters_on_same_device = None -else: - find_tied_parameters = None - if is_safetensors_available(): from safetensors import safe_open from safetensors.torch import load_file as safe_load_file @@ -2792,8 +2782,6 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P "If passing a string for `device_map`, please choose 'auto', 'balanced', 'balanced_low_0' or " "'sequential'." ) - elif device_map in ["balanced", "balanced_low_0"] and get_balanced_memory is None: - raise ValueError(f"`device_map={device_map}` requires a source install of Accelerate.") kwargs = {"no_split_module_classes": no_split_modules} if "special_dtypes" in inspect.signature(infer_auto_device_map).parameters: @@ -2803,7 +2791,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P "This model has some weights that should be kept in higher precision, you need to upgrade " "`accelerate` to properly deal with them (`pip install --upgrade accelerate`)." ) - if device_map != "sequential" and get_balanced_memory is not None: + if device_map != "sequential": max_memory = get_balanced_memory( model, dtype=target_dtype, @@ -2838,8 +2826,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P model.tie_weights() tied_params = find_tied_parameters(model) # check if we don't have tied param in different devices - if check_tied_parameters_on_same_device is not None: - check_tied_parameters_on_same_device(tied_params, device_map) + check_tied_parameters_on_same_device(tied_params, device_map) if from_tf: if resolved_archive_file.endswith(".index"): @@ -3031,7 +3018,7 @@ def _fix_key(key): missing_keys = list(set(expected_keys) - set(loaded_keys)) unexpected_keys = list(set(loaded_keys) - set(expected_keys)) - if find_tied_parameters is not None: + if is_accelerate_available(): tied_params = find_tied_parameters(model) else: tied_params = [] diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 502f380bf8db07..9429b12ff96fac 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -32,8 +32,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union -from tqdm.auto import tqdm - # Integrations must be imported before ML frameworks: # isort: off @@ -206,14 +204,9 @@ from peft import PeftModel -skip_first_batches = None if is_accelerate_available(): + from accelerate import Accelerator, skip_first_batches from accelerate import __version__ as accelerate_version - - if version.parse(accelerate_version) >= version.parse("0.16"): - from accelerate import skip_first_batches - - from accelerate import Accelerator from accelerate.utils import DistributedDataParallelKwargs if version.parse(accelerate_version) > version.parse("0.20.3"): @@ -322,6 +315,7 @@ class Trainer: """ + # Those are used as methods of the Trainer in examples. from .trainer_pt_utils import _get_learning_rate, log_metrics, metrics_format, save_metrics, save_state def __init__( @@ -1714,22 +1708,10 @@ def _inner_training_loop( logger.info(f" Continuing training from epoch {epochs_trained}") logger.info(f" Continuing training from global step {self.state.global_step}") if not args.ignore_data_skip: - if skip_first_batches is None: - logger.info( - f" Will skip the first {epochs_trained} epochs then the first" - f" {steps_trained_in_current_epoch} batches in the first epoch. If this takes a lot of time," - " you can install the latest version of Accelerate with `pip install -U accelerate`.You can" - " also add the `--ignore_data_skip` flag to your launch command, but you will resume the" - " training on data already seen by your model." - ) - else: - logger.info( - f" Will skip the first {epochs_trained} epochs then the first" - f" {steps_trained_in_current_epoch} batches in the first epoch." - ) - if self.is_local_process_zero() and not args.disable_tqdm and skip_first_batches is None: - steps_trained_progress_bar = tqdm(total=steps_trained_in_current_epoch) - steps_trained_progress_bar.set_description("Skipping the first batches") + logger.info( + f" Will skip the first {epochs_trained} epochs then the first" + f" {steps_trained_in_current_epoch} batches in the first epoch." + ) # Update the references self.callback_handler.model = self.model @@ -1787,7 +1769,7 @@ def _inner_training_loop( rng_to_sync = False steps_skipped = 0 - if skip_first_batches is not None and steps_trained_in_current_epoch > 0: + if steps_trained_in_current_epoch > 0: epoch_iterator = skip_first_batches(epoch_iterator, steps_trained_in_current_epoch) steps_skipped = steps_trained_in_current_epoch steps_trained_in_current_epoch = 0 From 0c3fdccf2f271fb7c44f6ea6e9f4ee234795f2c5 Mon Sep 17 00:00:00 2001 From: Matthijs Hollemans Date: Wed, 14 Jun 2023 18:57:23 +0200 Subject: [PATCH 03/15] [WIP] add EnCodec model (#23655) * boilerplate stuff * messing around with the feature extractor * fix feature extractor * unit tests for feature extractor * rename speech to audio * quick-and-dirty import of Meta's code * import weights (sort of) * cleaning up * more cleaning up * move encoder/decoder args into config * cleanup model * rename EnCodec -> Encodec * RVQ parameters in config * add slow test * add lstm init and test_init * Add save & load * finish EncodecModel * remove decoder_input_values as they are ont used anywhere (not removed from doc yet) * fix test feature extraction model name * Add better slow test * Fix tests * some fixup and cleaning * Improve further * cleaning up quantizer * fix up conversion script * test don't pass, _encode_fram does not work * update tests with output per encode and decode * more cleanup * rename _codebook * remove old config cruft * ratios & hop_length * use ModuleList instead of Sequential * clean up resnet block * update types * update tests * fixup * quick cleanup * fix padding * more styl,ing * add patrick feedback * fix copies * fixup * fix lstm * fix shape issues * fixup * rename conv layers * fixup * fix decoding * small conv refactoring * remove norm_params * simplify conv layers * rename conv layers * stuff * Clean up * Add padding logic use padding mask small conv refactoring remove norm_params simplify conv layers rename conv layers stuff add batched test update Clean up merge and update for padding fix padding fixup * clean up more * clean up more * More clean ups * cleanup convolutions * typo * fix typos * fixup * build PR doc? * start refactoring docstring * fix don't pad when no strid and chunk * update docstring * update docstring * nits * update going to lunch * update config and model * fix broken testse (becaue of the config changes) * fix scale computation * fixu[ * only return dict if speciefied or if config returns it * remove todos * update defaults in config * update conversion script * fix doctest * more docstring + fixup * nits on batched_tests * more nits * Apply suggestions from code review Co-authored-by: Patrick von Platen * update basxed on review * fix update * updaet tests * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * fixup * add overlap and chunl_length_s * cleanup feature extraction * teste edge cases truncation and padding * correct processor values * update config encodec, nits * fix tests * fixup * fix 24Hz test * elle tests are green * fix fixup * Apply suggestions from code review * revert readme changes * fixup * add example * use facebook checkpoints * fix typo * no pipeline tests * use slef.pad everywhere we can * Apply suggestions from code review Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * update based on review * update * update mdx * fix bug and tests * fixup * fix doctest * remove comment * more nits * add more coverage for `test_truncation_and_padding` * fixup * add last test * fix text * nits * Update tests/models/encodec/test_modeling_encodec.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * take care of the last comments * typo * fix test * nits * fixup * Update src/transformers/models/encodec/feature_extraction_encodec.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --------- Co-authored-by: Patrick von Platen Co-authored-by: arthur.zucker@gmail.com Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/encodec.mdx | 59 ++ src/transformers/__init__.py | 22 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + .../models/auto/feature_extraction_auto.py | 1 + src/transformers/models/auto/modeling_auto.py | 1 + src/transformers/models/encodec/__init__.py | 65 ++ .../models/encodec/configuration_encodec.py | 189 ++++ .../convert_encodec_checkpoint_to_pytorch.py | 352 ++++++++ .../encodec/feature_extraction_encodec.py | 206 +++++ .../models/encodec/modeling_encodec.py | 808 ++++++++++++++++++ src/transformers/utils/dummy_pt_objects.py | 17 + .../utils/dummy_speech_objects.py | 7 + tests/models/encodec/__init__.py | 0 .../test_feature_extraction_encodec.py | 255 ++++++ tests/models/encodec/test_modeling_encodec.py | 571 +++++++++++++ utils/check_config_attributes.py | 2 + utils/documentation_tests.txt | 2 + 27 files changed, 2572 insertions(+) create mode 100644 docs/source/en/model_doc/encodec.mdx create mode 100644 src/transformers/models/encodec/__init__.py create mode 100644 src/transformers/models/encodec/configuration_encodec.py create mode 100644 src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py create mode 100644 src/transformers/models/encodec/feature_extraction_encodec.py create mode 100644 src/transformers/models/encodec/modeling_encodec.py create mode 100644 tests/models/encodec/__init__.py create mode 100644 tests/models/encodec/test_feature_extraction_encodec.py create mode 100644 tests/models/encodec/test_modeling_encodec.py diff --git a/README.md b/README.md index b5ce283be4653c..7d6ea278788096 100644 --- a/README.md +++ b/README.md @@ -346,6 +346,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (from Meta AI) released with the paper [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) by Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. diff --git a/README_es.md b/README_es.md index 32f955dbdab0a5..76270b2c7cd91f 100644 --- a/README_es.md +++ b/README_es.md @@ -321,6 +321,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (from Meta AI) released with the paper [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) by Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. diff --git a/README_hd.md b/README_hd.md index 2195e7cc638d17..02fdadf751cc78 100644 --- a/README_hd.md +++ b/README_hd.md @@ -293,6 +293,7 @@ conda install -c huggingface transformers 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (Google रिसर्च/स्टैनफोर्ड यूनिवर्सिटी से) साथ में दिया गया पेपर [इलेक्ट्रा: जेनरेटर के बजाय भेदभाव करने वाले के रूप में टेक्स्ट एन्कोडर्स का पूर्व-प्रशिक्षण] (https://arxiv.org/abs/2003.10555) केविन क्लार्क, मिन्ह-थांग लुओंग, क्वोक वी. ले, क्रिस्टोफर डी. मैनिंग द्वारा पोस्ट किया गया। +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (Meta AI से) Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. द्वाराअनुसंधान पत्र [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) के साथ जारी किया गया 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (Google रिसर्च से) साथ में दिया गया पेपर [सीक्वेंस जेनरेशन टास्क के लिए प्री-ट्रेंड चेकपॉइंट का इस्तेमाल करना](https:/ /arxiv.org/abs/1907.12461) साशा रोठे, शशि नारायण, अलियाक्सि सेवेरिन द्वारा। 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)**(Baidu से) साथ देने वाला पेपर [ERNIE: एन्हांस्ड रिप्रेजेंटेशन थ्रू नॉलेज इंटीग्रेशन](https://arxiv.org/abs/1904.09223) यू सन, शुओहुआन वांग, युकुन ली, शिकुन फेंग, ज़ुई चेन, हान झांग, शिन तियान, डैनक्सियांग झू, हाओ तियान, हुआ वू द्वारा पोस्ट किया गया। 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (Baidu से) Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. द्वाराअनुसंधान पत्र [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) के साथ जारी किया गया diff --git a/README_ja.md b/README_ja.md index 2331bb06cf551f..2116fee408c7a9 100644 --- a/README_ja.md +++ b/README_ja.md @@ -355,6 +355,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (Snap Research から) Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. から公開された研究論文 [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (Google Research/Stanford University から) Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning から公開された研究論文: [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (Meta AI から) Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. から公開された研究論文 [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (Google Research から) Sascha Rothe, Shashi Narayan, Aliaksei Severyn から公開された研究論文: [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (Baidu から) Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu から公開された研究論文: [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (Baidu から) Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. から公開された研究論文 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) diff --git a/README_ko.md b/README_ko.md index b1fffac0225fff..34ffed005b6624 100644 --- a/README_ko.md +++ b/README_ko.md @@ -270,6 +270,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (Google Research/Stanford University 에서) Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning 의 [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) 논문과 함께 발표했습니다. +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (Meta AI 에서 제공)은 Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi.의 [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438)논문과 함께 발표했습니다. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (Google Research 에서) Sascha Rothe, Shashi Narayan, Aliaksei Severyn 의 [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) 논문과 함께 발표했습니다. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (Baidu 에서) Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu 의 [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) 논문과 함께 발표했습니다. 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (Baidu 에서 제공)은 Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang.의 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674)논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index a463c171e5c6b6..469fd867602e94 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -294,6 +294,7 @@ conda install -c huggingface transformers 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (来自 Snap Research) 伴随论文 [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) 由 Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren 发布。 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (来自 Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning 发布。 +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (来自 Meta AI) 伴随论文 [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) 由 Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi 发布。 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (来自 Google Research) 伴随论文 [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) 由 Sascha Rothe, Shashi Narayan, Aliaksei Severyn 发布。 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (来自 Baidu) 伴随论文 [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu 发布。 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (来自 Baidu) 伴随论文 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) 由 Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 7d28c925a75271..56b4bfad6dba74 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -306,6 +306,7 @@ conda install -c huggingface transformers 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. 1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. +1. **[EnCodec](https://huggingface.co/docs/transformers/main/model_doc/encodec)** (from Meta AI) released with the paper [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) by Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. 1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 60f98607bf62a9..51ef7287d8f70d 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -541,6 +541,8 @@ title: Audio Spectrogram Transformer - local: model_doc/clap title: CLAP + - local: model_doc/encodec + title: EnCodec - local: model_doc/hubert title: Hubert - local: model_doc/mctct diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 6075c0ecf3d213..1690b4cb7c5154 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -107,6 +107,7 @@ The documentation is organized into five sections: 1. **[EfficientFormer](model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. 1. **[EfficientNet](model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. +1. **[EnCodec](model_doc/encodec)** (from Meta AI) released with the paper [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) by Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. 1. **[EncoderDecoder](model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. 1. **[ErnieM](model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. @@ -318,6 +319,7 @@ Flax), PyTorch, and/or TensorFlow. | EfficientFormer | ❌ | ❌ | ✅ | ✅ | ❌ | | EfficientNet | ❌ | ❌ | ✅ | ❌ | ❌ | | ELECTRA | ✅ | ✅ | ✅ | ✅ | ✅ | +| EnCodec | ❌ | ❌ | ✅ | ❌ | ❌ | | Encoder decoder | ❌ | ❌ | ✅ | ✅ | ✅ | | ERNIE | ❌ | ❌ | ✅ | ❌ | ❌ | | ErnieM | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/encodec.mdx b/docs/source/en/model_doc/encodec.mdx new file mode 100644 index 00000000000000..f98156db1d9193 --- /dev/null +++ b/docs/source/en/model_doc/encodec.mdx @@ -0,0 +1,59 @@ + + +# EnCodec + +## Overview + +The EnCodec neural codec model was proposed in [High Fidelity Neural Audio Compression](https://arxiv.org/abs/2210.13438) by Alexandre Défossez, Jade Copet, Gabriel Synnaeve, Yossi Adi. + +The abstract from the paper is the following: + +*We introduce a state-of-the-art real-time, high-fidelity, audio codec leveraging neural networks. It consists in a streaming encoder-decoder architecture with quantized latent space trained in an end-to-end fashion. We simplify and speed-up the training by using a single multiscale spectrogram adversary that efficiently reduces artifacts and produce high-quality samples. We introduce a novel loss balancer mechanism to stabilize training: the weight of a loss now defines the fraction of the overall gradient it should represent, thus decoupling the choice of this hyper-parameter from the typical scale of the loss. Finally, we study how lightweight Transformer models can be used to further compress the obtained representation by up to 40%, while staying faster than real time. We provide a detailed description of the key design choices of the proposed model including: training objective, architectural changes and a study of various perceptual loss functions. We present an extensive subjective evaluation (MUSHRA tests) together with an ablation study for a range of bandwidths and audio domains, including speech, noisy-reverberant speech, and music. Our approach is superior to the baselines methods across all evaluated settings, considering both 24 kHz monophonic and 48 kHz stereophonic audio.* + +This model was contributed by [Matthijs](https://huggingface.co/Matthijs), [Patrick Von Platen](https://huggingface.co/patrickvonplaten) and [Arthur Zucker](https://huggingface.co/ArthurZ). +The original code can be found [here](https://github.com/facebookresearch/encodec). +Here is a quick example of how to encode and decode an audio using this model: + +```python +>>> from datasets import load_dataset, Audio +>>> from transformers import EncodecModel, AutoProcessor +>>> librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + +>>> model = EncodecModel.from_pretrained("facebook/encodec_24khz") +>>> processor = AutoProcessor.from_pretrained("facebook/encodec_24khz") +>>> librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) +>>> audio_sample = librispeech_dummy[-1]["audio"]["array"] +>>> inputs = processor(raw_audio=audio_sample, sampling_rate=processor.sampling_rate, return_tensors="pt") + +>>> encoder_outputs = model.encode(inputs["input_values"], inputs["padding_mask"]) +>>> audio_values = model.decode(encoder_outputs.audio_codes, encoder_outputs.audio_scales, inputs["padding_mask"])[0] +>>> # or the equivalent with a forward pass +>>> audio_values = model(inputs["input_values"], inputs["padding_mask"]).audio_values +``` + + +## EncodecConfig + +[[autodoc]] EncodecConfig + +## EncodecFeatureExtractor + +[[autodoc]] EncodecFeatureExtractor + - __call__ + +## EncodecModel + +[[autodoc]] EncodecModel + - decode + - encode + - forward diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index c2966423da531c..f4e53099b198e4 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -281,6 +281,10 @@ "models.efficientformer": ["EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "EfficientFormerConfig"], "models.efficientnet": ["EFFICIENTNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "EfficientNetConfig"], "models.electra": ["ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP", "ElectraConfig", "ElectraTokenizer"], + "models.encodec": [ + "ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP", + "EncodecConfig", + ], "models.encoder_decoder": ["EncoderDecoderConfig"], "models.ernie": [ "ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP", @@ -825,6 +829,7 @@ ] else: _import_structure["models.audio_spectrogram_transformer"].append("ASTFeatureExtractor") + _import_structure["models.encodec"].append("EncodecFeatureExtractor") _import_structure["models.mctct"].append("MCTCTFeatureExtractor") _import_structure["models.speech_to_text"].append("Speech2TextFeatureExtractor") _import_structure["models.speecht5"].append("SpeechT5FeatureExtractor") @@ -1568,6 +1573,13 @@ "load_tf_weights_in_electra", ] ) + _import_structure["models.encodec"].extend( + [ + "ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST", + "EncodecModel", + "EncodecPreTrainedModel", + ] + ) _import_structure["models.encoder_decoder"].append("EncoderDecoderModel") _import_structure["models.ernie"].extend( [ @@ -4100,6 +4112,10 @@ from .models.efficientformer import EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, EfficientFormerConfig from .models.efficientnet import EFFICIENTNET_PRETRAINED_CONFIG_ARCHIVE_MAP, EfficientNetConfig from .models.electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraTokenizer + from .models.encodec import ( + ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP, + EncodecConfig, + ) from .models.encoder_decoder import EncoderDecoderConfig from .models.ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig from .models.ernie_m import ERNIE_M_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieMConfig @@ -4598,6 +4614,7 @@ from .utils.dummy_speech_objects import * else: from .models.audio_spectrogram_transformer import ASTFeatureExtractor + from .models.encodec import EncodecFeatureExtractor from .models.mctct import MCTCTFeatureExtractor from .models.speech_to_text import Speech2TextFeatureExtractor from .models.speecht5 import SpeechT5FeatureExtractor @@ -5210,6 +5227,11 @@ ElectraPreTrainedModel, load_tf_weights_in_electra, ) + from .models.encodec import ( + ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST, + EncodecModel, + EncodecPreTrainedModel, + ) from .models.encoder_decoder import EncoderDecoderModel from .models.ernie import ( ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 0ee88e0ba39261..c41059698558fd 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -72,6 +72,7 @@ efficientformer, efficientnet, electra, + encodec, encoder_decoder, ernie, ernie_m, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 6445f62acb58f0..f2c1c8829994ef 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -80,6 +80,7 @@ ("efficientformer", "EfficientFormerConfig"), ("efficientnet", "EfficientNetConfig"), ("electra", "ElectraConfig"), + ("encodec", "EncodecConfig"), ("encoder-decoder", "EncoderDecoderConfig"), ("ernie", "ErnieConfig"), ("ernie_m", "ErnieMConfig"), @@ -273,6 +274,7 @@ ("efficientformer", "EFFICIENTFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("efficientnet", "EFFICIENTNET_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("electra", "ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("encodec", "ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("ernie", "ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("ernie_m", "ERNIE_M_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("esm", "ESM_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -461,6 +463,7 @@ ("efficientformer", "EfficientFormer"), ("efficientnet", "EfficientNet"), ("electra", "ELECTRA"), + ("encodec", "EnCodec"), ("encoder-decoder", "Encoder decoder"), ("ernie", "ERNIE"), ("ernie_m", "ErnieM"), diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index 681a59e0b28b57..b100b62ac024e6 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -54,6 +54,7 @@ ("dinat", "ViTFeatureExtractor"), ("donut-swin", "DonutFeatureExtractor"), ("dpt", "DPTFeatureExtractor"), + ("encodec", "EncodecFeatureExtractor"), ("flava", "FlavaFeatureExtractor"), ("glpn", "GLPNFeatureExtractor"), ("groupvit", "CLIPFeatureExtractor"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 2c23da3c556f4e..4c675ff0509783 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -79,6 +79,7 @@ ("efficientformer", "EfficientFormerModel"), ("efficientnet", "EfficientNetModel"), ("electra", "ElectraModel"), + ("encodec", "EncodecModel"), ("ernie", "ErnieModel"), ("ernie_m", "ErnieMModel"), ("esm", "EsmModel"), diff --git a/src/transformers/models/encodec/__init__.py b/src/transformers/models/encodec/__init__.py new file mode 100644 index 00000000000000..d3d9488968bf2c --- /dev/null +++ b/src/transformers/models/encodec/__init__.py @@ -0,0 +1,65 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, +) + + +_import_structure = { + "configuration_encodec": [ + "ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP", + "EncodecConfig", + ], + "feature_extraction_encodec": ["EncodecFeatureExtractor"], +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_encodec"] = [ + "ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST", + "EncodecModel", + "EncodecPreTrainedModel", + ] + +if TYPE_CHECKING: + from .configuration_encodec import ( + ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP, + EncodecConfig, + ) + from .feature_extraction_encodec import EncodecFeatureExtractor + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_encodec import ( + ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST, + EncodecModel, + EncodecPreTrainedModel, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/encodec/configuration_encodec.py b/src/transformers/models/encodec/configuration_encodec.py new file mode 100644 index 00000000000000..9ea2cfee945b0d --- /dev/null +++ b/src/transformers/models/encodec/configuration_encodec.py @@ -0,0 +1,189 @@ +# coding=utf-8 +# Copyright 2023 Meta Platforms, Inc. and affiliates, and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" EnCodec model configuration""" + + +import math +from typing import Optional + +import numpy as np + +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +ENCODEC_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "facebook/encodec_24khz": "https://huggingface.co/facebook/encodec_24khz/resolve/main/config.json", + "facebook/encodec_48khz": "https://huggingface.co/facebook/encodec_48khz/resolve/main/config.json", +} + + +class EncodecConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of an [`EncodecModel`]. It is used to instantiate a + Encodec model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the + [facebook/encodec_24khz](https://huggingface.co/facebook/encodec_24khz) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + target_bandwidths (`List[float]`, *optional*, defaults to `[1.5, 3.0, 6.0, 12.0, 24.0]`): + The range of diffent bandwiths the model can encode audio with. + sampling_rate (`int`, *optional*, defaults to 24000): + The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz). + audio_channels (`int`, *optional*, defaults to 1): + Number of channels in the audio data. Either 1 for mono or 2 for stereo. + normalize (`bool`, *optional*, defaults to `False`): + Whether the audio shall be normalized when passed. + chunk_length_s (`float`, *optional*): + If defined the audio is pre-processed into chunks of lengths `chunk_length_s` and then encoded. + overlap (`float`, *optional*): + Defines the overlap between each chunk. It is used to compute the `chunk_stride` using the following + formulae : `int((1.0 - self.overlap) * self.chunk_length)`. + hidden_size (`int`, *optional*, defaults to 128): + Intermediate representation dimension. + num_filters (`int`, *optional*, defaults to 32): + Number of convolution kernels of first `EncodecConv1d` down sampling layer. + num_residual_layers (`int`, *optional*, defaults to 1): + Number of residual layers. + upsampling_ratios (`Sequence[int]` , *optional*, defaults to `[8, 5, 4, 2]`): + Kernel size and stride ratios. The encoder uses downsampling ratios instead of upsampling ratios, hence it + will use the ratios in the reverse order to the ones specified here that must match the decoder order. + norm_type (`str`, *optional*, defaults to `"weight_norm"`): + Normalization method. Should be in `["weight_norm", "time_group_norm"]` + kernel_size (`int`, *optional*, defaults to 7): + Kernel size for the initial convolution. + last_kernel_size (`int`, *optional*, defaults to 7): + Kernel size for the last convolution layer. + residual_kernel_size (`int`, *optional*, defaults to 3): + Kernel size for the residual layers. + dilation_growth_rate (`int`, *optional*, defaults to 2): + How much to increase the dilation with each layer. + use_causal_conv (`bool`, *optional*, defaults to `True`): + Whether to use fully causal convolution. + pad_mode (`str`, *optional*, defaults to `"reflect"`): + Padding mode for the convolutions. + compress (`int`, *optional*, defaults to 2): + Reduced dimensionality in residual branches (from Demucs v3). + num_lstm_layers (`int`, *optional*, defaults to 2): + Number of LSTM layers at the end of the encoder. + trim_right_ratio (`float`, *optional*, defaults to 1.0): + Ratio for trimming at the right of the transposed convolution under the `use_causal_conv = True` setup. If + equal to 1.0, it means that all the trimming is done at the right. + codebook_size (`int`, *optional*, defaults to 1024): + Number of discret codes that make up VQVAE. + codebook_dim (`int`, *optional*): + Dimension of the codebook vectors. If not defined, uses `hidden_size`. + + Example: + + ```python + >>> from transformers import EncodecModel, EncodecConfig + + >>> # Initializing a "facebook/encodec_24khz" style configuration + >>> configuration = EncodecConfig() + + >>> # Initializing a model (with random weights) from the "facebook/encodec_24khz" style configuration + >>> model = EncodecModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "encodec" + + def __init__( + self, + target_bandwidths=[1.5, 3.0, 6.0, 12.0, 24.0], + sampling_rate=24_000, + audio_channels=1, + normalize=False, + chunk_length_s=None, + overlap=None, + hidden_size=128, + num_filters=32, + num_residual_layers=1, + upsampling_ratios=[8, 5, 4, 2], + norm_type="weight_norm", + kernel_size=7, + last_kernel_size=7, + residual_kernel_size=3, + dilation_growth_rate=2, + use_causal_conv=True, + pad_mode="reflect", + compress=2, + num_lstm_layers=2, + trim_right_ratio=1.0, + codebook_size=1024, + codebook_dim=None, + **kwargs, + ): + self.target_bandwidths = target_bandwidths + self.sampling_rate = sampling_rate + self.audio_channels = audio_channels + self.normalize = normalize + self.chunk_length_s = chunk_length_s + self.overlap = overlap + self.hidden_size = hidden_size + self.num_filters = num_filters + self.num_residual_layers = num_residual_layers + self.upsampling_ratios = upsampling_ratios + self.norm_type = norm_type + self.kernel_size = kernel_size + self.last_kernel_size = last_kernel_size + self.residual_kernel_size = residual_kernel_size + self.dilation_growth_rate = dilation_growth_rate + self.use_causal_conv = use_causal_conv + self.pad_mode = pad_mode + self.compress = compress + self.num_lstm_layers = num_lstm_layers + self.trim_right_ratio = trim_right_ratio + self.codebook_size = codebook_size + self.codebook_dim = codebook_dim if codebook_dim is not None else hidden_size + + if self.norm_type not in ["weight_norm", "time_group_norm"]: + raise ValueError( + f'self.norm_type must be one of `"weight_norm"`, `"time_group_norm"`), got {self.norm_type}' + ) + + super().__init__(**kwargs) + + # This is a property because you might want to change the chunk_length_s on the fly + @property + def chunk_length(self) -> Optional[int]: + if self.chunk_length_s is None: + return None + else: + return int(self.chunk_length_s * self.sampling_rate) + + # This is a property because you might want to change the chunk_length_s on the fly + @property + def chunk_stride(self) -> Optional[int]: + if self.chunk_length_s is None or self.overlap is None: + return None + else: + return max(1, int((1.0 - self.overlap) * self.chunk_length)) + + @property + def frame_rate(self) -> int: + hop_length = np.prod(self.upsampling_ratios) + return math.ceil(self.sampling_rate / hop_length) + + @property + def num_quantizers(self) -> int: + return int(1000 * self.target_bandwidths[-1] // (self.frame_rate * 10)) diff --git a/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py b/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py new file mode 100644 index 00000000000000..cd7ead3d72741b --- /dev/null +++ b/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py @@ -0,0 +1,352 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert EnCodec checkpoints.""" + +import argparse + +import torch + +from transformers import ( + EncodecConfig, + EncodecFeatureExtractor, + EncodecModel, + logging, +) + + +# checkpoints downloaded from: +# https://dl.fbaipublicfiles.com/encodec/v0/encodec_24khz-d7cc33bc.th +# https://dl.fbaipublicfiles.com/encodec/v0/encodec_48khz-7e698e3e.th + + +logging.set_verbosity_info() +logger = logging.get_logger("transformers.models.encodec") + +MAPPING_QUANTIZER = { + "quantizer.vq.layers.*._codebook.inited": "quantizer.layers.*.codebook.inited", + "quantizer.vq.layers.*._codebook.cluster_size": "quantizer.layers.*.codebook.cluster_size", + "quantizer.vq.layers.*._codebook.embed": "quantizer.layers.*.codebook.embed", + "quantizer.vq.layers.*._codebook.embed_avg": "quantizer.layers.*.codebook.embed_avg", +} +MAPPING_ENCODER = { + "encoder.model.0.conv.conv": "encoder.layers.0.conv", + "encoder.model.1.block.1.conv.conv": "encoder.layers.1.block.1.conv", + "encoder.model.1.block.3.conv.conv": "encoder.layers.1.block.3.conv", + "encoder.model.1.shortcut.conv.conv": "encoder.layers.1.shortcut.conv", + "encoder.model.3.conv.conv": "encoder.layers.3.conv", + "encoder.model.4.block.1.conv.conv": "encoder.layers.4.block.1.conv", + "encoder.model.4.block.3.conv.conv": "encoder.layers.4.block.3.conv", + "encoder.model.4.shortcut.conv.conv": "encoder.layers.4.shortcut.conv", + "encoder.model.6.conv.conv": "encoder.layers.6.conv", + "encoder.model.7.block.1.conv.conv": "encoder.layers.7.block.1.conv", + "encoder.model.7.block.3.conv.conv": "encoder.layers.7.block.3.conv", + "encoder.model.7.shortcut.conv.conv": "encoder.layers.7.shortcut.conv", + "encoder.model.9.conv.conv": "encoder.layers.9.conv", + "encoder.model.10.block.1.conv.conv": "encoder.layers.10.block.1.conv", + "encoder.model.10.block.3.conv.conv": "encoder.layers.10.block.3.conv", + "encoder.model.10.shortcut.conv.conv": "encoder.layers.10.shortcut.conv", + "encoder.model.12.conv.conv": "encoder.layers.12.conv", + "encoder.model.13.lstm": "encoder.layers.13.lstm", + "encoder.model.15.conv.conv": "encoder.layers.15.conv", +} +MAPPING_ENCODER_48K = { + "encoder.model.0.conv.norm": "encoder.layers.0.norm", + "encoder.model.1.block.1.conv.norm": "encoder.layers.1.block.1.norm", + "encoder.model.1.block.3.conv.norm": "encoder.layers.1.block.3.norm", + "encoder.model.1.shortcut.conv.norm": "encoder.layers.1.shortcut.norm", + "encoder.model.3.conv.norm": "encoder.layers.3.norm", + "encoder.model.4.block.1.conv.norm": "encoder.layers.4.block.1.norm", + "encoder.model.4.block.3.conv.norm": "encoder.layers.4.block.3.norm", + "encoder.model.4.shortcut.conv.norm": "encoder.layers.4.shortcut.norm", + "encoder.model.6.conv.norm": "encoder.layers.6.norm", + "encoder.model.7.block.1.conv.norm": "encoder.layers.7.block.1.norm", + "encoder.model.7.block.3.conv.norm": "encoder.layers.7.block.3.norm", + "encoder.model.7.shortcut.conv.norm": "encoder.layers.7.shortcut.norm", + "encoder.model.9.conv.norm": "encoder.layers.9.norm", + "encoder.model.10.block.1.conv.norm": "encoder.layers.10.block.1.norm", + "encoder.model.10.block.3.conv.norm": "encoder.layers.10.block.3.norm", + "encoder.model.10.shortcut.conv.norm": "encoder.layers.10.shortcut.norm", + "encoder.model.12.conv.norm": "encoder.layers.12.norm", + "encoder.model.15.conv.norm": "encoder.layers.15.norm", +} +MAPPING_DECODER = { + "decoder.model.0.conv.conv": "decoder.layers.0.conv", + "decoder.model.1.lstm": "decoder.layers.1.lstm", + "decoder.model.3.convtr.convtr": "decoder.layers.3.conv", + "decoder.model.4.block.1.conv.conv": "decoder.layers.4.block.1.conv", + "decoder.model.4.block.3.conv.conv": "decoder.layers.4.block.3.conv", + "decoder.model.4.shortcut.conv.conv": "decoder.layers.4.shortcut.conv", + "decoder.model.6.convtr.convtr": "decoder.layers.6.conv", + "decoder.model.7.block.1.conv.conv": "decoder.layers.7.block.1.conv", + "decoder.model.7.block.3.conv.conv": "decoder.layers.7.block.3.conv", + "decoder.model.7.shortcut.conv.conv": "decoder.layers.7.shortcut.conv", + "decoder.model.9.convtr.convtr": "decoder.layers.9.conv", + "decoder.model.10.block.1.conv.conv": "decoder.layers.10.block.1.conv", + "decoder.model.10.block.3.conv.conv": "decoder.layers.10.block.3.conv", + "decoder.model.10.shortcut.conv.conv": "decoder.layers.10.shortcut.conv", + "decoder.model.12.convtr.convtr": "decoder.layers.12.conv", + "decoder.model.13.block.1.conv.conv": "decoder.layers.13.block.1.conv", + "decoder.model.13.block.3.conv.conv": "decoder.layers.13.block.3.conv", + "decoder.model.13.shortcut.conv.conv": "decoder.layers.13.shortcut.conv", + "decoder.model.15.conv.conv": "decoder.layers.15.conv", +} +MAPPING_DECODER_48K = { + "decoder.model.0.conv.norm": "decoder.layers.0.norm", + "decoder.model.3.convtr.norm": "decoder.layers.3.norm", + "decoder.model.4.block.1.conv.norm": "decoder.layers.4.block.1.norm", + "decoder.model.4.block.3.conv.norm": "decoder.layers.4.block.3.norm", + "decoder.model.4.shortcut.conv.norm": "decoder.layers.4.shortcut.norm", + "decoder.model.6.convtr.norm": "decoder.layers.6.norm", + "decoder.model.7.block.1.conv.norm": "decoder.layers.7.block.1.norm", + "decoder.model.7.block.3.conv.norm": "decoder.layers.7.block.3.norm", + "decoder.model.7.shortcut.conv.norm": "decoder.layers.7.shortcut.norm", + "decoder.model.9.convtr.norm": "decoder.layers.9.norm", + "decoder.model.10.block.1.conv.norm": "decoder.layers.10.block.1.norm", + "decoder.model.10.block.3.conv.norm": "decoder.layers.10.block.3.norm", + "decoder.model.10.shortcut.conv.norm": "decoder.layers.10.shortcut.norm", + "decoder.model.12.convtr.norm": "decoder.layers.12.norm", + "decoder.model.13.block.1.conv.norm": "decoder.layers.13.block.1.norm", + "decoder.model.13.block.3.conv.norm": "decoder.layers.13.block.3.norm", + "decoder.model.13.shortcut.conv.norm": "decoder.layers.13.shortcut.norm", + "decoder.model.15.conv.norm": "decoder.layers.15.norm", +} +MAPPING_24K = { + **MAPPING_QUANTIZER, + **MAPPING_ENCODER, + **MAPPING_DECODER, +} +MAPPING_48K = { + **MAPPING_QUANTIZER, + **MAPPING_ENCODER, + **MAPPING_ENCODER_48K, + **MAPPING_DECODER, + **MAPPING_DECODER_48K, +} +TOP_LEVEL_KEYS = [] +IGNORE_KEYS = [] + + +def set_recursively(hf_pointer, key, value, full_name, weight_type): + for attribute in key.split("."): + hf_pointer = getattr(hf_pointer, attribute) + + if weight_type is not None: + hf_shape = getattr(hf_pointer, weight_type).shape + else: + hf_shape = hf_pointer.shape + + if hf_shape != value.shape: + raise ValueError( + f"Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be" + f" {value.shape} for {full_name}" + ) + + if weight_type == "weight": + hf_pointer.weight.data = value + elif weight_type == "weight_g": + hf_pointer.weight_g.data = value + elif weight_type == "weight_v": + hf_pointer.weight_v.data = value + elif weight_type == "bias": + hf_pointer.bias.data = value + elif weight_type == "running_mean": + hf_pointer.running_mean.data = value + elif weight_type == "running_var": + hf_pointer.running_var.data = value + elif weight_type == "num_batches_tracked": + hf_pointer.num_batches_tracked.data = value + elif weight_type == "weight_ih_l0": + hf_pointer.weight_ih_l0.data = value + elif weight_type == "weight_hh_l0": + hf_pointer.weight_hh_l0.data = value + elif weight_type == "bias_ih_l0": + hf_pointer.bias_ih_l0.data = value + elif weight_type == "bias_hh_l0": + hf_pointer.bias_hh_l0.data = value + elif weight_type == "weight_ih_l1": + hf_pointer.weight_ih_l1.data = value + elif weight_type == "weight_hh_l1": + hf_pointer.weight_hh_l1.data = value + elif weight_type == "bias_ih_l1": + hf_pointer.bias_ih_l1.data = value + elif weight_type == "bias_hh_l1": + hf_pointer.bias_hh_l1.data = value + else: + hf_pointer.data = value + + logger.info(f"{key + ('.' + weight_type if weight_type is not None else '')} was initialized from {full_name}.") + + +def should_ignore(name, ignore_keys): + for key in ignore_keys: + if key.endswith(".*"): + if name.startswith(key[:-1]): + return True + elif ".*." in key: + prefix, suffix = key.split(".*.") + if prefix in name and suffix in name: + return True + elif key in name: + return True + return False + + +def recursively_load_weights(orig_dict, hf_model, model_name): + unused_weights = [] + + if model_name == "encodec_24khz": + MAPPING = MAPPING_24K + elif model_name == "encodec_48khz": + MAPPING = MAPPING_48K + else: + raise ValueError(f"Unsupported model: {model_name}") + + for name, value in orig_dict.items(): + if should_ignore(name, IGNORE_KEYS): + logger.info(f"{name} was ignored") + continue + + is_used = False + for key, mapped_key in MAPPING.items(): + if "*" in key: + prefix, suffix = key.split(".*.") + if prefix in name and suffix in name: + key = suffix + + if key in name: + # HACK otherwise .embed gets initialized with .embed_avg too + if key.endswith("embed") and name.endswith("embed_avg"): + continue + + is_used = True + if "*" in mapped_key: + layer_index = name.split(key)[0].split(".")[-2] + mapped_key = mapped_key.replace("*", layer_index) + if "weight_g" in name: + weight_type = "weight_g" + elif "weight_v" in name: + weight_type = "weight_v" + elif "weight_ih_l0" in name: + weight_type = "weight_ih_l0" + elif "weight_hh_l0" in name: + weight_type = "weight_hh_l0" + elif "bias_ih_l0" in name: + weight_type = "bias_ih_l0" + elif "bias_hh_l0" in name: + weight_type = "bias_hh_l0" + elif "weight_ih_l1" in name: + weight_type = "weight_ih_l1" + elif "weight_hh_l1" in name: + weight_type = "weight_hh_l1" + elif "bias_ih_l1" in name: + weight_type = "bias_ih_l1" + elif "bias_hh_l1" in name: + weight_type = "bias_hh_l1" + elif "bias" in name: + weight_type = "bias" + elif "weight" in name: + weight_type = "weight" + elif "running_mean" in name: + weight_type = "running_mean" + elif "running_var" in name: + weight_type = "running_var" + elif "num_batches_tracked" in name: + weight_type = "num_batches_tracked" + else: + weight_type = None + set_recursively(hf_model, mapped_key, value, name, weight_type) + continue + if not is_used: + unused_weights.append(name) + + logger.warning(f"Unused weights: {unused_weights}") + + +@torch.no_grad() +def convert_checkpoint( + model_name, + checkpoint_path, + pytorch_dump_folder_path, + config_path=None, + repo_id=None, +): + """ + Copy/paste/tweak model's weights to transformers design. + """ + if config_path is not None: + config = EncodecConfig.from_pretrained(config_path) + else: + config = EncodecConfig() + + if model_name == "encodec_24khz": + pass # config is already correct + elif model_name == "encodec_48khz": + config.upsampling_ratios = [8, 5, 4, 2] + config.target_bandwidths = [3.0, 6.0, 12.0, 24.0] + config.sampling_rate = 48_000 + config.audio_channels = 2 + config.use_causal_conv = False + config.norm_type = "time_group_norm" + config.normalize = True + config.chunk_length_s = 1.0 + config.overlap = 0.01 + else: + raise ValueError(f"Unknown model name: {model_name}") + + model = EncodecModel(config) + + feature_extractor = EncodecFeatureExtractor( + feature_size=config.audio_channels, + sampling_rate=config.sampling_rate, + chunk_length_s=config.chunk_length_s, + overlap=config.overlap, + ) + feature_extractor.save_pretrained(pytorch_dump_folder_path) + + original_checkpoint = torch.load(checkpoint_path) + recursively_load_weights(original_checkpoint, model, model_name) + model.save_pretrained(pytorch_dump_folder_path) + + if repo_id: + print("Pushing to the hub...") + feature_extractor.push_to_hub(repo_id) + model.push_to_hub(repo_id) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", + default="encodec_24khz", + type=str, + help="The model to convert. Should be one of 'encodec_24khz', 'encodec_48khz'.", + ) + parser.add_argument("--checkpoint_path", required=True, default=None, type=str, help="Path to original checkpoint") + parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert") + parser.add_argument( + "--pytorch_dump_folder_path", required=True, default=None, type=str, help="Path to the output PyTorch model." + ) + parser.add_argument( + "--push_to_hub", default=None, type=str, help="Where to upload the converted model on the 🤗 hub." + ) + + args = parser.parse_args() + convert_checkpoint( + args.model, + args.checkpoint_path, + args.pytorch_dump_folder_path, + args.config_path, + args.push_to_hub, + ) diff --git a/src/transformers/models/encodec/feature_extraction_encodec.py b/src/transformers/models/encodec/feature_extraction_encodec.py new file mode 100644 index 00000000000000..6f7536a52e9f99 --- /dev/null +++ b/src/transformers/models/encodec/feature_extraction_encodec.py @@ -0,0 +1,206 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Feature extractor class for EnCodec.""" + +from typing import List, Optional, Union + +import numpy as np + +from ...feature_extraction_sequence_utils import SequenceFeatureExtractor +from ...feature_extraction_utils import BatchFeature +from ...utils import PaddingStrategy, TensorType, logging + + +logger = logging.get_logger(__name__) + + +class EncodecFeatureExtractor(SequenceFeatureExtractor): + r""" + Constructs an EnCodec feature extractor. + + This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains + most of the main methods. Users should refer to this superclass for more information regarding those methods. + + Instantiating a feature extractor with the defaults will yield a similar configuration to that of the + [facebook/encodec_24khz](https://huggingface.co/facebook/encodec_24khz) architecture. + + Args: + feature_size (`int`, *optional*, defaults to 1): + The feature dimension of the extracted features. Use 1 for mono, 2 for stereo. + sampling_rate (`int`, *optional*, defaults to 24000): + The sampling rate at which the audio waveform should be digitalized expressed in hertz (Hz). + padding_value (`float`, *optional*, defaults to 0.0): + The value that is used to fill the padding values. + chunk_length_s (`float`, *optional*): + If defined the audio is pre-processed into chunks of lengths `chunk_length_s` and then encoded. + overlap (`float`, *optional*): + Defines the overlap between each chunk. It is used to compute the `chunk_stride` using the following + formulae : `int((1.0 - self.overlap) * self.chunk_length)`. + """ + + model_input_names = ["input_values", "padding_mask"] + + def __init__( + self, + feature_size: int = 1, + sampling_rate: int = 24000, + padding_value: float = 0.0, + chunk_length_s: float = None, + overlap: float = None, + **kwargs, + ): + super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs) + self.chunk_length_s = chunk_length_s + self.overlap = overlap + + # This is a property because you might want to change the chunk_length_s on the fly + @property + def chunk_length(self) -> Optional[int]: + if self.chunk_length_s is None: + return None + else: + return int(self.chunk_length_s * self.sampling_rate) + + # This is a property because you might want to change the chunk_length_s on the fly + @property + def chunk_stride(self) -> Optional[int]: + if self.chunk_length_s is None or self.overlap is None: + return None + else: + return max(1, int((1.0 - self.overlap) * self.chunk_length)) + + def __call__( + self, + raw_audio: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]], + padding: Optional[Union[bool, str, PaddingStrategy]] = None, + truncation: Optional[bool] = False, + max_length: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + sampling_rate: Optional[int] = None, + ) -> BatchFeature: + """ + Main method to featurize and prepare for the model one or several sequence(s). + + Args: + raw_audio (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`): + The sequence or batch of sequences to be processed. Each sequence can be a numpy array, a list of float + values, a list of numpy arrays or a list of list of float values. The numpy array must be of shape + `(num_samples,)` for mono audio (`feature_size = 1`), or `(2, num_samples)` for stereo audio + (`feature_size = 2`). + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding + index) among: + + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + truncation (`bool`, *optional*, defaults to `False`): + Activates truncation to cut input sequences longer than `max_length` to `max_length`. + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors instead of list of python integers. Acceptable values are: + + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return Numpy `np.ndarray` objects. + sampling_rate (`int`, *optional*): + The sampling rate at which the `audio` input was sampled. It is strongly recommended to pass + `sampling_rate` at the forward call to prevent silent errors. + """ + if sampling_rate is not None: + if sampling_rate != self.sampling_rate: + raise ValueError( + f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of" + f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with" + f" {self.sampling_rate} and not {sampling_rate}." + ) + else: + logger.warning( + "It is strongly recommended to pass the `sampling_rate` argument to this function. " + "Failing to do so can result in silent errors that might be hard to debug." + ) + + if padding and truncation: + raise ValueError("Both padding and truncation were set. Make sure you only set one.") + elif padding is None: + # by default let's pad the inputs + padding = True + + is_batched = bool( + isinstance(raw_audio, (list, tuple)) and (isinstance(raw_audio[0], (np.ndarray, tuple, list))) + ) + + if is_batched: + raw_audio = [np.asarray(audio, dtype=np.float32).T for audio in raw_audio] + elif not is_batched and not isinstance(raw_audio, np.ndarray): + raw_audio = np.asarray(raw_audio, dtype=np.float32) + elif isinstance(raw_audio, np.ndarray) and raw_audio.dtype is np.dtype(np.float64): + raw_audio = raw_audio.astype(np.float32) + + # always return batch + if not is_batched: + raw_audio = [np.asarray(raw_audio).T] + + # verify inputs are valid + for idx, example in enumerate(raw_audio): + if example.ndim > 2: + raise ValueError(f"Expected input shape (channels, length) but got shape {example.shape}") + if self.feature_size == 1 and example.ndim != 1: + raise ValueError(f"Expected mono audio but example has {example.shape[-1]} channels") + if self.feature_size == 2 and example.shape[-1] != 2: + raise ValueError(f"Expected stereo audio but example has {example.shape[-1]} channels") + + padded_inputs = None + input_values = BatchFeature({"input_values": raw_audio}) + if self.chunk_stride is not None and self.chunk_length is not None and max_length is None: + if truncation: + max_length = min(array.shape[0] for array in raw_audio) + nb_step = int(np.floor(max_length / self.chunk_stride)) + max_length = (nb_step - 1) * self.chunk_stride + self.chunk_length + elif padding: + max_length = max(array.shape[0] for array in raw_audio) + nb_step = int(np.ceil(max_length / self.chunk_stride)) + max_length = (nb_step - 1) * self.chunk_stride + self.chunk_length + padding = "max_length" + else: + padded_inputs = input_values + + # normal padding on batch + if padded_inputs is None: + padded_inputs = self.pad( + input_values, + max_length=max_length, + truncation=truncation, + padding=padding, + return_attention_mask=padding, + ) + if padding: + padded_inputs["padding_mask"] = padded_inputs.pop("attention_mask") + + input_values = [] + for example in padded_inputs.pop("input_values"): + if self.feature_size == 1: + example = example[..., None] + input_values.append(example.T) + + padded_inputs["input_values"] = input_values + if return_tensors is not None: + padded_inputs = padded_inputs.convert_to_tensors(return_tensors) + + return padded_inputs diff --git a/src/transformers/models/encodec/modeling_encodec.py b/src/transformers/models/encodec/modeling_encodec.py new file mode 100644 index 00000000000000..ad1f6a0ee83a9e --- /dev/null +++ b/src/transformers/models/encodec/modeling_encodec.py @@ -0,0 +1,808 @@ +# coding=utf-8 +# Copyright 2023 Meta Platforms, Inc. and affiliates, and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch EnCodec model.""" + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn + +from ...modeling_utils import PreTrainedModel +from ...utils import ( + ModelOutput, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_encodec import EncodecConfig + + +logger = logging.get_logger(__name__) + + +# General docstring +_CONFIG_FOR_DOC = "EncodecConfig" + + +ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "facebook/encodec_24khz", + "facebook/encodec_48khz", + # See all EnCodec models at https://huggingface.co/models?filter=encodec +] + + +@dataclass +class EncodecOutput(ModelOutput): + """ + Args: + audio_codes (`torch.FloatTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*): + Discret code embeddings computed using `model.encode`. + audio_values (`torch.FlaotTensor` of shape `(batch_size, sequence_length)`, *optional*) + Decoded audio values, obtained using the decoder part of Encodec. + """ + + audio_codes: torch.FloatTensor = None + audio_values: torch.FloatTensor = None + + +@dataclass +class EncodecEncoderOutput(ModelOutput): + """ + Args: + audio_codes (`torch.FloatTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*): + Discret code embeddings computed using `model.encode`. + audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*): + Scaling factor for each `audio_codes` input. This is used to unscale each chunk of audio when decoding. + """ + + audio_codes: torch.FloatTensor = None + audio_scales: torch.FloatTensor = None + + +@dataclass +class EncodecDecoderOutput(ModelOutput): + """ + Args: + audio_values (`torch.FloatTensor` of shape `(batch_size, segment_length)`, *optional*): + Decoded audio values, obtained using the decoder part of Encodec. + """ + + audio_values: torch.FloatTensor = None + + +class EncodecConv1d(nn.Module): + """Conv1d with asymmetric or causal padding and normalization.""" + + def __init__( + self, config, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1, dilation: int = 1 + ): + super().__init__() + self.causal = config.use_causal_conv + self.pad_mode = config.pad_mode + self.norm_type = config.norm_type + + if self.norm_type not in ["weight_norm", "time_group_norm"]: + raise ValueError( + f'self.norm_type must be one of `"weight_norm"`, `"time_group_norm"`), got {self.norm_type}' + ) + + # warn user on unusual setup between dilation and stride + if stride > 1 and dilation > 1: + logger.warning( + "EncodecConv1d has been initialized with stride > 1 and dilation > 1" + f" (kernel_size={kernel_size} stride={stride}, dilation={dilation})." + ) + + self.conv = nn.Conv1d(in_channels, out_channels, kernel_size, stride, dilation=dilation) + if self.norm_type == "weight_norm": + self.conv = nn.utils.weight_norm(self.conv) + elif self.norm_type == "time_group_norm": + self.norm = nn.GroupNorm(1, out_channels) + + @staticmethod + def _get_extra_padding_for_conv1d( + hidden_states: torch.Tensor, kernel_size: int, stride: int, padding_total: int = 0 + ) -> int: + """See `pad_for_conv1d`.""" + length = hidden_states.shape[-1] + n_frames = (length - kernel_size + padding_total) / stride + 1 + ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total) + return ideal_length - length + + @staticmethod + def _pad1d(hidden_states: torch.Tensor, paddings: Tuple[int, int], mode: str = "zero", value: float = 0.0): + """Tiny wrapper around torch.nn.functional.pad, just to allow for reflect padding on small input. + If this is the case, we insert extra 0 padding to the right before the reflection happens. + """ + length = hidden_states.shape[-1] + padding_left, padding_right = paddings + if not mode == "reflect": + return nn.functional.pad(hidden_states, paddings, mode, value) + + max_pad = max(padding_left, padding_right) + extra_pad = 0 + if length <= max_pad: + extra_pad = max_pad - length + 1 + hidden_states = nn.functional.pad(hidden_states, (0, extra_pad)) + padded = nn.functional.pad(hidden_states, paddings, mode, value) + end = padded.shape[-1] - extra_pad + return padded[..., :end] + + def forward(self, hidden_states): + kernel_size = self.conv.kernel_size[0] + stride = self.conv.stride[0] + dilation = self.conv.dilation[0] + kernel_size = (kernel_size - 1) * dilation + 1 # effective kernel size with dilations + padding_total = kernel_size - stride + extra_padding = self._get_extra_padding_for_conv1d(hidden_states, kernel_size, stride, padding_total) + + if self.causal: + # Left padding for causal + hidden_states = self._pad1d(hidden_states, (padding_total, extra_padding), mode=self.pad_mode) + else: + # Asymmetric padding required for odd strides + padding_right = padding_total // 2 + padding_left = padding_total - padding_right + hidden_states = self._pad1d( + hidden_states, (padding_left, padding_right + extra_padding), mode=self.pad_mode + ) + + hidden_states = self.conv(hidden_states) + + if self.norm_type == "time_group_norm": + hidden_states = self.norm(hidden_states) + + return hidden_states + + +class EncodecConvTranspose1d(nn.Module): + """ConvTranspose1d with asymmetric or causal padding and normalization.""" + + def __init__(self, config, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1): + super().__init__() + self.causal = config.use_causal_conv + self.trim_right_ratio = config.trim_right_ratio + self.norm_type = config.norm_type + if self.norm_type not in ["weight_norm", "time_group_norm"]: + raise ValueError( + f'self.norm_type must be one of `"weight_norm"`, `"time_group_norm"`), got {self.norm_type}' + ) + + self.conv = nn.ConvTranspose1d(in_channels, out_channels, kernel_size, stride) + if config.norm_type == "weight_norm": + self.conv = nn.utils.weight_norm(self.conv) + elif config.norm_type == "time_group_norm": + self.norm = nn.GroupNorm(1, out_channels) + + if not (self.causal or self.trim_right_ratio == 1.0): + raise ValueError("`trim_right_ratio` != 1.0 only makes sense for causal convolutions") + + def forward(self, hidden_states): + kernel_size = self.conv.kernel_size[0] + stride = self.conv.stride[0] + padding_total = kernel_size - stride + + hidden_states = self.conv(hidden_states) + + if self.norm_type == "time_group_norm": + hidden_states = self.norm(hidden_states) + + # We will only trim fixed padding. Extra padding from `pad_for_conv1d` would be + # removed at the very end, when keeping only the right length for the output, + # as removing it here would require also passing the length at the matching layer + # in the encoder. + if self.causal: + # Trim the padding on the right according to the specified ratio + # if trim_right_ratio = 1.0, trim everything from right + padding_right = math.ceil(padding_total * self.trim_right_ratio) + else: + # Asymmetric padding required for odd strides + padding_right = padding_total // 2 + + padding_left = padding_total - padding_right + + # unpad + end = hidden_states.shape[-1] - padding_right + hidden_states = hidden_states[..., padding_left:end] + return hidden_states + + +class EncodecLSTM(nn.Module): + """ + LSTM without worrying about the hidden state, nor the layout of the data. Expects input as convolutional layout. + """ + + def __init__(self, config, dimension): + super().__init__() + self.lstm = nn.LSTM(dimension, dimension, config.num_lstm_layers) + + def forward(self, hidden_states): + hidden_states = hidden_states.permute(2, 0, 1) + hidden_states = self.lstm(hidden_states)[0] + hidden_states + hidden_states = hidden_states.permute(1, 2, 0) + return hidden_states + + +class EncodecResnetBlock(nn.Module): + """ + Residual block from SEANet model as used by EnCodec. + """ + + def __init__(self, config: EncodecConfig, dim: int, dilations: List[int]): + super().__init__() + kernel_sizes = (config.residual_kernel_size, 1) + if len(kernel_sizes) != len(dilations): + raise ValueError("Number of kernel sizes should match number of dilations") + + hidden = dim // config.compress + block = [] + for i, (kernel_size, dilation) in enumerate(zip(kernel_sizes, dilations)): + in_chs = dim if i == 0 else hidden + out_chs = dim if i == len(kernel_sizes) - 1 else hidden + block += [nn.ELU()] + block += [EncodecConv1d(config, in_chs, out_chs, kernel_size, dilation=dilation)] + self.block = nn.ModuleList(block) + + self.shortcut = EncodecConv1d(config, dim, dim, kernel_size=1) + + def forward(self, hidden_states): + residual = hidden_states + for layer in self.block: + hidden_states = layer(hidden_states) + + return self.shortcut(residual) + hidden_states + + +class EncodecEncoder(nn.Module): + """SEANet encoder as used by EnCodec.""" + + def __init__(self, config: EncodecConfig): + super().__init__() + model = [EncodecConv1d(config, config.audio_channels, config.num_filters, config.kernel_size)] + scaling = 1 + + # Downsample to raw audio scale + for ratio in reversed(config.upsampling_ratios): + current_scale = scaling * config.num_filters + # Add residual layers + for j in range(config.num_residual_layers): + model += [EncodecResnetBlock(config, current_scale, [config.dilation_growth_rate**j, 1])] + # Add downsampling layers + model += [nn.ELU()] + model += [EncodecConv1d(config, current_scale, current_scale * 2, kernel_size=ratio * 2, stride=ratio)] + scaling *= 2 + + model += [EncodecLSTM(config, scaling * config.num_filters)] + model += [nn.ELU()] + model += [EncodecConv1d(config, scaling * config.num_filters, config.hidden_size, config.last_kernel_size)] + + self.layers = nn.ModuleList(model) + + def forward(self, hidden_states): + for layer in self.layers: + hidden_states = layer(hidden_states) + return hidden_states + + +class EncodecDecoder(nn.Module): + """SEANet decoder as used by EnCodec.""" + + def __init__(self, config: EncodecConfig): + super().__init__() + scaling = int(2 ** len(config.upsampling_ratios)) + model = [EncodecConv1d(config, config.hidden_size, scaling * config.num_filters, config.kernel_size)] + + model += [EncodecLSTM(config, scaling * config.num_filters)] + + # Upsample to raw audio scale + for ratio in config.upsampling_ratios: + current_scale = scaling * config.num_filters + # Add upsampling layers + model += [nn.ELU()] + model += [ + EncodecConvTranspose1d(config, current_scale, current_scale // 2, kernel_size=ratio * 2, stride=ratio) + ] + # Add residual layers + for j in range(config.num_residual_layers): + model += [EncodecResnetBlock(config, current_scale // 2, (config.dilation_growth_rate**j, 1))] + scaling //= 2 + + # Add final layers + model += [nn.ELU()] + model += [EncodecConv1d(config, config.num_filters, config.audio_channels, config.last_kernel_size)] + self.layers = nn.ModuleList(model) + + def forward(self, hidden_states): + for layer in self.layers: + hidden_states = layer(hidden_states) + return hidden_states + + +class EncodecEuclideanCodebook(nn.Module): + """Codebook with Euclidean distance.""" + + def __init__(self, config: EncodecConfig): + super().__init__() + embed = torch.zeros(config.codebook_size, config.codebook_dim) + + self.codebook_size = config.codebook_size + + self.register_buffer("inited", torch.Tensor([True])) + self.register_buffer("cluster_size", torch.zeros(config.codebook_size)) + self.register_buffer("embed", embed) + self.register_buffer("embed_avg", embed.clone()) + + def quantize(self, hidden_states): + embed = self.embed.t() + scaled_states = hidden_states.pow(2).sum(1, keepdim=True) + dist = -(scaled_states - 2 * hidden_states @ embed + embed.pow(2).sum(0, keepdim=True)) + embed_ind = dist.max(dim=-1).indices + return embed_ind + + def encode(self, hidden_states): + shape = hidden_states.shape + # pre-process + hidden_states = hidden_states.reshape((-1, shape[-1])) + # quantize + embed_ind = self.quantize(hidden_states) + # post-process + embed_ind = embed_ind.view(*shape[:-1]) + return embed_ind + + def decode(self, embed_ind): + quantize = nn.functional.embedding(embed_ind, self.embed) + return quantize + + +class EncodecVectorQuantization(nn.Module): + """ + Vector quantization implementation. Currently supports only euclidean distance. + """ + + def __init__(self, config: EncodecConfig): + super().__init__() + self.codebook = EncodecEuclideanCodebook(config) + + def encode(self, hidden_states): + hidden_states = hidden_states.permute(0, 2, 1) + embed_in = self.codebook.encode(hidden_states) + return embed_in + + def decode(self, embed_ind): + quantize = self.codebook.decode(embed_ind) + quantize = quantize.permute(0, 2, 1) + return quantize + + +class EncodecResidualVectorQuantizer(nn.Module): + """Residual Vector Quantizer.""" + + def __init__(self, config: EncodecConfig): + super().__init__() + self.codebook_size = config.codebook_size + self.frame_rate = config.frame_rate + self.num_quantizers = config.num_quantizers + self.layers = nn.ModuleList([EncodecVectorQuantization(config) for _ in range(config.num_quantizers)]) + + def get_num_quantizers_for_bandwidth(self, bandwidth: Optional[float] = None) -> int: + """Return num_quantizers based on specified target bandwidth.""" + bw_per_q = math.log2(self.codebook_size) * self.frame_rate + num_quantizers = self.num_quantizers + if bandwidth is not None and bandwidth > 0.0: + num_quantizers = int(max(1, math.floor(bandwidth * 1000 / bw_per_q))) + return num_quantizers + + def encode(self, embeddings: torch.Tensor, bandwidth: Optional[float] = None) -> torch.Tensor: + """ + Encode a given input tensor with the specified frame rate at the given bandwidth. The RVQ encode method sets + the appropriate number of quantizers to use and returns indices for each quantizer. + """ + num_quantizers = self.get_num_quantizers_for_bandwidth(bandwidth) + residual = embeddings + all_indices = [] + for layer in self.layers[:num_quantizers]: + indices = layer.encode(residual) + quantized = layer.decode(indices) + residual = residual - quantized + all_indices.append(indices) + out_indices = torch.stack(all_indices) + return out_indices + + def decode(self, codes: torch.Tensor) -> torch.Tensor: + """Decode the given codes to the quantized representation.""" + quantized_out = torch.tensor(0.0, device=codes.device) + for i, indices in enumerate(codes): + layer = self.layers[i] + quantized = layer.decode(indices) + quantized_out = quantized_out + quantized + return quantized_out + + +class EncodecPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = EncodecConfig + base_model_prefix = "encodec" + main_input_name = "input_values" + supports_gradient_checkpointing = True + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + elif isinstance(module, nn.Conv1d): + nn.init.kaiming_normal_(module.weight) + if module.bias is not None: + k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0])) + nn.init.uniform_(module.bias, a=-k, b=k) + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LSTM): + for name, param in module.named_parameters(): + if "weight" in name: + nn.init.xavier_uniform_(param) + elif "bias" in name: + nn.init.constant_(param, 0.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (EncodecEncoder, EncodecDecoder)): + module.gradient_checkpointing = value + + +ENCODEC_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`EncodecConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +ENCODEC_INPUTS_DOCSTRING = r""" + Args: + input_values (`torch.FloatTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): + Raw audio input converted to Float and padded to the approriate length in order to be encoded using chunks + of length self.chunk_length and a stride of `config.chunk_stride`. + padding_mask (`torch.BoolTensor` of shape `(batch_size, channels, sequence_length)`, *optional*): + Mask to avoid computing scaling factors on padding token indices (can we avoid computing conv on these+). + Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + + + `padding_mask` should always be passed, unless the input was truncated or not padded. This is because in + order to process tensors effectively, the input audio should be padded so that `input_length % stride = + step` with `step = chunk_length-stride`. This ensures that all chunks are of the same shape + + + + bandwidth (`float`, *optional*): + The target bandwidth. Must be one of `config.target_bandwidths`. If `None`, uses the smallest possible + bandwidth. bandwidth is represented as a thousandth of what it is, e.g. 6kbps bandwidth is represented as + `bandwidth == 6.0` + audio_codes (`torch.FloatTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*): + Discret code embeddings computed using `model.encode`. + audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*): + Scaling factor for each `audio_codes` input. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The EnCodec neural audio codec model.", + ENCODEC_START_DOCSTRING, +) +class EncodecModel(EncodecPreTrainedModel): + def __init__(self, config: EncodecConfig): + super().__init__(config) + self.config = config + + self.encoder = EncodecEncoder(config) + self.decoder = EncodecDecoder(config) + + self.quantizer = EncodecResidualVectorQuantizer(config) + + self.bits_per_codebook = int(math.log2(self.config.codebook_size)) + if 2**self.bits_per_codebook != self.config.codebook_size: + raise ValueError("The codebook_size must be a power of 2.") + + # Initialize weights and apply final processing + self.post_init() + + def get_encoder(self): + return self.encoder + + def get_decoder(self): + return self.decoder + + def _encode_frame( + self, input_values: torch.Tensor, bandwidth: float, padding_mask: int + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Encodes the given input using the underlying VQVAE. If `config.normalize` is set to `True` the input is first + normalized. The padding mask is required to compute the correct scale. + """ + length = input_values.shape[-1] + duration = length / self.config.sampling_rate + + if self.config.chunk_length_s is not None and duration > 1e-5 + self.config.chunk_length_s: + raise RuntimeError(f"Duration of frame ({duration}) is longer than chunk {self.config.chunk_length_s}") + + scale = None + if self.config.normalize: + # if the padding is non zero + input_values = input_values * padding_mask + mono = torch.sum(input_values, 1, keepdim=True) / input_values.shape[1] + scale = mono.pow(2).mean(dim=-1, keepdim=True).sqrt() + 1e-8 + input_values = input_values / scale + + embeddings = self.encoder(input_values) + codes = self.quantizer.encode(embeddings, bandwidth) + codes = codes.transpose(0, 1) + return codes, scale + + def encode( + self, + input_values: torch.Tensor, + padding_mask: torch.Tensor = None, + bandwidth: Optional[float] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor, Optional[torch.Tensor]], EncodecEncoderOutput]: + """ + Encodes the input audio waveform into discrete codes. + + Args: + input_values (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`): + Float values of the input audio waveform. + padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`): + Padding mask used to pad the `input_values`. + bandwidth (`float`, *optional*): + The target bandwidth. Must be one of `config.target_bandwidths`. If `None`, uses the smallest possible + bandwidth. bandwidth is represented as a thousandth of what it is, e.g. 6kbps bandwidth is represented + as bandwidth == 6.0 + + Returns: + A list of frames containing the discrete encoded codes for the input audio waveform, along with rescaling + factors for each chunk when `normalize` is True. Each frames is a tuple `(codebook, scale)`, with + `codebook` of shape `[batch_size, num_codebooks, frames]`. + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if bandwidth is None: + bandwidth = self.config.target_bandwidths[0] + if bandwidth not in self.config.target_bandwidths: + raise ValueError( + f"This model doesn't support the bandwidth {bandwidth}. " + f"Select one of {self.config.target_bandwidths}." + ) + + _, channels, input_length = input_values.shape + + if channels < 1 or channels > 2: + raise ValueError(f"Number of audio channels must be 1 or 2, but got {channels}") + + chunk_length = self.config.chunk_length + if chunk_length is None: + chunk_length = input_length + stride = input_length + else: + stride = self.config.chunk_stride + + if padding_mask is None: + padding_mask = torch.ones_like(input_values).bool() + + encoded_frames = [] + scales = [] + + step = chunk_length - stride + if (input_length % stride) - step != 0: + raise ValueError( + "The input length is not properly padded for batched chunked decoding. Make sure to pad the input correctly." + ) + + for offset in range(0, input_length - step, stride): + mask = padding_mask[..., offset : offset + chunk_length].bool() + frame = input_values[:, :, offset : offset + chunk_length] + encoded_frame, scale = self._encode_frame(frame, bandwidth, mask) + encoded_frames.append(encoded_frame) + scales.append(scale) + + encoded_frames = torch.stack(encoded_frames) + + if not return_dict: + return (encoded_frames, scales) + + return EncodecEncoderOutput(encoded_frames, scales) + + @staticmethod + def _linear_overlap_add(frames: List[torch.Tensor], stride: int): + # Generic overlap add, with linear fade-in/fade-out, supporting complex scenario + # e.g., more than 2 frames per position. + # The core idea is to use a weight function that is a triangle, + # with a maximum value at the middle of the chunk. + # We use this weighting when summing the frames, and divide by the sum of weights + # for each positions at the end. Thus: + # - if a frame is the only one to cover a position, the weighting is a no-op. + # - if 2 frames cover a position: + # ... ... + # / \/ \ + # / /\ \ + # S T , i.e. S offset of second frame starts, T end of first frame. + # Then the weight function for each one is: (t - S), (T - t), with `t` a given offset. + # After the final normalization, the weight of the second frame at position `t` is + # (t - S) / (t - S + (T - t)) = (t - S) / (T - S), which is exactly what we want. + # + # - if more than 2 frames overlap at a given point, we hope that by induction + # something sensible happens. + if len(frames) == 0: + raise ValueError("`frames` cannot be an empty list.") + + device = frames[0].device + dtype = frames[0].dtype + shape = frames[0].shape[:-1] + total_size = stride * (len(frames) - 1) + frames[-1].shape[-1] + + frame_length = frames[0].shape[-1] + time_vec = torch.linspace(0, 1, frame_length + 2, device=device, dtype=dtype)[1:-1] + weight = 0.5 - (time_vec - 0.5).abs() + + sum_weight = torch.zeros(total_size, device=device, dtype=dtype) + out = torch.zeros(*shape, total_size, device=device, dtype=dtype) + offset: int = 0 + + for frame in frames: + frame_length = frame.shape[-1] + out[..., offset : offset + frame_length] += weight[:frame_length] * frame + sum_weight[offset : offset + frame_length] += weight[:frame_length] + offset += stride + + if sum_weight.min() == 0: + raise ValueError(f"`sum_weight` minimum element must be bigger than zero: {sum_weight}`") + + return out / sum_weight + + def _decode_frame(self, codes: torch.Tensor, scale: Optional[torch.Tensor] = None) -> torch.Tensor: + codes = codes.transpose(0, 1) + embeddings = self.quantizer.decode(codes) + outputs = self.decoder(embeddings) + if scale is not None: + outputs = outputs * scale.view(-1, 1, 1) + return outputs + + def decode( + self, + audio_codes: torch.Tensor, + audio_scales: torch.Tensor, + padding_mask: Optional[torch.Tensor] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor, torch.Tensor], EncodecDecoderOutput]: + """ + Decodes the given frames into an output audio waveform. + + Note that the output might be a bit bigger than the input. In that case, any extra steps at the end can be + trimmed. + + Args: + audio_codes (`torch.FloatTensor` of shape `(batch_size, nb_chunks, chunk_length)`, *optional*): + Discret code embeddings computed using `model.encode`. + audio_scales (`torch.Tensor` of shape `(batch_size, nb_chunks)`, *optional*): + Scaling factor for each `audio_codes` input. + padding_mask (`torch.Tensor` of shape `(batch_size, channels, sequence_length)`): + Padding mask used to pad the `input_values`. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + """ + return_dict = return_dict or self.config.return_dict + + chunk_length = self.config.chunk_length + if chunk_length is None: + if len(audio_codes) != 1: + raise ValueError(f"Expected one frame, got {len(audio_codes)}") + audio_values = self._decode_frame(audio_codes[0], audio_scales[0]) + else: + decoded_frames = [] + + for frame, scale in zip(audio_codes, audio_scales): + frames = self._decode_frame(frame, scale) + decoded_frames.append(frames) + + audio_values = self._linear_overlap_add(decoded_frames, self.config.chunk_stride or 1) + + # truncate based on padding mask + if padding_mask is not None and padding_mask.shape[-1] < audio_values.shape[-1]: + audio_values = audio_values[..., : padding_mask.shape[-1]] + + if not return_dict: + return (audio_values,) + return EncodecDecoderOutput(audio_values) + + @add_start_docstrings_to_model_forward(ENCODEC_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=EncodecOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_values: torch.Tensor, + padding_mask: Optional[torch.Tensor] = None, + bandwidth: Optional[float] = None, + audio_codes: Optional[torch.Tensor] = None, + audio_scales: Optional[torch.Tensor] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor, torch.Tensor], EncodecOutput]: + r""" + Returns: + + Examples: + + ```python + >>> from datasets import load_dataset + >>> from transformers import AutoProcessor, EncodecModel + + >>> dataset = load_dataset("ashraq/esc50") + >>> audio_sample = dataset["train"]["audio"][0]["array"] + + >>> model_id = "facebook/encodec_24khz" + >>> model = EncodecModel.from_pretrained(model_id) + >>> processor = AutoProcessor.from_pretrained(model_id) + + >>> inputs = processor(raw_audio=audio_sample, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> audio_codes = outputs.audio_codes + >>> audio_values = outputs.audio_values + ```""" + return_dict = return_dict or self.config.return_dict + + if padding_mask is None: + padding_mask = torch.ones_like(input_values).bool() + + if audio_codes is not None and audio_scales is None: + raise ValueError("You specified `audio_codes` but did not specify the `audio_scales`") + + if audio_scales is not None and audio_codes is None: + raise ValueError("You specified `audio_scales` but did not specify the `audio_codes`") + + if audio_scales is None and audio_codes is None: + audio_codes, audio_scales = self.encode(input_values, padding_mask, bandwidth, False) + + audio_values = self.decode(audio_codes, audio_scales, padding_mask, return_dict=return_dict)[0] + if not return_dict: + return (audio_codes, audio_values) + + return EncodecOutput(audio_codes=audio_codes, audio_values=audio_values) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 1eba8c0ca754b6..11e763e2a08eda 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -2655,6 +2655,23 @@ def load_tf_weights_in_electra(*args, **kwargs): requires_backends(load_tf_weights_in_electra, ["torch"]) +ENCODEC_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class EncodecModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class EncodecPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class EncoderDecoderModel(metaclass=DummyObject): _backends = ["torch"] diff --git a/src/transformers/utils/dummy_speech_objects.py b/src/transformers/utils/dummy_speech_objects.py index f85182c8bc40ed..6c591588581deb 100644 --- a/src/transformers/utils/dummy_speech_objects.py +++ b/src/transformers/utils/dummy_speech_objects.py @@ -9,6 +9,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["speech"]) +class EncodecFeatureExtractor(metaclass=DummyObject): + _backends = ["speech"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["speech"]) + + class MCTCTFeatureExtractor(metaclass=DummyObject): _backends = ["speech"] diff --git a/tests/models/encodec/__init__.py b/tests/models/encodec/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/tests/models/encodec/test_feature_extraction_encodec.py b/tests/models/encodec/test_feature_extraction_encodec.py new file mode 100644 index 00000000000000..95639fcda5ff1e --- /dev/null +++ b/tests/models/encodec/test_feature_extraction_encodec.py @@ -0,0 +1,255 @@ +# coding=utf-8 +# Copyright 2021-2023 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for the EnCodec feature extractor.""" + +import itertools +import random +import unittest + +import numpy as np + +from transformers import is_speech_available +from transformers.testing_utils import require_torch +from transformers.utils.import_utils import is_torch_available + +from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin + + +if is_speech_available(): + from transformers import EncodecFeatureExtractor + +if is_torch_available(): + import torch + + +global_rng = random.Random() + + +def floats_list(shape, scale=1.0, rng=None, name=None): + """Creates a random float32 tensor""" + if rng is None: + rng = global_rng + + values = [] + for batch_idx in range(shape[0]): + values.append([]) + for _ in range(shape[1]): + values[-1].append(rng.random() * scale) + + return values + + +@require_torch +class EnCodecFeatureExtractionTester(unittest.TestCase): + def __init__( + self, + parent, + batch_size=7, + min_seq_length=400, + max_seq_length=2000, + feature_size=1, + padding_value=0.0, + sampling_rate=24000, + return_attention_mask=True, + ): + self.parent = parent + self.batch_size = batch_size + self.min_seq_length = min_seq_length + self.max_seq_length = max_seq_length + self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1) + self.feature_size = feature_size + self.padding_value = padding_value + self.sampling_rate = sampling_rate + self.return_attention_mask = return_attention_mask + + def prepare_feat_extract_dict(self): + return { + "feature_size": self.feature_size, + "padding_value": self.padding_value, + "sampling_rate": self.sampling_rate, + "return_attention_mask": self.return_attention_mask, + } + + def prepare_inputs_for_common(self, equal_length=False, numpify=False): + def _flatten(list_of_lists): + return list(itertools.chain(*list_of_lists)) + + if equal_length: + audio_inputs = floats_list((self.batch_size, self.max_seq_length)) + else: + # make sure that inputs increase in size + audio_inputs = [ + _flatten(floats_list((x, self.feature_size))) + for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff) + ] + + if numpify: + audio_inputs = [np.asarray(x) for x in audio_inputs] + + return audio_inputs + + +@require_torch +class EnCodecFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase): + feature_extraction_class = EncodecFeatureExtractor if is_speech_available() else None + + def setUp(self): + self.feat_extract_tester = EnCodecFeatureExtractionTester(self) + + def test_call(self): + # Tests that all call wrap to encode_plus and batch_encode_plus + feat_extract = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()) + # create three inputs of length 800, 1000, and 1200 + audio_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)] + np_audio_inputs = [np.asarray(audio_input) for audio_input in audio_inputs] + + # Test not batched input + encoded_sequences_1 = feat_extract(audio_inputs[0], return_tensors="np").input_values + encoded_sequences_2 = feat_extract(np_audio_inputs[0], return_tensors="np").input_values + self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3)) + + # Test batched + encoded_sequences_1 = feat_extract(audio_inputs, padding=True, return_tensors="np").input_values + encoded_sequences_2 = feat_extract(np_audio_inputs, padding=True, return_tensors="np").input_values + for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2): + self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3)) + + def test_double_precision_pad(self): + feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()) + np_audio_inputs = np.random.rand(100).astype(np.float64) + py_audio_inputs = np_audio_inputs.tolist() + + for inputs in [py_audio_inputs, np_audio_inputs]: + np_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="np") + self.assertTrue(np_processed.input_values.dtype == np.float32) + pt_processed = feature_extractor.pad([{"input_values": inputs}], return_tensors="pt") + self.assertTrue(pt_processed.input_values.dtype == torch.float32) + + def _load_datasamples(self, num_samples): + from datasets import load_dataset + + ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + # automatic decoding with librispeech + audio_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"] + + return [x["array"] for x in audio_samples] + + def test_integration(self): + # fmt: off + EXPECTED_INPUT_VALUES = torch.tensor( + [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03, + 3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03, + 2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04, + 4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03, + 7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04, + 4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03] + ) + # fmt: on + input_audio = self._load_datasamples(1) + feature_extractor = EncodecFeatureExtractor() + input_values = feature_extractor(input_audio, return_tensors="pt").input_values + self.assertEquals(input_values.shape, (1, 1, 93680)) + self.assertTrue(torch.allclose(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, atol=1e-6)) + + def test_integration_stereo(self): + # fmt: off + EXPECTED_INPUT_VALUES = torch.tensor( + [2.3804e-03, 2.0752e-03, 1.9836e-03, 2.1057e-03, 1.6174e-03, + 3.0518e-04, 9.1553e-05, 3.3569e-04, 9.7656e-04, 1.8311e-03, + 2.0142e-03, 2.1057e-03, 1.7395e-03, 4.5776e-04, -3.9673e-04, + 4.5776e-04, 1.0071e-03, 9.1553e-05, 4.8828e-04, 1.1597e-03, + 7.3242e-04, 9.4604e-04, 1.8005e-03, 1.8311e-03, 8.8501e-04, + 4.2725e-04, 4.8828e-04, 7.3242e-04, 1.0986e-03, 2.1057e-03] + ) + # fmt: on + input_audio = self._load_datasamples(1) + input_audio = [np.tile(input_audio[0][None], reps=(2, 1))] + input_audio[0][1] *= 0.5 + feature_extractor = EncodecFeatureExtractor(feature_size=2) + input_values = feature_extractor(input_audio, return_tensors="pt").input_values + self.assertEquals(input_values.shape, (1, 2, 93680)) + self.assertTrue(torch.allclose(input_values[0, 0, :30], EXPECTED_INPUT_VALUES, atol=1e-6)) + self.assertTrue(torch.allclose(input_values[0, 1, :30], EXPECTED_INPUT_VALUES * 0.5, atol=1e-6)) + + def test_truncation_and_padding(self): + input_audio = self._load_datasamples(2) + # would be easier if the stride was like + feature_extractor = EncodecFeatureExtractor(feature_size=1, chunk_length_s=1, overlap=0.01) + + # pad and trunc raise an error ? + with self.assertRaisesRegex( + ValueError, + "^Both padding and truncation were set. Make sure you only set one.$", + ): + truncated_outputs = feature_extractor( + input_audio, padding="max_length", truncation=True, return_tensors="pt" + ).input_values + + # truncate to chunk + truncated_outputs = feature_extractor(input_audio, truncation=True, return_tensors="pt").input_values + self.assertEquals(truncated_outputs.shape, (2, 1, 71520)) # 2 chunks + + # force truncate to max_length + truncated_outputs = feature_extractor( + input_audio, truncation=True, max_length=48000, return_tensors="pt" + ).input_values + self.assertEquals(truncated_outputs.shape, (2, 1, 48000)) + + # pad to chunk + padded_outputs = feature_extractor(input_audio, padding=True, return_tensors="pt").input_values + self.assertEquals(padded_outputs.shape, (2, 1, 95280)) + + # pad to chunk + truncated_outputs = feature_extractor(input_audio, return_tensors="pt").input_values + self.assertEquals(truncated_outputs.shape, (2, 1, 95280)) + + # force pad to max length + truncated_outputs = feature_extractor( + input_audio, padding="max_length", max_length=100000, return_tensors="pt" + ).input_values + self.assertEquals(truncated_outputs.shape, (2, 1, 100000)) + + # force no pad + with self.assertRaisesRegex( + ValueError, + "^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$", + ): + truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values + + truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values + self.assertEquals(truncated_outputs.shape, (1, 1, 93680)) + + # no pad if no chunk_length_s + feature_extractor.chunk_length_s = None + with self.assertRaisesRegex( + ValueError, + "^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$", + ): + truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values + + truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values + self.assertEquals(truncated_outputs.shape, (1, 1, 93680)) + + # no pad if no overlap + feature_extractor.chunk_length_s = 2 + feature_extractor.overlap = None + with self.assertRaisesRegex( + ValueError, + "^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$", + ): + truncated_outputs = feature_extractor(input_audio, padding=False, return_tensors="pt").input_values + + truncated_outputs = feature_extractor(input_audio[0], padding=False, return_tensors="pt").input_values + self.assertEquals(truncated_outputs.shape, (1, 1, 93680)) diff --git a/tests/models/encodec/test_modeling_encodec.py b/tests/models/encodec/test_modeling_encodec.py new file mode 100644 index 00000000000000..23b2114a5db378 --- /dev/null +++ b/tests/models/encodec/test_modeling_encodec.py @@ -0,0 +1,571 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch Encodec model. """ + +import copy +import inspect +import os +import tempfile +import unittest +from typing import Dict, List, Tuple + +import numpy as np +from datasets import Audio, load_dataset + +from transformers import AutoProcessor, EncodecConfig +from transformers.testing_utils import ( + is_torch_available, + require_torch, + slow, + torch_device, +) + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ( + ModelTesterMixin, + _config_zero_init, + floats_tensor, +) +from ...test_pipeline_mixin import PipelineTesterMixin + + +if is_torch_available(): + import torch + + from transformers import EncodecModel + + +def prepare_inputs_dict( + config, + input_ids=None, + input_values=None, + decoder_input_ids=None, + attention_mask=None, + decoder_attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, +): + if input_ids is not None: + encoder_dict = {"input_ids": input_ids} + else: + encoder_dict = {"input_values": input_values} + + decoder_dict = {"decoder_input_ids": decoder_input_ids} if decoder_input_ids is not None else {} + + return {**encoder_dict, **decoder_dict} + + +@require_torch +class EncodecModelTester: + def __init__( + self, + parent, + batch_size=13, + num_channels=2, + is_training=False, + num_hidden_layers=4, + intermediate_size=40, + ): + self.parent = parent + self.batch_size = batch_size + self.num_channels = num_channels + self.is_training = is_training + + self.num_hidden_layers = num_hidden_layers + self.intermediate_size = intermediate_size + + def prepare_config_and_inputs(self): + input_values = floats_tensor([self.batch_size, self.num_channels, self.intermediate_size], scale=1.0) + config = self.get_config() + inputs_dict = {"input_values": input_values} + return config, inputs_dict + + def prepare_config_and_inputs_for_common(self): + config, inputs_dict = self.prepare_config_and_inputs() + return config, inputs_dict + + def get_config(self): + return EncodecConfig(audio_channels=self.num_channels, chunk_in_sec=None) + + def create_and_check_model_forward(self, config, inputs_dict): + model = EncodecModel(config=config).to(torch_device).eval() + + input_values = inputs_dict["input_values"] + result = model(input_values) + self.parent.assertEqual( + result.audio_values.shape, (self.batch_size, self.num_channels, self.intermediate_size) + ) + + +@require_torch +class EncodecModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): + all_model_classes = (EncodecModel,) if is_torch_available() else () + is_encoder_decoder = True + test_pruning = False + test_headmasking = False + test_resize_embeddings = False + pipeline_model_mapping = {} + input_name = "input_values" + + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): + # model does not have attention and does not support returning hidden states + inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) + if "output_attentions" in inputs_dict: + inputs_dict.pop("output_attentions") + if "output_hidden_states" in inputs_dict: + inputs_dict.pop("output_hidden_states") + return inputs_dict + + def setUp(self): + self.model_tester = EncodecModelTester(self) + self.config_tester = ConfigTester( + self, config_class=EncodecConfig, hidden_size=37, common_properties=[], has_text_modality=False + ) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model_forward(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model_forward(*config_and_inputs) + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["input_values", "padding_mask", "bandwidth"] + self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have `inputs_embeds` logics") + def test_inputs_embeds(self): + pass + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have `inputs_embeds` logics") + def test_model_common_attributes(self): + pass + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have the usual `attention` logic") + def test_retain_grad_hidden_states_attentions(self): + pass + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have the usual `attention` logic") + def test_torchscript_output_attentions(self): + pass + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have the usual `hidden_states` logic") + def test_torchscript_output_hidden_state(self): + pass + + def _create_and_check_torchscript(self, config, inputs_dict): + if not self.test_torchscript: + return + + configs_no_init = _config_zero_init(config) # To be sure we have no Nan + configs_no_init.torchscript = True + configs_no_init.return_dict = False + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + model.to(torch_device) + model.eval() + inputs = self._prepare_for_class(inputs_dict, model_class) + + main_input_name = model_class.main_input_name + + try: + main_input = inputs[main_input_name] + model(main_input) + traced_model = torch.jit.trace(model, main_input) + except RuntimeError: + self.fail("Couldn't trace module.") + + with tempfile.TemporaryDirectory() as tmp_dir_name: + pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt") + + try: + torch.jit.save(traced_model, pt_file_name) + except Exception: + self.fail("Couldn't save module.") + + try: + loaded_model = torch.jit.load(pt_file_name) + except Exception: + self.fail("Couldn't load module.") + + model.to(torch_device) + model.eval() + + loaded_model.to(torch_device) + loaded_model.eval() + + model_state_dict = model.state_dict() + loaded_model_state_dict = loaded_model.state_dict() + + non_persistent_buffers = {} + for key in loaded_model_state_dict.keys(): + if key not in model_state_dict.keys(): + non_persistent_buffers[key] = loaded_model_state_dict[key] + + loaded_model_state_dict = { + key: value for key, value in loaded_model_state_dict.items() if key not in non_persistent_buffers + } + + self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys())) + + model_buffers = list(model.buffers()) + for non_persistent_buffer in non_persistent_buffers.values(): + found_buffer = False + for i, model_buffer in enumerate(model_buffers): + if torch.equal(non_persistent_buffer, model_buffer): + found_buffer = True + break + + self.assertTrue(found_buffer) + model_buffers.pop(i) + + models_equal = True + for layer_name, p1 in model_state_dict.items(): + if layer_name in loaded_model_state_dict: + p2 = loaded_model_state_dict[layer_name] + if p1.data.ne(p2.data).sum() > 0: + models_equal = False + + self.assertTrue(models_equal) + + # Avoid memory leak. Without this, each call increase RAM usage by ~20MB. + # (Even with this call, there are still memory leak by ~0.04MB) + self.clear_torch_jit_class_registry() + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have the usual `attention` logic") + def test_attention_outputs(self): + pass + + def test_feed_forward_chunking(self): + (original_config, inputs_dict) = self.model_tester.prepare_config_and_inputs_for_common() + for model_class in self.all_model_classes: + torch.manual_seed(0) + config = copy.deepcopy(original_config) + config.chunk_length_s = None + config.overlap = None + config.sampling_rate = 10 + + model = model_class(config) + model.to(torch_device) + model.eval() + inputs = self._prepare_for_class(inputs_dict, model_class) + inputs["input_values"] = inputs["input_values"].repeat(1, 1, 10) + + hidden_states_no_chunk = model(**inputs)[0] + + torch.manual_seed(0) + config.chunk_length_s = 1 + config.overlap = 0 + config.sampling_rate = 10 + + model = model_class(config) + model.to(torch_device) + model.eval() + + hidden_states_with_chunk = model(**inputs)[0] + self.assertTrue(torch.allclose(hidden_states_no_chunk, hidden_states_with_chunk, atol=1e-3)) + + @unittest.skip("The EncodecModel is not transformers based, thus it does not have the usual `hidden_states` logic") + def test_hidden_states_output(self): + pass + + def test_determinism(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + def check_determinism(first, second): + # outputs are not tensors but list (since each sequence don't have the same frame_length) + out_1 = first.cpu().numpy() + out_2 = second.cpu().numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) + + for model_class in self.all_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + first = model(**self._prepare_for_class(inputs_dict, model_class))[0] + second = model(**self._prepare_for_class(inputs_dict, model_class))[0] + + if isinstance(first, tuple) and isinstance(second, tuple): + for tensor1, tensor2 in zip(first, second): + check_determinism(tensor1, tensor2) + else: + check_determinism(first, second) + + def test_model_outputs_equivalence(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + def set_nan_tensor_to_zero(t): + t[t != t] = 0 + return t + + def check_equivalence(model, tuple_inputs, dict_inputs, additional_kwargs={}): + with torch.no_grad(): + tuple_output = model(**tuple_inputs, return_dict=False, **additional_kwargs) + dict_output = model(**dict_inputs, return_dict=True, **additional_kwargs) + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip( + tuple_object.values(), dict_object.values() + ): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + torch.allclose( + set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5 + ), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has" + f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}." + ), + ) + + recursive_check(tuple_output, dict_output) + + for model_class in self.all_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + tuple_inputs = self._prepare_for_class(inputs_dict, model_class) + dict_inputs = self._prepare_for_class(inputs_dict, model_class) + check_equivalence(model, tuple_inputs, dict_inputs) + + def test_initialization(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + configs_no_init = _config_zero_init(config) + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + for name, param in model.named_parameters(): + uniform_init_parms = ["conv"] + ignore_init = ["lstm"] + if param.requires_grad: + if any([x in name for x in uniform_init_parms]): + self.assertTrue( + -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + elif not any([x in name for x in ignore_init]): + self.assertIn( + ((param.data.mean() * 1e9).round() / 1e9).item(), + [0.0, 1.0], + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + + +def normalize(arr): + norm = np.linalg.norm(arr) + normalized_arr = arr / norm + return normalized_arr + + +def compute_rmse(arr1, arr2): + arr1_normalized = normalize(arr1) + arr2_normalized = normalize(arr2) + return np.sqrt(((arr1_normalized - arr2_normalized) ** 2).mean()) + + +@slow +@require_torch +class EncodecIntegrationTest(unittest.TestCase): + def test_integration_24kHz(self): + expected_rmse = { + "1.5": 0.0025, + "24.0": 0.0015, + } + expected_codesums = { + "1.5": [367184], + "24.0": [6648961], + } + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + model_id = "facebook/encodec_24khz" + + model = EncodecModel.from_pretrained(model_id).to(torch_device) + processor = AutoProcessor.from_pretrained(model_id) + + librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) + audio_sample = librispeech_dummy[-1]["audio"]["array"] + + inputs = processor( + raw_audio=audio_sample, + sampling_rate=processor.sampling_rate, + return_tensors="pt", + ).to(torch_device) + + for bandwidth, expected_rmse in expected_rmse.items(): + with torch.no_grad(): + # use max bandwith for best possible reconstruction + encoder_outputs = model.encode(inputs["input_values"], bandwidth=float(bandwidth)) + + audio_code_sums = [a[0].sum().cpu().item() for a in encoder_outputs[0]] + + # make sure audio encoded codes are correct + self.assertListEqual(audio_code_sums, expected_codesums[bandwidth]) + + audio_codes, scales = encoder_outputs.to_tuple() + input_values_dec = model.decode(audio_codes, scales, inputs["padding_mask"])[0] + input_values_enc_dec = model( + inputs["input_values"], inputs["padding_mask"], bandwidth=float(bandwidth) + )[-1] + + # make sure forward and decode gives same result + self.assertTrue(torch.allclose(input_values_dec, input_values_enc_dec, atol=1e-3)) + + # make sure shape matches + self.assertTrue(inputs["input_values"].shape == input_values_enc_dec.shape) + + arr = inputs["input_values"][0].cpu().numpy() + arr_enc_dec = input_values_enc_dec[0].cpu().numpy() + + # make sure audios are more or less equal + # the RMSE of two random gaussian noise vectors with ~N(0, 1) is around 1.0 + rmse = compute_rmse(arr, arr_enc_dec) + self.assertTrue(rmse < expected_rmse) + + def test_integration_48kHz(self): + expected_rmse = { + "3.0": 0.001, + "24.0": 0.0005, + } + expected_codesums = { + "3.0": [142174, 147901, 154090, 178965, 161879], + "24.0": [1561048, 1284593, 1278330, 1487220, 1659404], + } + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + model_id = "facebook/encodec_48khz" + + model = EncodecModel.from_pretrained(model_id).to(torch_device) + model = model.eval() + processor = AutoProcessor.from_pretrained(model_id) + + librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) + audio_sample = librispeech_dummy[-1]["audio"]["array"] + + # transform mono to stereo + audio_sample = np.array([audio_sample, audio_sample]) + + inputs = processor(raw_audio=audio_sample, sampling_rate=processor.sampling_rate, return_tensors="pt").to( + torch_device + ) + + for bandwidth, expected_rmse in expected_rmse.items(): + with torch.no_grad(): + # use max bandwith for best possible reconstruction + encoder_outputs = model.encode( + inputs["input_values"], inputs["padding_mask"], bandwidth=float(bandwidth), return_dict=False + ) + audio_code_sums = [a[0].sum().cpu().item() for a in encoder_outputs[0]] + + # make sure audio encoded codes are correct + self.assertListEqual(audio_code_sums, expected_codesums[bandwidth]) + audio_codes, scales = encoder_outputs + input_values_dec = model.decode(audio_codes, scales, inputs["padding_mask"])[0] + input_values_enc_dec = model( + inputs["input_values"], inputs["padding_mask"], bandwidth=float(bandwidth) + )[-1] + + # make sure forward and decode gives same result + self.assertTrue(torch.allclose(input_values_dec, input_values_enc_dec, atol=1e-3)) + + # make sure shape matches + self.assertTrue(inputs["input_values"].shape == input_values_enc_dec.shape) + + arr = inputs["input_values"][0].cpu().numpy() + arr_enc_dec = input_values_enc_dec[0].cpu().numpy() + + # make sure audios are more or less equal + # the RMSE of two random gaussian noise vectors with ~N(0, 1) is around 1.0 + rmse = compute_rmse(arr, arr_enc_dec) + self.assertTrue(rmse < expected_rmse) + + def test_batch_48kHz(self): + expected_rmse = { + "3.0": 0.001, + "24.0": 0.0005, + } + expected_codesums = { + "3.0": [ + [71689, 78549, 75644, 88889, 73100, 82509, 71449, 82835], + [84427, 82356, 75809, 52509, 80137, 87672, 87436, 70456], + ], + "24.0": [ + [71689, 78549, 75644, 88889, 73100, 82509, 71449, 82835], + [84427, 82356, 75809, 52509, 80137, 87672, 87436, 70456], + ], + } + librispeech_dummy = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + model_id = "facebook/encodec_48khz" + + model = EncodecModel.from_pretrained(model_id).to(torch_device) + processor = AutoProcessor.from_pretrained(model_id, chunk_length_s=1, overlap=0.01) + + librispeech_dummy = librispeech_dummy.cast_column("audio", Audio(sampling_rate=processor.sampling_rate)) + + audio_samples = [ + np.array([audio_sample["array"], audio_sample["array"]]) + for audio_sample in librispeech_dummy[-2:]["audio"] + ] + + inputs = processor(raw_audio=audio_samples, sampling_rate=processor.sampling_rate, return_tensors="pt") + input_values = inputs["input_values"].to(torch_device) + for bandwidth, expected_rmse in expected_rmse.items(): + with torch.no_grad(): + # use max bandwith for best possible reconstruction + encoder_outputs = model.encode(input_values, bandwidth=float(bandwidth), return_dict=False) + audio_code_sums_0 = [a[0][0].sum().cpu().item() for a in encoder_outputs[0]] + audio_code_sums_1 = [a[0][1].sum().cpu().item() for a in encoder_outputs[0]] + + # make sure audio encoded codes are correct + self.assertListEqual(audio_code_sums_0, expected_codesums[bandwidth][0]) + self.assertListEqual(audio_code_sums_1, expected_codesums[bandwidth][1]) + + audio_codes, scales = encoder_outputs + input_values_dec = model.decode(audio_codes, scales)[0] + input_values_enc_dec = model(input_values, bandwidth=float(bandwidth))[-1] + + # make sure forward and decode gives same result + self.assertTrue(torch.allclose(input_values_dec, input_values_enc_dec, atol=1e-3)) + + # make sure shape matches + self.assertTrue(input_values.shape == input_values_enc_dec.shape) + + arr = input_values[0].cpu().numpy() + arr_enc_dec = input_values_enc_dec[0].cpu().numpy() + + # make sure audios are more or less equal + # the RMSE of two random gaussian noise vectors with ~N(0, 1) is around 1.0 + rmse = compute_rmse(arr, arr_enc_dec) + self.assertTrue(rmse < expected_rmse) diff --git a/utils/check_config_attributes.py b/utils/check_config_attributes.py index 02c3d2276f5d30..63b1bacbbe7451 100644 --- a/utils/check_config_attributes.py +++ b/utils/check_config_attributes.py @@ -31,6 +31,8 @@ CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING SPECIAL_CASES_TO_ALLOW = { + # used to compute the property `self.chunk_length` + "EncodecConfig": ["overlap"], # used as `self.bert_model = BertModel(config, ...)` "DPRConfig": True, # not used in modeling files, but it's an important information diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 5cdded64bbe8fb..63eb4a000265e0 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -83,6 +83,7 @@ src/transformers/models/electra/configuration_electra.py src/transformers/models/electra/modeling_electra.py src/transformers/models/electra/modeling_tf_electra.py src/transformers/models/efficientformer/modeling_tf_efficientformer.py +src/transformers/models/encodec/modeling_encodec.py src/transformers/models/ernie/configuration_ernie.py src/transformers/models/ernie_m/configuration_ernie_m.py src/transformers/models/ernie_m/modeling_ernie_m.py @@ -395,6 +396,7 @@ src/transformers/models/deit/feature_extraction_deit.py src/transformers/models/detr/feature_extraction_detr.py src/transformers/models/donut/feature_extraction_donut.py src/transformers/models/dpt/feature_extraction_dpt.py +src/transformers/models/encodec/feature_extraction_encodec.py src/transformers/models/flava/feature_extraction_flava.py src/transformers/models/glpn/feature_extraction_glpn.py src/transformers/models/imagegpt/feature_extraction_imagegpt.py From 1609a436eca115853b5a4cfd80b9ec2302bb9fcc Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 15 Jun 2023 01:10:27 +0200 Subject: [PATCH 04/15] Add MMS CTC Fine-Tuning (#24281) * Add mms ctc fine tuning * make style * More fixes that are needed * make fix-copies * make draft for README * add new file * move to new file * make style * make style * add quick test * make style * make style --- examples/pytorch/speech-recognition/README.md | 109 +++ .../run_speech_recognition_ctc_adapter.py | 799 ++++++++++++++++++ examples/pytorch/test_pytorch_examples.py | 33 + .../models/hubert/modeling_hubert.py | 8 + src/transformers/models/sew/modeling_sew.py | 8 + .../models/sew_d/modeling_sew_d.py | 8 + .../models/unispeech/modeling_unispeech.py | 8 + .../unispeech_sat/modeling_unispeech_sat.py | 8 + .../models/wav2vec2/modeling_wav2vec2.py | 21 + .../models/wavlm/modeling_wavlm.py | 8 + 10 files changed, 1010 insertions(+) create mode 100755 examples/pytorch/speech-recognition/run_speech_recognition_ctc_adapter.py diff --git a/examples/pytorch/speech-recognition/README.md b/examples/pytorch/speech-recognition/README.md index cf5a05c017839f..6ae2e1abef6026 100644 --- a/examples/pytorch/speech-recognition/README.md +++ b/examples/pytorch/speech-recognition/README.md @@ -26,6 +26,10 @@ limitations under the License. - [Librispeech](#librispeech-ctc) - [Common Voice](#common-voice-ctc) - [Multilingual Librispeech](#multilingual-librispeech-ctc) +- [Automatic Speech Recognition with CTC and Adapter Layers](#connectionist-temporal-classification-with-adapters) + - [Massive Multilingual Speech (MMS)](#mms-model) + - [Examples](#examples-ctc-adapter) + - [Common Voice](#common-voice-ctc-adapter) - [Automatic Speech Recognition with Sequence-to-Sequence](#sequence-to-sequence) - [Whisper Model](#whisper-model) - [Speech-Encoder-Decoder Model](#warm-started-speech-encoder-decoder-model) @@ -243,6 +247,111 @@ they can serve as a baseline to improve upon. | [Multilingual Librispeech](https://huggingface.co/datasets/multilingual_librispeech)| `"german"` | [facebook/wav2vec2-large-xlsr-53](https://huggingface.co/facebook/wav2vec2-large-xlsr-53) | 0.13 | - | 1 GPU Titan 24 GB RAM | 15h04 | [here](https://huggingface.co/patrickvonplaten/wav2vec2-xlsr-53-300m-mls-german-ft) | [run.sh](https://huggingface.co/patrickvonplaten/wav2vec2-xlsr-53-300m-mls-german-ft/blob/main/run.sh) | | [Multilingual Librispeech](https://huggingface.co/datasets/multilingual_librispeech)| `"german"` | [facebook/wav2vec2-xls-r-300m](https://huggingface.co/facebook/wav2vec2-xls-r-300m) | 0.15 | - | 1 GPU Titan 24 GB RAM | 15h04 | [here](https://huggingface.co/patrickvonplaten/wav2vec2-300m-mls-german-ft) | [run.sh](https://huggingface.co/patrickvonplaten/wav2vec2-300m-mls-german-ft/blob/main/run.sh) | +## Connectionist Temporal Classification With Adapters + +The script [`run_speech_recognition_ctc_adapter.py`](https://github.com/huggingface/transformers/blob/main/examples/pytorch/speech-recognition/run_speech_recognition_ctc_adapter.py) can be used to fine-tune adapter layers for [Wav2Vec2-like models like MMS](https://huggingface.co/docs/transformers/main/en/model_doc/mms) for automatic speech recognition. + +### MMS Model + +The [Massive Multilingual Speech (MMS) model](https://huggingface.co/facebook/mms-1b-all) has been pre-trained and fine-tuned +on 1000+ languages. The model makes use of adapter attention layers to fine-tune only a small part +of the model on a specific language. The model already comes with fine-tuned adapter layers for 1000+ languages and +can be used for inference for 1000+ languages out of the box. + +However, for improved performance or more specific use cases one can re-initialize the adapter weights, freeze all +other weights and fine-tune them on a specific dataset as shown in the [example below](#examples-ctc-adapter). + +Note that the adapter weights include low dimensional linear layers for every attention block as well as the final language +model head layers. + +### Examples CTC Adapter + +In the following we will look at how one can fine-tune adapter weights for any of the +[MMS CTC checkpoints](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&other=mms&sort=downloads) in less than 1 hour. + +#### Common Voice CTC Adapter + +As in the examples [above](#examples-ctc), we fine-tune on Common Voice's 6 dataset in Turkish as an example. +Contrary to [`run_speech_recognition_ctc.py`](https://github.com/huggingface/transformers/blob/main/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py) before there is a `--target_language` which has to be defined to state for which +language or concept the adapter layers shall be trained. The adapter weights will then +accordingly be called `adapter.{=1.18.0", "To fix: pip install -r examples/pytorch/speech-recognition/requirements.txt") + + +logger = logging.getLogger(__name__) + + +def list_field(default=None, metadata=None): + return field(default_factory=lambda: default, metadata=metadata) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} + ) + tokenizer_name_or_path: Optional[str] = field( + default=None, + metadata={"help": "Path to pretrained tokenizer or tokenizer identifier from huggingface.co/models"}, + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + final_dropout: float = field( + default=0.0, + metadata={"help": "The dropout probability for the final projection layer."}, + ) + mask_time_prob: float = field( + default=0.05, + metadata={ + "help": ( + "Probability of each feature vector along the time axis to be chosen as the start of the vector" + "span to be masked. Approximately ``mask_time_prob * sequence_length // mask_time_length`` feature" + "vectors will be masked along the time axis." + ) + }, + ) + mask_time_length: int = field( + default=10, + metadata={"help": "Length of vector span to mask along the time axis."}, + ) + mask_feature_prob: float = field( + default=0.0, + metadata={ + "help": ( + "Probability of each feature vector along the feature axis to be chosen as the start of the vectorspan" + " to be masked. Approximately ``mask_feature_prob * sequence_length // mask_feature_length`` feature" + " bins will be masked along the time axis." + ) + }, + ) + mask_feature_length: int = field( + default=10, + metadata={"help": "Length of vector span to mask along the feature axis."}, + ) + layerdrop: float = field(default=0.0, metadata={"help": "The LayerDrop probability."}) + ctc_loss_reduction: Optional[str] = field( + default="mean", metadata={"help": "The way the ctc loss should be reduced. Should be one of 'mean' or 'sum'."} + ) + adapter_attn_dim: int = field( + default=16, + metadata={ + "help": "The hidden dimension of the adapter layers that will be randomly initialized and trained. The higher the dimension, the more capacity is given to the adapter weights. Note that only the adapter weights are fine-tuned." + }, + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + + Using `HfArgumentParser` we can turn this class + into argparse arguments to be able to specify them on + the command line. + """ + + dataset_name: str = field( + metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + target_language: Optional[str] = field( + metadata={ + "help": ( + "The target language on which the adapter attention layers" + " should be trained on in ISO 693-3 code, e.g. `tur` for Turkish" + " Wav2Vec2's MMS ISO codes can be looked up here: https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html" + " If you are not training the adapter layers on a language, simply choose" + " another accronym that fits your data." + ) + }, + ) + dataset_config_name: str = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + train_split_name: str = field( + default="train+validation", + metadata={ + "help": ( + "The name of the training data set split to use (via the datasets library). Defaults to " + "'train+validation'" + ) + }, + ) + eval_split_name: str = field( + default="test", + metadata={ + "help": "The name of the evaluation data set split to use (via the datasets library). Defaults to 'test'" + }, + ) + audio_column_name: str = field( + default="audio", + metadata={"help": "The name of the dataset column containing the audio data. Defaults to 'audio'"}, + ) + text_column_name: str = field( + default="text", + metadata={"help": "The name of the dataset column containing the text data. Defaults to 'text'"}, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of validation examples to this " + "value if set." + ) + }, + ) + chars_to_ignore: Optional[List[str]] = list_field( + default=None, + metadata={"help": "A list of characters to remove from the transcripts."}, + ) + eval_metrics: List[str] = list_field( + default=["wer"], + metadata={"help": "A list of metrics the model should be evaluated on. E.g. `'wer cer'`"}, + ) + max_duration_in_seconds: float = field( + default=20.0, + metadata={ + "help": ( + "Filter audio files that are longer than `max_duration_in_seconds` seconds to" + " 'max_duration_in_seconds`" + ) + }, + ) + min_duration_in_seconds: float = field( + default=0.0, metadata={"help": "Filter audio files that are shorter than `min_duration_in_seconds` seconds"} + ) + preprocessing_only: bool = field( + default=False, + metadata={ + "help": ( + "Whether to only do data preprocessing and skip training. This is especially useful when data" + " preprocessing errors out in distributed training due to timeout. In this case, one should run the" + " preprocessing in a non-distributed setup with `preprocessing_only=True` so that the cached datasets" + " can consequently be loaded in distributed training" + ) + }, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "If :obj:`True`, will use the token generated when running" + ":obj:`huggingface-cli login` as HTTP bearer authorization for remote files." + ) + }, + ) + unk_token: str = field( + default="[UNK]", + metadata={"help": "The unk token for the tokenizer"}, + ) + pad_token: str = field( + default="[PAD]", + metadata={"help": "The padding token for the tokenizer"}, + ) + word_delimiter_token: str = field( + default="|", + metadata={"help": "The word delimiter token for the tokenizer"}, + ) + overwrite_lang_vocab: bool = field( + default=False, + metadata={"help": ("If :obj:`True`, will overwrite existing `target_language` vocabulary of tokenizer.")}, + ) + + +@dataclass +class DataCollatorCTCWithPadding: + """ + Data collator that will dynamically pad the inputs received. + Args: + processor (:class:`~transformers.AutoProcessor`) + The processor used for proccessing the data. + padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding index) + among: + * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the + maximum acceptable input length for the model if that argument is not provided. + * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of + different lengths). + max_length (:obj:`int`, `optional`): + Maximum length of the ``input_values`` of the returned list and optionally padding length (see above). + max_length_labels (:obj:`int`, `optional`): + Maximum length of the ``labels`` returned list and optionally padding length (see above). + pad_to_multiple_of (:obj:`int`, `optional`): + If set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.5 (Volta). + """ + + processor: AutoProcessor + padding: Union[bool, str] = "longest" + pad_to_multiple_of: Optional[int] = None + pad_to_multiple_of_labels: Optional[int] = None + + def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]: + # split inputs and labels since they have to be of different lenghts and need + # different padding methods + input_features = [{"input_values": feature["input_values"]} for feature in features] + label_features = [{"input_ids": feature["labels"]} for feature in features] + + batch = self.processor.pad( + input_features, + padding=self.padding, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors="pt", + ) + + labels_batch = self.processor.pad( + labels=label_features, + padding=self.padding, + pad_to_multiple_of=self.pad_to_multiple_of_labels, + return_tensors="pt", + ) + + # replace padding with -100 to ignore loss correctly + labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100) + + batch["labels"] = labels + if "attention_mask" in batch: + batch["attention_mask"] = batch["attention_mask"].to(torch.long) + + return batch + + +def create_vocabulary_from_data( + datasets: DatasetDict, + word_delimiter_token: Optional[str] = None, + unk_token: Optional[str] = None, + pad_token: Optional[str] = None, +): + # Given training and test labels create vocabulary + def extract_all_chars(batch): + all_text = " ".join(batch["target_text"]) + vocab = list(set(all_text)) + return {"vocab": [vocab], "all_text": [all_text]} + + vocabs = datasets.map( + extract_all_chars, + batched=True, + batch_size=-1, + keep_in_memory=True, + remove_columns=datasets["train"].column_names, + ) + + # take union of all unique characters in each dataset + vocab_set = functools.reduce( + lambda vocab_1, vocab_2: set(vocab_1["vocab"][0]) | set(vocab_2["vocab"][0]), vocabs.values() + ) + + vocab_dict = {v: k for k, v in enumerate(sorted(vocab_set))} + + # replace white space with delimiter token + if word_delimiter_token is not None: + vocab_dict[word_delimiter_token] = vocab_dict[" "] + del vocab_dict[" "] + + # add unk and pad token + if unk_token is not None: + vocab_dict[unk_token] = len(vocab_dict) + + if pad_token is not None: + vocab_dict[pad_token] = len(vocab_dict) + + return vocab_dict + + +def main(): + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_speech_recognition_ctc_adapter", model_args, data_args) + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN) + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + # Set the verbosity to info of the Transformers logger (on main process only): + if is_main_process(training_args.local_rank): + transformers.utils.logging.set_verbosity_info() + logger.info("Training/evaluation parameters %s", training_args) + + # Set seed before initializing model. + set_seed(training_args.seed) + + # 1. First, let's load the dataset + raw_datasets = DatasetDict() + + if training_args.do_train: + raw_datasets["train"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=data_args.train_split_name, + use_auth_token=data_args.use_auth_token, + ) + + if data_args.audio_column_name not in raw_datasets["train"].column_names: + raise ValueError( + f"--audio_column_name '{data_args.audio_column_name}' not found in dataset '{data_args.dataset_name}'." + " Make sure to set `--audio_column_name` to the correct audio column - one of" + f" {', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.text_column_name not in raw_datasets["train"].column_names: + raise ValueError( + f"--text_column_name {data_args.text_column_name} not found in dataset '{data_args.dataset_name}'. " + "Make sure to set `--text_column_name` to the correct text column - one of " + f"{', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.max_train_samples is not None: + raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples)) + + if training_args.do_eval: + raw_datasets["eval"] = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + split=data_args.eval_split_name, + use_auth_token=data_args.use_auth_token, + ) + + if data_args.max_eval_samples is not None: + raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples)) + + # 2. We remove some special characters from the datasets + # that make training complicated and do not help in transcribing the speech + # E.g. characters, such as `,` and `.` do not really have an acoustic characteristic + # that could be easily picked up by the model + chars_to_ignore_regex = ( + f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None + ) + text_column_name = data_args.text_column_name + + def remove_special_characters(batch): + if chars_to_ignore_regex is not None: + batch["target_text"] = re.sub(chars_to_ignore_regex, "", batch[text_column_name]).lower() + " " + else: + batch["target_text"] = batch[text_column_name].lower() + " " + return batch + + with training_args.main_process_first(desc="dataset map special characters removal"): + raw_datasets = raw_datasets.map( + remove_special_characters, + remove_columns=[text_column_name], + desc="remove special characters from datasets", + ) + + # save special tokens for tokenizer + word_delimiter_token = data_args.word_delimiter_token + unk_token = data_args.unk_token + pad_token = data_args.pad_token + + # 3. Next, let's load the config as we might need it to create + # the tokenizer + # load config + config = AutoConfig.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + ) + + # 4. Next, if no tokenizer file is defined, + # we create the vocabulary of the model by extracting all unique characters from + # the training and evaluation datasets + # We need to make sure that only first rank saves vocabulary + # make sure all processes wait until vocab is created + tokenizer_name_or_path = model_args.tokenizer_name_or_path + tokenizer_kwargs = {} + + vocab_dict = {} + if tokenizer_name_or_path is not None: + # load vocabulary of other adapter languages so that new language can be appended + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_auth_token=data_args.use_auth_token) + vocab_dict = tokenizer.vocab.copy() + if tokenizer.target_lang is None: + raise ValueError("Make sure to load a multi-lingual tokenizer with a set target language.") + + if data_args.target_language in tokenizer.vocab and not data_args.overwrite_lang_vocab: + logger.info( + "Adapter language already exists." + " Skipping vocabulary creating. If you want to create a new vocabulary" + f" for {data_args.target_language} make sure to add '--overwrite_lang_vocab'" + ) + else: + tokenizer_name_or_path = None + + if tokenizer_name_or_path is None: + # save vocab in training output dir + tokenizer_name_or_path = training_args.output_dir + + vocab_file = os.path.join(tokenizer_name_or_path, "vocab.json") + + with training_args.main_process_first(): + if training_args.overwrite_output_dir and os.path.isfile(vocab_file): + try: + os.remove(vocab_file) + except OSError: + # in shared file-systems it might be the case that + # two processes try to delete the vocab file at the some time + pass + + with training_args.main_process_first(desc="dataset map vocabulary creation"): + if not os.path.isfile(vocab_file): + os.makedirs(tokenizer_name_or_path, exist_ok=True) + lang_dict = create_vocabulary_from_data( + raw_datasets, + word_delimiter_token=word_delimiter_token, + unk_token=unk_token, + pad_token=pad_token, + ) + + # if we doing adapter language training, save + # vocab with adpter language + if data_args.target_language is not None: + vocab_dict[data_args.target_language] = lang_dict + + # save vocab dict to be loaded into tokenizer + with open(vocab_file, "w") as file: + json.dump(vocab_dict, file) + + # if tokenizer has just been created + # it is defined by `tokenizer_class` if present in config else by `model_type` + tokenizer_kwargs = { + "config": config if config.tokenizer_class is not None else None, + "tokenizer_type": config.model_type if config.tokenizer_class is None else None, + "unk_token": unk_token, + "pad_token": pad_token, + "word_delimiter_token": word_delimiter_token, + "target_lang": data_args.target_language, + } + + # 5. Now we can instantiate the feature extractor, tokenizer and model + # Note for distributed training, the .from_pretrained methods guarantee that only + # one local process can concurrently download model & vocab. + + # load feature_extractor and tokenizer + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name_or_path, + use_auth_token=data_args.use_auth_token, + **tokenizer_kwargs, + ) + feature_extractor = AutoFeatureExtractor.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + ) + + # adapt config + config.update( + { + "final_dropout": model_args.final_dropout, + "mask_time_prob": model_args.mask_time_prob, + "mask_time_length": model_args.mask_time_length, + "mask_feature_prob": model_args.mask_feature_prob, + "mask_feature_length": model_args.mask_feature_length, + "gradient_checkpointing": training_args.gradient_checkpointing, + "layerdrop": model_args.layerdrop, + "ctc_loss_reduction": model_args.ctc_loss_reduction, + "pad_token_id": tokenizer.pad_token_id, + "vocab_size": len(tokenizer), + "adapter_attn_dim": model_args.adapter_attn_dim, + } + ) + + # create model + model = AutoModelForCTC.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + config=config, + use_auth_token=data_args.use_auth_token, + ignore_mismatched_sizes=True, + ) + + # if attn adapter is defined, freeze all non-adapter weights + if model.config.adapter_attn_dim is not None: + model.init_adapter_layers() + # first we freeze the whole base model + model.freeze_base_model() + + # next we unfreeze all adapter layers + adapter_weights = model._get_adapters() + for param in adapter_weights.values(): + param.requires_grad = True + + # 6. Now we preprocess the datasets including loading the audio, resampling and normalization + # Thankfully, `datasets` takes care of automatically loading and resampling the audio, + # so that we just need to set the correct target sampling rate and normalize the input + # via the `feature_extractor` + + # make sure that dataset decodes audio with correct sampling rate + dataset_sampling_rate = next(iter(raw_datasets.values())).features[data_args.audio_column_name].sampling_rate + if dataset_sampling_rate != feature_extractor.sampling_rate: + raw_datasets = raw_datasets.cast_column( + data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate) + ) + + # derive max & min input length for sample rate & max duration + max_input_length = data_args.max_duration_in_seconds * feature_extractor.sampling_rate + min_input_length = data_args.min_duration_in_seconds * feature_extractor.sampling_rate + audio_column_name = data_args.audio_column_name + num_workers = data_args.preprocessing_num_workers + + # Preprocessing the datasets. + # We need to read the audio files as arrays and tokenize the targets. + def prepare_dataset(batch): + # load audio + sample = batch[audio_column_name] + + inputs = feature_extractor(sample["array"], sampling_rate=sample["sampling_rate"]) + batch["input_values"] = inputs.input_values[0] + batch["input_length"] = len(batch["input_values"]) + + # encode targets + batch["labels"] = tokenizer(batch["target_text"]).input_ids + return batch + + with training_args.main_process_first(desc="dataset map preprocessing"): + vectorized_datasets = raw_datasets.map( + prepare_dataset, + remove_columns=next(iter(raw_datasets.values())).column_names, + num_proc=num_workers, + desc="preprocess datasets", + ) + + def is_audio_in_length_range(length): + return length > min_input_length and length < max_input_length + + # filter data that is shorter than min_input_length + vectorized_datasets = vectorized_datasets.filter( + is_audio_in_length_range, + num_proc=num_workers, + input_columns=["input_length"], + ) + + # 7. Next, we can prepare the training. + # Let's use word error rate (WER) as our evaluation metric, + # instantiate a data collator and the trainer + + # Define evaluation metrics during training, *i.e.* word error rate, character error rate + eval_metrics = {metric: evaluate.load(metric) for metric in data_args.eval_metrics} + + # for large datasets it is advised to run the preprocessing on a + # single machine first with ``args.preprocessing_only`` since there will mostly likely + # be a timeout when running the script in distributed mode. + # In a second step ``args.preprocessing_only`` can then be set to `False` to load the + # cached dataset + if data_args.preprocessing_only: + logger.info(f"Data preprocessing finished. Files cached at {vectorized_datasets.cache_files}") + return + + def compute_metrics(pred): + pred_logits = pred.predictions + pred_ids = np.argmax(pred_logits, axis=-1) + + pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id + + pred_str = tokenizer.batch_decode(pred_ids) + # we do not want to group tokens when computing the metrics + label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False) + + metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()} + + return metrics + + # Now save everything to be able to create a single processor later + # make sure all processes wait until data is saved + with training_args.main_process_first(): + # only the main process saves them + if is_main_process(training_args.local_rank): + # save feature extractor, tokenizer and config + feature_extractor.save_pretrained(training_args.output_dir) + tokenizer.save_pretrained(training_args.output_dir) + config.save_pretrained(training_args.output_dir) + + try: + processor = AutoProcessor.from_pretrained(training_args.output_dir) + except (OSError, KeyError): + warnings.warn( + "Loading a processor from a feature extractor config that does not" + " include a `processor_class` attribute is deprecated and will be removed in v5. Please add the following " + " attribute to your `preprocessor_config.json` file to suppress this warning: " + " `'processor_class': 'Wav2Vec2Processor'`", + FutureWarning, + ) + processor = Wav2Vec2Processor.from_pretrained(training_args.output_dir) + + # Instantiate custom data collator + data_collator = DataCollatorCTCWithPadding(processor=processor) + + # Initialize Trainer + trainer = Trainer( + model=model, + data_collator=data_collator, + args=training_args, + compute_metrics=compute_metrics, + train_dataset=vectorized_datasets["train"] if training_args.do_train else None, + eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None, + tokenizer=processor, + ) + + # 8. Finally, we can start training + + # Training + if training_args.do_train: + # use last checkpoint if exist + if last_checkpoint is not None: + checkpoint = last_checkpoint + elif os.path.isdir(model_args.model_name_or_path): + checkpoint = model_args.model_name_or_path + else: + checkpoint = None + + train_result = trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() + + metrics = train_result.metrics + max_train_samples = ( + data_args.max_train_samples + if data_args.max_train_samples is not None + else len(vectorized_datasets["train"]) + ) + metrics["train_samples"] = min(max_train_samples, len(vectorized_datasets["train"])) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + results = {} + if training_args.do_eval: + logger.info("*** Evaluate ***") + metrics = trainer.evaluate() + max_eval_samples = ( + data_args.max_eval_samples if data_args.max_eval_samples is not None else len(vectorized_datasets["eval"]) + ) + metrics["eval_samples"] = min(max_eval_samples, len(vectorized_datasets["eval"])) + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + # Write model card and (optionally) push to hub + config_name = data_args.dataset_config_name if data_args.dataset_config_name is not None else "na" + kwargs = { + "finetuned_from": model_args.model_name_or_path, + "tasks": "automatic-speech-recognition", + "tags": ["automatic-speech-recognition", data_args.dataset_name, "mms"], + "dataset_args": ( + f"Config: {config_name}, Training split: {data_args.train_split_name}, Eval split:" + f" {data_args.eval_split_name}" + ), + "dataset": f"{data_args.dataset_name.upper()} - {config_name.upper()}", + } + if "common_voice" in data_args.dataset_name: + kwargs["language"] = config_name + + # make sure that adapter weights are saved seperately + adapter_file = WAV2VEC2_ADAPTER_SAFE_FILE.format(data_args.target_language) + adapter_file = os.path.join(training_args.output_dir, adapter_file) + logger.info(f"Saving adapter weights under {adapter_file}...") + safe_save_file(model._get_adapters(), adapter_file, metadata={"format": "pt"}) + + if training_args.push_to_hub: + trainer.push_to_hub(**kwargs) + else: + trainer.create_model_card(**kwargs) + + return results + + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/test_pytorch_examples.py b/examples/pytorch/test_pytorch_examples.py index f4682b8933e7e5..a94e23e614d5dc 100644 --- a/examples/pytorch/test_pytorch_examples.py +++ b/examples/pytorch/test_pytorch_examples.py @@ -63,6 +63,7 @@ import run_semantic_segmentation import run_seq2seq_qa as run_squad_seq2seq import run_speech_recognition_ctc + import run_speech_recognition_ctc_adapter import run_speech_recognition_seq2seq import run_summarization import run_swag @@ -446,6 +447,38 @@ def test_run_speech_recognition_ctc(self): result = get_results(tmp_dir) self.assertLess(result["eval_loss"], result["train_loss"]) + def test_run_speech_recognition_ctc_adapter(self): + tmp_dir = self.get_auto_remove_tmp_dir() + testargs = f""" + run_speech_recognition_ctc_adapter.py + --output_dir {tmp_dir} + --model_name_or_path hf-internal-testing/tiny-random-wav2vec2 + --dataset_name hf-internal-testing/librispeech_asr_dummy + --dataset_config_name clean + --train_split_name validation + --eval_split_name validation + --do_train + --do_eval + --learning_rate 1e-4 + --per_device_train_batch_size 2 + --per_device_eval_batch_size 1 + --remove_unused_columns False + --overwrite_output_dir True + --preprocessing_num_workers 16 + --max_steps 10 + --target_language tur + --seed 42 + """.split() + + if is_cuda_and_apex_available(): + testargs.append("--fp16") + + with patch.object(sys, "argv", testargs): + run_speech_recognition_ctc_adapter.main() + result = get_results(tmp_dir) + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "./adapter.tur.safetensors"))) + self.assertLess(result["eval_loss"], result["train_loss"]) + def test_run_speech_recognition_seq2seq(self): tmp_dir = self.get_auto_remove_tmp_dir() testargs = f""" diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py index cd8b5f165d9495..c7436c5ea4adbc 100755 --- a/src/transformers/models/hubert/modeling_hubert.py +++ b/src/transformers/models/hubert/modeling_hubert.py @@ -1181,6 +1181,14 @@ def freeze_feature_encoder(self): """ self.hubert.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.hubert.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(HUBERT_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py index 1d45facf22e822..928b2fd3ade744 100644 --- a/src/transformers/models/sew/modeling_sew.py +++ b/src/transformers/models/sew/modeling_sew.py @@ -1016,6 +1016,14 @@ def freeze_feature_encoder(self): """ self.sew.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.sew.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(SEW_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py index b39ac0466f3384..403c865cdd4ce3 100644 --- a/src/transformers/models/sew_d/modeling_sew_d.py +++ b/src/transformers/models/sew_d/modeling_sew_d.py @@ -1556,6 +1556,14 @@ def freeze_feature_encoder(self): """ self.sew_d.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.sew_d.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(SEWD_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index c156886fe25c7b..da81bc4b140463 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -1425,6 +1425,14 @@ def freeze_feature_encoder(self): """ self.unispeech.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.unispeech.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(UNISPEECH_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py index 5fff3265085db7..58ba244ade0f92 100755 --- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py @@ -1432,6 +1432,14 @@ def freeze_feature_encoder(self): """ self.unispeech_sat.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.unispeech_sat.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(UNISPEECH_SAT_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py index 80d05d3777f005..26cfe3c6cb4916 100755 --- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py @@ -1194,6 +1194,19 @@ def _get_adapters(self): return adapter_weights + def init_adapter_layers(self): + """ + (Re-)initialize attention adapter layers and lm head for adapter-only fine-tuning + """ + # init attention adapters + for module in self.modules(): + if isinstance(module, Wav2Vec2AttnAdapterLayer): + self._init_weights(module) + + # init lm head + if isinstance(self, Wav2Vec2ForCTC): + self._init_weights(self.lm_head) + def load_adapter(self, target_lang: str, **kwargs): r""" Load a language adapter model from a pre-trained adapter model. @@ -1888,6 +1901,14 @@ def freeze_feature_encoder(self): """ self.wav2vec2.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.wav2vec2.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(WAV_2_VEC_2_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py index 59662053ba75d3..4181794ce1692c 100755 --- a/src/transformers/models/wavlm/modeling_wavlm.py +++ b/src/transformers/models/wavlm/modeling_wavlm.py @@ -1319,6 +1319,14 @@ def freeze_feature_encoder(self): """ self.wavlm.feature_extractor._freeze_parameters() + def freeze_base_model(self): + """ + Calling this function will disable the gradient computation for the base model so that its parameters will not + be updated during training. Only the classification head will be updated. + """ + for param in self.wavlm.parameters(): + param.requires_grad = False + @add_start_docstrings_to_model_forward(WAVLM_INPUTS_DOCSTRING) @add_code_sample_docstrings( checkpoint=_CHECKPOINT_FOR_DOC, From 6793f0cfe0006d7cedfb9b6081f55d9d38eae18a Mon Sep 17 00:00:00 2001 From: Stephan Tulkens Date: Thu, 15 Jun 2023 10:41:57 +0200 Subject: [PATCH 05/15] Fix bug in slow tokenizer conversion, make it a lot faster (#24266) * Make conversion faster, fix None vs 0 bug * Add second sort for consistency * Update src/transformers/convert_slow_tokenizer.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --------- Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --- src/transformers/convert_slow_tokenizer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 8ff40c573e3c2b..082ccb47bdfafc 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -54,12 +54,15 @@ def extract(self, vocab_scores=None) -> Tuple[Dict[str, int], List[Tuple]]: # Merges merges = [] - for piece_l in vocab.keys(): - for piece_r in vocab.keys(): - merge = f"{piece_l}{piece_r}" - piece_score = vocab_scores.get(merge, None) - if piece_score: - merges += [(piece_l, piece_r, piece_score)] + for merge, piece_score in vocab_scores.items(): + local = [] + for index in range(1, len(merge)): + piece_l, piece_r = merge[:index], merge[index:] + if piece_l in vocab and piece_r in vocab: + local.append((piece_l, piece_r, piece_score)) + local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]])) + merges.extend(local) + merges = sorted(merges, key=lambda val: val[2], reverse=reverse) merges = [(val[0], val[1]) for val in merges] return vocab, merges From 7504be35ab88639ec3def32f0392f0a43c9551ee Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 15 Jun 2023 11:39:20 +0200 Subject: [PATCH 06/15] Fix `check_config_attributes`: check all configuration classes (#24231) * fix --------- Co-authored-by: ydshieh --- .../models/align/configuration_align.py | 4 --- .../models/blip/configuration_blip.py | 2 -- .../models/blip_2/configuration_blip_2.py | 17 ----------- .../bridgetower/configuration_bridgetower.py | 8 ----- .../models/clap/configuration_clap.py | 8 ----- .../pix2struct/configuration_pix2struct.py | 16 ---------- .../models/sam/configuration_sam.py | 16 ---------- utils/check_config_attributes.py | 30 ++++++++++++++++--- 8 files changed, 26 insertions(+), 75 deletions(-) diff --git a/src/transformers/models/align/configuration_align.py b/src/transformers/models/align/configuration_align.py index 0436b278f0a737..cb7baddfe19eb1 100644 --- a/src/transformers/models/align/configuration_align.py +++ b/src/transformers/models/align/configuration_align.py @@ -206,8 +206,6 @@ class AlignVisionConfig(PretrainedConfig): The epsilon used by the batch normalization layers. batch_norm_momentum (`float`, *optional*, defaults to 0.99): The momentum used by the batch normalization layers. - dropout_rate (`float`, *optional*, defaults to 0.5): - The dropout rate to be applied before final classifier layer. drop_connect_rate (`float`, *optional*, defaults to 0.2): The drop rate for skip connections. @@ -249,7 +247,6 @@ def __init__( initializer_range: float = 0.02, batch_norm_eps: float = 0.001, batch_norm_momentum: float = 0.99, - dropout_rate: float = 0.5, drop_connect_rate: float = 0.2, **kwargs, ): @@ -274,7 +271,6 @@ def __init__( self.initializer_range = initializer_range self.batch_norm_eps = batch_norm_eps self.batch_norm_momentum = batch_norm_momentum - self.dropout_rate = dropout_rate self.drop_connect_rate = drop_connect_rate self.num_hidden_layers = sum(num_block_repeats) * 4 diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index f03f167c2941aa..53a77852828569 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -234,7 +234,6 @@ def __init__( projection_dim=512, num_hidden_layers=12, num_attention_heads=12, - num_channels=3, image_size=384, patch_size=16, hidden_act="gelu", @@ -250,7 +249,6 @@ def __init__( self.projection_dim = projection_dim self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads - self.num_channels = num_channels self.patch_size = patch_size self.image_size = image_size self.initializer_range = initializer_range diff --git a/src/transformers/models/blip_2/configuration_blip_2.py b/src/transformers/models/blip_2/configuration_blip_2.py index 8a80510db9afcc..6adf85e6115f8f 100644 --- a/src/transformers/models/blip_2/configuration_blip_2.py +++ b/src/transformers/models/blip_2/configuration_blip_2.py @@ -58,15 +58,10 @@ class Blip2VisionConfig(PretrainedConfig): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon used by the layer normalization layers. - dropout (`float`, *optional*, defaults to 0.0): - The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - initializer_factor (`float``, *optional*, defaults to 1): - A factor for initializing all weight matrices (should be kept to 1, used internally for initialization - testing). qkv_bias (`bool`, *optional*, defaults to `True`): Whether to add a bias to the queries and values in the self-attention layers. @@ -91,18 +86,14 @@ def __init__( self, hidden_size=1408, intermediate_size=6144, - projection_dim=512, num_hidden_layers=39, num_attention_heads=16, - num_channels=3, image_size=224, patch_size=14, hidden_act="gelu", layer_norm_eps=0.00001, - dropout=0.0, attention_dropout=0.0, initializer_range=1e-10, - initializer_factor=1.0, qkv_bias=True, **kwargs, ): @@ -110,15 +101,11 @@ def __init__( self.hidden_size = hidden_size self.intermediate_size = intermediate_size - self.projection_dim = projection_dim - self.dropout = dropout self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads - self.num_channels = num_channels self.patch_size = patch_size self.image_size = image_size self.initializer_range = initializer_range - self.initializer_factor = initializer_factor self.attention_dropout = attention_dropout self.layer_norm_eps = layer_norm_eps self.hidden_act = hidden_act @@ -184,8 +171,6 @@ class Blip2QFormerConfig(PretrainedConfig): [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658). - classifier_dropout (`float`, *optional*): - The dropout ratio for the classification head. cross_attention_frequency (`int`, *optional*, defaults to 2): The frequency of adding cross-attention to the Transformer layers. encoder_hidden_size (`int`, *optional*, defaults to 1408): @@ -221,7 +206,6 @@ def __init__( layer_norm_eps=1e-12, pad_token_id=0, position_embedding_type="absolute", - classifier_dropout=None, cross_attention_frequency=2, encoder_hidden_size=1408, **kwargs, @@ -240,7 +224,6 @@ def __init__( self.initializer_range = initializer_range self.layer_norm_eps = layer_norm_eps self.position_embedding_type = position_embedding_type - self.classifier_dropout = classifier_dropout self.cross_attention_frequency = cross_attention_frequency self.encoder_hidden_size = encoder_hidden_size diff --git a/src/transformers/models/bridgetower/configuration_bridgetower.py b/src/transformers/models/bridgetower/configuration_bridgetower.py index 3149b34efaac4c..17c9cadaf838d1 100644 --- a/src/transformers/models/bridgetower/configuration_bridgetower.py +++ b/src/transformers/models/bridgetower/configuration_bridgetower.py @@ -155,8 +155,6 @@ class BridgeTowerTextConfig(PretrainedConfig): initializer_factor (`float``, *optional*, defaults to 1): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. layer_norm_eps (`float`, *optional*, defaults to 1e-05): The epsilon used by the layer normalization layers. position_embedding_type (`str`, *optional*, defaults to `"absolute"`): @@ -170,8 +168,6 @@ class BridgeTowerTextConfig(PretrainedConfig): use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions (not used by all models). Only relevant if `config.is_decoder=True`. - classifier_dropout (`float`, *optional*): - The dropout ratio for the classification head. Example: @@ -199,14 +195,12 @@ def __init__( attention_probs_dropout_prob=0.1, max_position_embeddings=514, type_vocab_size=1, - initializer_range=0.02, layer_norm_eps=1e-05, pad_token_id=1, bos_token_id=0, eos_token_id=2, position_embedding_type="absolute", use_cache=True, - classifier_dropout=None, **kwargs, ): super().__init__(**kwargs) @@ -222,11 +216,9 @@ def __init__( self.attention_probs_dropout_prob = attention_probs_dropout_prob self.max_position_embeddings = max_position_embeddings self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range self.layer_norm_eps = layer_norm_eps self.position_embedding_type = position_embedding_type self.use_cache = use_cache - self.classifier_dropout = classifier_dropout self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id self.eos_token_id = eos_token_id diff --git a/src/transformers/models/clap/configuration_clap.py b/src/transformers/models/clap/configuration_clap.py index 13d1f7b7e05973..9886c614da7b10 100644 --- a/src/transformers/models/clap/configuration_clap.py +++ b/src/transformers/models/clap/configuration_clap.py @@ -65,8 +65,6 @@ class ClapTextConfig(PretrainedConfig): just in case (e.g., 512 or 1024 or 2048). type_vocab_size (`int`, *optional*, defaults to 2): The vocabulary size of the `token_type_ids` passed when calling [`ClapTextModel`]. - initializer_range (`float`, *optional*, defaults to 0.02): - The standard deviation of the truncated_normal_initializer for initializing all weight matrices. layer_norm_eps (`float`, *optional*, defaults to 1e-12): The epsilon used by the layer normalization layers. position_embedding_type (`str`, *optional*, defaults to `"absolute"`): @@ -80,8 +78,6 @@ class ClapTextConfig(PretrainedConfig): use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions (not used by all models). Only relevant if `config.is_decoder=True`. - classifier_dropout (`float`, *optional*): - The dropout ratio for the classification head. projection_hidden_act (`str`, *optional*, defaults to `"relu"`): The non-linear activation function (function or string) in the projection layer. If string, `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported. @@ -116,7 +112,6 @@ def __init__( attention_probs_dropout_prob=0.1, max_position_embeddings=514, type_vocab_size=1, - initializer_range=0.02, initializer_factor=1.0, layer_norm_eps=1e-12, projection_dim=512, @@ -125,7 +120,6 @@ def __init__( eos_token_id=2, position_embedding_type="absolute", use_cache=True, - classifier_dropout=None, projection_hidden_act="relu", **kwargs, ): @@ -141,12 +135,10 @@ def __init__( self.attention_probs_dropout_prob = attention_probs_dropout_prob self.max_position_embeddings = max_position_embeddings self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range self.initializer_factor = initializer_factor self.layer_norm_eps = layer_norm_eps self.position_embedding_type = position_embedding_type self.use_cache = use_cache - self.classifier_dropout = classifier_dropout self.projection_hidden_act = projection_hidden_act self.projection_dim = projection_dim diff --git a/src/transformers/models/pix2struct/configuration_pix2struct.py b/src/transformers/models/pix2struct/configuration_pix2struct.py index 32aa34941f8fcb..cf1dd91b463584 100644 --- a/src/transformers/models/pix2struct/configuration_pix2struct.py +++ b/src/transformers/models/pix2struct/configuration_pix2struct.py @@ -187,16 +187,10 @@ class Pix2StructVisionConfig(PretrainedConfig): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. d_kv (`int`, *optional*, defaults to 64): Dimensionality of the key, query, value projections per attention head. - projection_dim (`int`, *optional*, defaults to 768): - Dimensionality of the projection layer in the Transformer encoder. num_hidden_layers (`int`, *optional*, defaults to 12): Number of hidden layers in the Transformer encoder. num_attention_heads (`int`, *optional*, defaults to 12): Number of attention heads for each attention layer in the Transformer encoder. - num_channels (`int`, *optional*, defaults to 3): - Number of channels of the input images. - patch_size (`int`, *optional*, defaults to 16): - The size (resolution) of each patch. dense_act_fn (`str` or `function`, *optional*, defaults to `"gelu_new"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. @@ -213,8 +207,6 @@ class Pix2StructVisionConfig(PretrainedConfig): testing). seq_len (`int`, *optional*, defaults to 4096): Maximum sequence length (here number of patches) supported by the model. - layer_norm_bias (`bool`, *optional*, defaults to `False`): - Whether or not to add a bias to the layer normalization layers. relative_attention_num_buckets (`int`, *optional*, defaults to 32): The number of buckets to use for each attention layer. relative_attention_max_distance (`int`, *optional*, defaults to 128): @@ -243,11 +235,8 @@ def __init__( patch_embed_hidden_size=768, d_ff=2048, d_kv=64, - projection_dim=768, num_hidden_layers=12, num_attention_heads=12, - num_channels=3, - patch_size=16, dense_act_fn="gelu_new", layer_norm_eps=1e-6, dropout_rate=0.0, @@ -255,7 +244,6 @@ def __init__( initializer_range=1e-10, initializer_factor=1.0, seq_len=4096, - layer_norm_bias=False, relative_attention_num_buckets=32, relative_attention_max_distance=128, **kwargs, @@ -265,19 +253,15 @@ def __init__( self.hidden_size = hidden_size self.patch_embed_hidden_size = patch_embed_hidden_size self.d_ff = d_ff - self.projection_dim = projection_dim self.dropout_rate = dropout_rate self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads - self.num_channels = num_channels - self.patch_size = patch_size self.initializer_range = initializer_range self.initializer_factor = initializer_factor self.attention_dropout = attention_dropout self.layer_norm_eps = layer_norm_eps self.dense_act_fn = dense_act_fn self.seq_len = seq_len - self.layer_norm_bias = layer_norm_bias self.relative_attention_num_buckets = relative_attention_num_buckets self.relative_attention_max_distance = relative_attention_max_distance self.d_kv = d_kv diff --git a/src/transformers/models/sam/configuration_sam.py b/src/transformers/models/sam/configuration_sam.py index fc3701fca734cb..87427d457310d2 100644 --- a/src/transformers/models/sam/configuration_sam.py +++ b/src/transformers/models/sam/configuration_sam.py @@ -150,10 +150,6 @@ class SamVisionConfig(PretrainedConfig): Args: hidden_size (`int`, *optional*, defaults to 768): Dimensionality of the encoder layers and the pooler layer. - intermediate_size (`int`, *optional*, defaults to 6144): - Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. - projection_dim (`int`, *optional*, defaults to 512): - Dimensionality of the projection layer in the Transformer encoder. output_channels (`int`, *optional*, defaults to 256): Dimensionality of the output channels in the Patch Encoder. num_hidden_layers (`int`, *optional*, defaults to 12): @@ -170,14 +166,10 @@ class SamVisionConfig(PretrainedConfig): The non-linear activation function (function or string) layer_norm_eps (`float`, *optional*, defaults to 1e-6): The epsilon used by the layer normalization layers. - dropout (`float`, *optional*, defaults to 0.0): - The dropout probability. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. initializer_range (`float`, *optional*, defaults to 1e-10): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - initializer_factor (`float`, *optional*, defaults to 1.0): - A factor for multiplying the initializer range. qkv_bias (`bool`, *optional*, defaults to `True`): Whether to add a bias to query, key, value projections. mlp_ratio (`float`, *optional*, defaults to 4.0): @@ -200,8 +192,6 @@ class SamVisionConfig(PretrainedConfig): def __init__( self, hidden_size=768, - intermediate_size=6144, - projection_dim=512, output_channels=256, num_hidden_layers=12, num_attention_heads=12, @@ -210,10 +200,8 @@ def __init__( patch_size=16, hidden_act="gelu", layer_norm_eps=1e-06, - dropout=0.0, attention_dropout=0.0, initializer_range=1e-10, - initializer_factor=1.0, qkv_bias=True, mlp_ratio=4.0, use_abs_pos=True, @@ -227,8 +215,6 @@ def __init__( super().__init__(**kwargs) self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.projection_dim = projection_dim self.output_channels = output_channels self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads @@ -237,10 +223,8 @@ def __init__( self.patch_size = patch_size self.hidden_act = hidden_act self.layer_norm_eps = layer_norm_eps - self.dropout = dropout self.attention_dropout = attention_dropout self.initializer_range = initializer_range - self.initializer_factor = initializer_factor self.qkv_bias = qkv_bias self.mlp_ratio = mlp_ratio self.use_abs_pos = use_abs_pos diff --git a/utils/check_config_attributes.py b/utils/check_config_attributes.py index 63b1bacbbe7451..dad6888c802199 100644 --- a/utils/check_config_attributes.py +++ b/utils/check_config_attributes.py @@ -17,6 +17,7 @@ import os import re +from transformers.configuration_utils import PretrainedConfig from transformers.utils import direct_transformers_import @@ -77,6 +78,12 @@ "TimeSeriesTransformerConfig": ["num_static_real_features", "num_time_features"], # used internally to calculate the feature size "AutoformerConfig": ["num_static_real_features", "num_time_features"], + # used internally to calculate `mlp_dim` + "SamVisionConfig": ["mlp_ratio"], + # For (head) training, but so far not implemented + "ClapAudioConfig": ["num_classes"], + # Not used, but providing useful information to users + "SpeechT5HifiGanConfig": ["sampling_rate"], } @@ -113,6 +120,10 @@ "VanConfig": True, "WavLMConfig": True, "WhisperConfig": True, + # TODO: @Arthur (for `alignment_head` and `alignment_layer`) + "JukeboxPriorConfig": True, + # TODO: @Younes (for `is_decoder`) + "Pix2StructTextConfig": True, } ) @@ -254,10 +265,21 @@ def check_config_attributes_being_used(config_class): def check_config_attributes(): """Check the arguments in `__init__` of all configuration classes are used in python files""" configs_with_unused_attributes = {} - for config_class in list(CONFIG_MAPPING.values()): - unused_attributes = check_config_attributes_being_used(config_class) - if len(unused_attributes) > 0: - configs_with_unused_attributes[config_class.__name__] = unused_attributes + for _config_class in list(CONFIG_MAPPING.values()): + # Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.) + config_classes_in_module = [ + cls + for name, cls in inspect.getmembers( + inspect.getmodule(_config_class), + lambda x: inspect.isclass(x) + and issubclass(x, PretrainedConfig) + and inspect.getmodule(x) == inspect.getmodule(_config_class), + ) + ] + for config_class in config_classes_in_module: + unused_attributes = check_config_attributes_being_used(config_class) + if len(unused_attributes) > 0: + configs_with_unused_attributes[config_class.__name__] = unused_attributes if len(configs_with_unused_attributes) > 0: error = "The following configuration classes contain unused attributes in the corresponding modeling files:\n" From 33196b459caa700272531537437f276005b3f82a Mon Sep 17 00:00:00 2001 From: Fei Wang <19998174+FeiWang96@users.noreply.github.com> Date: Thu, 15 Jun 2023 03:28:48 -0700 Subject: [PATCH 07/15] Fix LLaMa beam search when using parallelize (#24224) * Fix LLaMa beam search when using parallelize same issue as T5 #11717 * fix code format in modeling_llama.py * fix format of _reorder_cache in modeling_llama.py --- src/transformers/models/llama/modeling_llama.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index e0580880114b3d..c9debdd252dc7a 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -762,7 +762,9 @@ def prepare_inputs_for_generation( def _reorder_cache(past_key_values, beam_idx): reordered_past = () for layer_past in past_key_values: - reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) return reordered_past From a611ac9b3f9493a80e2d0adf491f4868c71f71c5 Mon Sep 17 00:00:00 2001 From: JayL0321 <31190549+JayL0321@users.noreply.github.com> Date: Thu, 15 Jun 2023 03:39:32 -0700 Subject: [PATCH 08/15] remove unused is_decoder parameter in DetrAttention (#24226) * issue#24161 remove unused is_decoder parameter in DetrAttention * #24161 fix check_repository_consistency fail --- .../models/conditional_detr/modeling_conditional_detr.py | 1 - src/transformers/models/detr/modeling_detr.py | 3 --- src/transformers/models/maskformer/modeling_maskformer.py | 3 --- .../models/table_transformer/modeling_table_transformer.py | 3 --- 4 files changed, 10 deletions(-) diff --git a/src/transformers/models/conditional_detr/modeling_conditional_detr.py b/src/transformers/models/conditional_detr/modeling_conditional_detr.py index 6110fb4491ce77..023cb278484193 100644 --- a/src/transformers/models/conditional_detr/modeling_conditional_detr.py +++ b/src/transformers/models/conditional_detr/modeling_conditional_detr.py @@ -534,7 +534,6 @@ def __init__( embed_dim: int, num_heads: int, dropout: float = 0.0, - is_decoder: bool = False, bias: bool = True, ): super().__init__() diff --git a/src/transformers/models/detr/modeling_detr.py b/src/transformers/models/detr/modeling_detr.py index bd4eae128394e2..c92c43e46d18e9 100644 --- a/src/transformers/models/detr/modeling_detr.py +++ b/src/transformers/models/detr/modeling_detr.py @@ -499,7 +499,6 @@ def __init__( embed_dim: int, num_heads: int, dropout: float = 0.0, - is_decoder: bool = False, bias: bool = True, ): super().__init__() @@ -697,7 +696,6 @@ def __init__(self, config: DetrConfig): embed_dim=self.embed_dim, num_heads=config.decoder_attention_heads, dropout=config.attention_dropout, - is_decoder=True, ) self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] @@ -708,7 +706,6 @@ def __init__(self, config: DetrConfig): self.embed_dim, config.decoder_attention_heads, dropout=config.attention_dropout, - is_decoder=True, ) self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) diff --git a/src/transformers/models/maskformer/modeling_maskformer.py b/src/transformers/models/maskformer/modeling_maskformer.py index 4a73910f873fa6..830f8b23c81602 100644 --- a/src/transformers/models/maskformer/modeling_maskformer.py +++ b/src/transformers/models/maskformer/modeling_maskformer.py @@ -416,7 +416,6 @@ def __init__( embed_dim: int, num_heads: int, dropout: float = 0.0, - is_decoder: bool = False, bias: bool = True, ): super().__init__() @@ -545,7 +544,6 @@ def __init__(self, config: DetrConfig): embed_dim=self.embed_dim, num_heads=config.decoder_attention_heads, dropout=config.attention_dropout, - is_decoder=True, ) self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] @@ -556,7 +554,6 @@ def __init__(self, config: DetrConfig): self.embed_dim, config.decoder_attention_heads, dropout=config.attention_dropout, - is_decoder=True, ) self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) diff --git a/src/transformers/models/table_transformer/modeling_table_transformer.py b/src/transformers/models/table_transformer/modeling_table_transformer.py index 6716a78be3b51e..733ff7b9b453df 100644 --- a/src/transformers/models/table_transformer/modeling_table_transformer.py +++ b/src/transformers/models/table_transformer/modeling_table_transformer.py @@ -440,7 +440,6 @@ def __init__( embed_dim: int, num_heads: int, dropout: float = 0.0, - is_decoder: bool = False, bias: bool = True, ): super().__init__() @@ -642,7 +641,6 @@ def __init__(self, config: TableTransformerConfig): embed_dim=self.embed_dim, num_heads=config.decoder_attention_heads, dropout=config.attention_dropout, - is_decoder=True, ) self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] @@ -653,7 +651,6 @@ def __init__(self, config: TableTransformerConfig): self.embed_dim, config.decoder_attention_heads, dropout=config.attention_dropout, - is_decoder=True, ) self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) From 372f50030bd422309d7b859d730842d7755d3371 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 15 Jun 2023 07:30:24 -0400 Subject: [PATCH 09/15] Split common test from core tests (#24284) --- tests/test_configuration_common.py | 270 +------ tests/test_configuration_utils.py | 286 +++++++ tests/test_feature_extraction_common.py | 127 +-- tests/test_feature_extraction_utils.py | 144 ++++ tests/test_image_processing_common.py | 149 +--- tests/test_image_processing_utils.py | 154 ++++ tests/test_modeling_common.py | 934 +---------------------- tests/test_modeling_flax_common.py | 175 +---- tests/test_modeling_flax_utils.py | 186 +++++ tests/test_modeling_tf_common.py | 575 +------------- tests/test_modeling_tf_utils.py | 627 +++++++++++++++ tests/test_modeling_utils.py | 976 ++++++++++++++++++++++++ tests/test_tokenization_common.py | 258 +------ tests/test_tokenization_utils.py | 280 +++++++ 14 files changed, 2663 insertions(+), 2478 deletions(-) create mode 100644 tests/test_configuration_utils.py create mode 100644 tests/test_feature_extraction_utils.py create mode 100644 tests/test_image_processing_utils.py create mode 100644 tests/test_modeling_flax_utils.py create mode 100644 tests/test_modeling_tf_utils.py create mode 100755 tests/test_modeling_utils.py create mode 100644 tests/test_tokenization_utils.py diff --git a/tests/test_configuration_common.py b/tests/test_configuration_common.py index fdb679529d2b41..18d9a76e97dc05 100644 --- a/tests/test_configuration_common.py +++ b/tests/test_configuration_common.py @@ -16,80 +16,11 @@ import copy import json import os -import shutil -import sys import tempfile -import unittest -import unittest.mock as mock -from pathlib import Path -from huggingface_hub import HfFolder, delete_repo -from requests.exceptions import HTTPError +from transformers import is_torch_available -from transformers import AutoConfig, BertConfig, GPT2Config, is_torch_available -from transformers.configuration_utils import PretrainedConfig -from transformers.testing_utils import TOKEN, USER, is_staging_test - - -sys.path.append(str(Path(__file__).parent.parent / "utils")) - -from test_module.custom_configuration import CustomConfig # noqa E402 - - -config_common_kwargs = { - "return_dict": False, - "output_hidden_states": True, - "output_attentions": True, - "torchscript": True, - "torch_dtype": "float16", - "use_bfloat16": True, - "tf_legacy_loss": True, - "pruned_heads": {"a": 1}, - "tie_word_embeddings": False, - "is_decoder": True, - "cross_attention_hidden_size": 128, - "add_cross_attention": True, - "tie_encoder_decoder": True, - "max_length": 50, - "min_length": 3, - "do_sample": True, - "early_stopping": True, - "num_beams": 3, - "num_beam_groups": 3, - "diversity_penalty": 0.5, - "temperature": 2.0, - "top_k": 10, - "top_p": 0.7, - "typical_p": 0.2, - "repetition_penalty": 0.8, - "length_penalty": 0.8, - "no_repeat_ngram_size": 5, - "encoder_no_repeat_ngram_size": 5, - "bad_words_ids": [1, 2, 3], - "num_return_sequences": 3, - "chunk_size_feed_forward": 5, - "output_scores": True, - "return_dict_in_generate": True, - "forced_bos_token_id": 2, - "forced_eos_token_id": 3, - "remove_invalid_values": True, - "architectures": ["BertModel"], - "finetuning_task": "translation", - "id2label": {0: "label"}, - "label2id": {"label": "0"}, - "tokenizer_class": "BertTokenizerFast", - "prefix": "prefix", - "bos_token_id": 6, - "pad_token_id": 7, - "eos_token_id": 8, - "sep_token_id": 9, - "decoder_start_token_id": 10, - "exponential_decay_length_penalty": (5, 1.01), - "suppress_tokens": [0, 1], - "begin_suppress_tokens": 2, - "task_specific_params": {"translation": "some_params"}, - "problem_type": "regression", -} +from .test_configuration_utils import config_common_kwargs class ConfigTester(object): @@ -220,200 +151,3 @@ def run_common_tests(self): self.create_and_test_config_with_num_labels() self.check_config_can_be_init_without_params() self.check_config_arguments_init() - - -@is_staging_test -class ConfigPushToHubTester(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-config") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-config-org") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="test-dynamic-config") - except HTTPError: - pass - - def test_push_to_hub(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - config.push_to_hub("test-config", use_auth_token=self._token) - - new_config = BertConfig.from_pretrained(f"{USER}/test-config") - for k, v in config.to_dict().items(): - if k != "transformers_version": - self.assertEqual(v, getattr(new_config, k)) - - # Reset repo - delete_repo(token=self._token, repo_id="test-config") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - config.save_pretrained(tmp_dir, repo_id="test-config", push_to_hub=True, use_auth_token=self._token) - - new_config = BertConfig.from_pretrained(f"{USER}/test-config") - for k, v in config.to_dict().items(): - if k != "transformers_version": - self.assertEqual(v, getattr(new_config, k)) - - def test_push_to_hub_in_organization(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - config.push_to_hub("valid_org/test-config-org", use_auth_token=self._token) - - new_config = BertConfig.from_pretrained("valid_org/test-config-org") - for k, v in config.to_dict().items(): - if k != "transformers_version": - self.assertEqual(v, getattr(new_config, k)) - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-config-org") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - config.save_pretrained( - tmp_dir, repo_id="valid_org/test-config-org", push_to_hub=True, use_auth_token=self._token - ) - - new_config = BertConfig.from_pretrained("valid_org/test-config-org") - for k, v in config.to_dict().items(): - if k != "transformers_version": - self.assertEqual(v, getattr(new_config, k)) - - def test_push_to_hub_dynamic_config(self): - CustomConfig.register_for_auto_class() - config = CustomConfig(attribute=42) - - config.push_to_hub("test-dynamic-config", use_auth_token=self._token) - - # This has added the proper auto_map field to the config - self.assertDictEqual(config.auto_map, {"AutoConfig": "custom_configuration.CustomConfig"}) - - new_config = AutoConfig.from_pretrained(f"{USER}/test-dynamic-config", trust_remote_code=True) - # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module - self.assertEqual(new_config.__class__.__name__, "CustomConfig") - self.assertEqual(new_config.attribute, 42) - - -class ConfigTestUtils(unittest.TestCase): - def test_config_from_string(self): - c = GPT2Config() - - # attempt to modify each of int/float/bool/str config records and verify they were updated - n_embd = c.n_embd + 1 # int - resid_pdrop = c.resid_pdrop + 1.0 # float - scale_attn_weights = not c.scale_attn_weights # bool - summary_type = c.summary_type + "foo" # str - c.update_from_string( - f"n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}" - ) - self.assertEqual(n_embd, c.n_embd, "mismatch for key: n_embd") - self.assertEqual(resid_pdrop, c.resid_pdrop, "mismatch for key: resid_pdrop") - self.assertEqual(scale_attn_weights, c.scale_attn_weights, "mismatch for key: scale_attn_weights") - self.assertEqual(summary_type, c.summary_type, "mismatch for key: summary_type") - - def test_config_common_kwargs_is_complete(self): - base_config = PretrainedConfig() - missing_keys = [key for key in base_config.__dict__ if key not in config_common_kwargs] - # If this part of the test fails, you have arguments to addin config_common_kwargs above. - self.assertListEqual( - missing_keys, ["is_encoder_decoder", "_name_or_path", "_commit_hash", "transformers_version"] - ) - keys_with_defaults = [key for key, value in config_common_kwargs.items() if value == getattr(base_config, key)] - if len(keys_with_defaults) > 0: - raise ValueError( - "The following keys are set with the default values in" - " `test_configuration_common.config_common_kwargs` pick another value for them:" - f" {', '.join(keys_with_defaults)}." - ) - - def test_from_pretrained_subfolder(self): - with self.assertRaises(OSError): - # config is in subfolder, the following should not work without specifying the subfolder - _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder") - - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder", subfolder="bert") - - self.assertIsNotNone(config) - - def test_cached_files_are_used_when_internet_is_down(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Under the mock environment we get a 500 error when trying to reach the model. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - # This check we did call the fake head request - mock_head.assert_called() - - def test_legacy_load_from_url(self): - # This test is for deprecated behavior and can be removed in v5 - _ = BertConfig.from_pretrained( - "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json" - ) - - -class ConfigurationVersioningTest(unittest.TestCase): - def test_local_versioning(self): - configuration = AutoConfig.from_pretrained("bert-base-cased") - configuration.configuration_files = ["config.4.0.0.json"] - - with tempfile.TemporaryDirectory() as tmp_dir: - configuration.save_pretrained(tmp_dir) - configuration.hidden_size = 2 - json.dump(configuration.to_dict(), open(os.path.join(tmp_dir, "config.4.0.0.json"), "w")) - - # This should pick the new configuration file as the version of Transformers is > 4.0.0 - new_configuration = AutoConfig.from_pretrained(tmp_dir) - self.assertEqual(new_configuration.hidden_size, 2) - - # Will need to be adjusted if we reach v42 and this test is still here. - # Should pick the old configuration file as the version of Transformers is < 4.42.0 - configuration.configuration_files = ["config.42.0.0.json"] - configuration.hidden_size = 768 - configuration.save_pretrained(tmp_dir) - shutil.move(os.path.join(tmp_dir, "config.4.0.0.json"), os.path.join(tmp_dir, "config.42.0.0.json")) - new_configuration = AutoConfig.from_pretrained(tmp_dir) - self.assertEqual(new_configuration.hidden_size, 768) - - def test_repo_versioning_before(self): - # This repo has two configuration files, one for v4.0.0 and above with a different hidden size. - repo = "hf-internal-testing/test-two-configs" - - import transformers as new_transformers - - new_transformers.configuration_utils.__version__ = "v4.0.0" - new_configuration, kwargs = new_transformers.models.auto.AutoConfig.from_pretrained( - repo, return_unused_kwargs=True - ) - self.assertEqual(new_configuration.hidden_size, 2) - # This checks `_configuration_file` ia not kept in the kwargs by mistake. - self.assertDictEqual(kwargs, {}) - - # Testing an older version by monkey-patching the version in the module it's used. - import transformers as old_transformers - - old_transformers.configuration_utils.__version__ = "v3.0.0" - old_configuration = old_transformers.models.auto.AutoConfig.from_pretrained(repo) - self.assertEqual(old_configuration.hidden_size, 768) diff --git a/tests/test_configuration_utils.py b/tests/test_configuration_utils.py new file mode 100644 index 00000000000000..fe278cc2b14908 --- /dev/null +++ b/tests/test_configuration_utils.py @@ -0,0 +1,286 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import sys +import tempfile +import unittest +import unittest.mock as mock +from pathlib import Path + +from huggingface_hub import HfFolder, delete_repo +from requests.exceptions import HTTPError + +from transformers import AutoConfig, BertConfig, GPT2Config +from transformers.configuration_utils import PretrainedConfig +from transformers.testing_utils import TOKEN, USER, is_staging_test + + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from test_module.custom_configuration import CustomConfig # noqa E402 + + +config_common_kwargs = { + "return_dict": False, + "output_hidden_states": True, + "output_attentions": True, + "torchscript": True, + "torch_dtype": "float16", + "use_bfloat16": True, + "tf_legacy_loss": True, + "pruned_heads": {"a": 1}, + "tie_word_embeddings": False, + "is_decoder": True, + "cross_attention_hidden_size": 128, + "add_cross_attention": True, + "tie_encoder_decoder": True, + "max_length": 50, + "min_length": 3, + "do_sample": True, + "early_stopping": True, + "num_beams": 3, + "num_beam_groups": 3, + "diversity_penalty": 0.5, + "temperature": 2.0, + "top_k": 10, + "top_p": 0.7, + "typical_p": 0.2, + "repetition_penalty": 0.8, + "length_penalty": 0.8, + "no_repeat_ngram_size": 5, + "encoder_no_repeat_ngram_size": 5, + "bad_words_ids": [1, 2, 3], + "num_return_sequences": 3, + "chunk_size_feed_forward": 5, + "output_scores": True, + "return_dict_in_generate": True, + "forced_bos_token_id": 2, + "forced_eos_token_id": 3, + "remove_invalid_values": True, + "architectures": ["BertModel"], + "finetuning_task": "translation", + "id2label": {0: "label"}, + "label2id": {"label": "0"}, + "tokenizer_class": "BertTokenizerFast", + "prefix": "prefix", + "bos_token_id": 6, + "pad_token_id": 7, + "eos_token_id": 8, + "sep_token_id": 9, + "decoder_start_token_id": 10, + "exponential_decay_length_penalty": (5, 1.01), + "suppress_tokens": [0, 1], + "begin_suppress_tokens": 2, + "task_specific_params": {"translation": "some_params"}, + "problem_type": "regression", +} + + +@is_staging_test +class ConfigPushToHubTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-config") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-config-org") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="test-dynamic-config") + except HTTPError: + pass + + def test_push_to_hub(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + config.push_to_hub("test-config", use_auth_token=self._token) + + new_config = BertConfig.from_pretrained(f"{USER}/test-config") + for k, v in config.to_dict().items(): + if k != "transformers_version": + self.assertEqual(v, getattr(new_config, k)) + + # Reset repo + delete_repo(token=self._token, repo_id="test-config") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + config.save_pretrained(tmp_dir, repo_id="test-config", push_to_hub=True, use_auth_token=self._token) + + new_config = BertConfig.from_pretrained(f"{USER}/test-config") + for k, v in config.to_dict().items(): + if k != "transformers_version": + self.assertEqual(v, getattr(new_config, k)) + + def test_push_to_hub_in_organization(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + config.push_to_hub("valid_org/test-config-org", use_auth_token=self._token) + + new_config = BertConfig.from_pretrained("valid_org/test-config-org") + for k, v in config.to_dict().items(): + if k != "transformers_version": + self.assertEqual(v, getattr(new_config, k)) + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-config-org") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + config.save_pretrained( + tmp_dir, repo_id="valid_org/test-config-org", push_to_hub=True, use_auth_token=self._token + ) + + new_config = BertConfig.from_pretrained("valid_org/test-config-org") + for k, v in config.to_dict().items(): + if k != "transformers_version": + self.assertEqual(v, getattr(new_config, k)) + + def test_push_to_hub_dynamic_config(self): + CustomConfig.register_for_auto_class() + config = CustomConfig(attribute=42) + + config.push_to_hub("test-dynamic-config", use_auth_token=self._token) + + # This has added the proper auto_map field to the config + self.assertDictEqual(config.auto_map, {"AutoConfig": "custom_configuration.CustomConfig"}) + + new_config = AutoConfig.from_pretrained(f"{USER}/test-dynamic-config", trust_remote_code=True) + # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module + self.assertEqual(new_config.__class__.__name__, "CustomConfig") + self.assertEqual(new_config.attribute, 42) + + +class ConfigTestUtils(unittest.TestCase): + def test_config_from_string(self): + c = GPT2Config() + + # attempt to modify each of int/float/bool/str config records and verify they were updated + n_embd = c.n_embd + 1 # int + resid_pdrop = c.resid_pdrop + 1.0 # float + scale_attn_weights = not c.scale_attn_weights # bool + summary_type = c.summary_type + "foo" # str + c.update_from_string( + f"n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}" + ) + self.assertEqual(n_embd, c.n_embd, "mismatch for key: n_embd") + self.assertEqual(resid_pdrop, c.resid_pdrop, "mismatch for key: resid_pdrop") + self.assertEqual(scale_attn_weights, c.scale_attn_weights, "mismatch for key: scale_attn_weights") + self.assertEqual(summary_type, c.summary_type, "mismatch for key: summary_type") + + def test_config_common_kwargs_is_complete(self): + base_config = PretrainedConfig() + missing_keys = [key for key in base_config.__dict__ if key not in config_common_kwargs] + # If this part of the test fails, you have arguments to addin config_common_kwargs above. + self.assertListEqual( + missing_keys, ["is_encoder_decoder", "_name_or_path", "_commit_hash", "transformers_version"] + ) + keys_with_defaults = [key for key, value in config_common_kwargs.items() if value == getattr(base_config, key)] + if len(keys_with_defaults) > 0: + raise ValueError( + "The following keys are set with the default values in" + " `test_configuration_common.config_common_kwargs` pick another value for them:" + f" {', '.join(keys_with_defaults)}." + ) + + def test_from_pretrained_subfolder(self): + with self.assertRaises(OSError): + # config is in subfolder, the following should not work without specifying the subfolder + _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder") + + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder", subfolder="bert") + + self.assertIsNotNone(config) + + def test_cached_files_are_used_when_internet_is_down(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + # This check we did call the fake head request + mock_head.assert_called() + + def test_legacy_load_from_url(self): + # This test is for deprecated behavior and can be removed in v5 + _ = BertConfig.from_pretrained( + "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json" + ) + + def test_local_versioning(self): + configuration = AutoConfig.from_pretrained("bert-base-cased") + configuration.configuration_files = ["config.4.0.0.json"] + + with tempfile.TemporaryDirectory() as tmp_dir: + configuration.save_pretrained(tmp_dir) + configuration.hidden_size = 2 + json.dump(configuration.to_dict(), open(os.path.join(tmp_dir, "config.4.0.0.json"), "w")) + + # This should pick the new configuration file as the version of Transformers is > 4.0.0 + new_configuration = AutoConfig.from_pretrained(tmp_dir) + self.assertEqual(new_configuration.hidden_size, 2) + + # Will need to be adjusted if we reach v42 and this test is still here. + # Should pick the old configuration file as the version of Transformers is < 4.42.0 + configuration.configuration_files = ["config.42.0.0.json"] + configuration.hidden_size = 768 + configuration.save_pretrained(tmp_dir) + shutil.move(os.path.join(tmp_dir, "config.4.0.0.json"), os.path.join(tmp_dir, "config.42.0.0.json")) + new_configuration = AutoConfig.from_pretrained(tmp_dir) + self.assertEqual(new_configuration.hidden_size, 768) + + def test_repo_versioning_before(self): + # This repo has two configuration files, one for v4.0.0 and above with a different hidden size. + repo = "hf-internal-testing/test-two-configs" + + import transformers as new_transformers + + new_transformers.configuration_utils.__version__ = "v4.0.0" + new_configuration, kwargs = new_transformers.models.auto.AutoConfig.from_pretrained( + repo, return_unused_kwargs=True + ) + self.assertEqual(new_configuration.hidden_size, 2) + # This checks `_configuration_file` ia not kept in the kwargs by mistake. + self.assertDictEqual(kwargs, {}) + + # Testing an older version by monkey-patching the version in the module it's used. + import transformers as old_transformers + + old_transformers.configuration_utils.__version__ = "v3.0.0" + old_configuration = old_transformers.models.auto.AutoConfig.from_pretrained(repo) + self.assertEqual(old_configuration.hidden_size, 768) diff --git a/tests/test_feature_extraction_common.py b/tests/test_feature_extraction_common.py index 6ffec669133c8f..49937309d0e01e 100644 --- a/tests/test_feature_extraction_common.py +++ b/tests/test_feature_extraction_common.py @@ -16,25 +16,9 @@ import json import os -import sys import tempfile -import unittest -import unittest.mock as mock -from pathlib import Path -from huggingface_hub import HfFolder, delete_repo -from requests.exceptions import HTTPError - -from transformers import AutoFeatureExtractor, Wav2Vec2FeatureExtractor -from transformers.testing_utils import TOKEN, USER, check_json_file_has_correct_format, get_tests_dir, is_staging_test - - -sys.path.append(str(Path(__file__).parent.parent / "utils")) - -from test_module.custom_feature_extraction import CustomFeatureExtractor # noqa E402 - - -SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR = get_tests_dir("fixtures") +from transformers.testing_utils import check_json_file_has_correct_format class FeatureExtractionSavingTestMixin: @@ -69,112 +53,3 @@ def test_feat_extract_from_and_save_pretrained(self): def test_init_without_params(self): feat_extract = self.feature_extraction_class() self.assertIsNotNone(feat_extract) - - -class FeatureExtractorUtilTester(unittest.TestCase): - def test_cached_files_are_used_when_internet_is_down(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = Wav2Vec2FeatureExtractor.from_pretrained("hf-internal-testing/tiny-random-wav2vec2") - # Under the mock environment we get a 500 error when trying to reach the model. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = Wav2Vec2FeatureExtractor.from_pretrained("hf-internal-testing/tiny-random-wav2vec2") - # This check we did call the fake head request - mock_head.assert_called() - - def test_legacy_load_from_url(self): - # This test is for deprecated behavior and can be removed in v5 - _ = Wav2Vec2FeatureExtractor.from_pretrained( - "https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json" - ) - - -@is_staging_test -class FeatureExtractorPushToHubTester(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-feature-extractor") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-feature-extractor-org") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="test-dynamic-feature-extractor") - except HTTPError: - pass - - def test_push_to_hub(self): - feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) - feature_extractor.push_to_hub("test-feature-extractor", use_auth_token=self._token) - - new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor") - for k, v in feature_extractor.__dict__.items(): - self.assertEqual(v, getattr(new_feature_extractor, k)) - - # Reset repo - delete_repo(token=self._token, repo_id="test-feature-extractor") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - feature_extractor.save_pretrained( - tmp_dir, repo_id="test-feature-extractor", push_to_hub=True, use_auth_token=self._token - ) - - new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor") - for k, v in feature_extractor.__dict__.items(): - self.assertEqual(v, getattr(new_feature_extractor, k)) - - def test_push_to_hub_in_organization(self): - feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) - feature_extractor.push_to_hub("valid_org/test-feature-extractor", use_auth_token=self._token) - - new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("valid_org/test-feature-extractor") - for k, v in feature_extractor.__dict__.items(): - self.assertEqual(v, getattr(new_feature_extractor, k)) - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-feature-extractor") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - feature_extractor.save_pretrained( - tmp_dir, repo_id="valid_org/test-feature-extractor-org", push_to_hub=True, use_auth_token=self._token - ) - - new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("valid_org/test-feature-extractor-org") - for k, v in feature_extractor.__dict__.items(): - self.assertEqual(v, getattr(new_feature_extractor, k)) - - def test_push_to_hub_dynamic_feature_extractor(self): - CustomFeatureExtractor.register_for_auto_class() - feature_extractor = CustomFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) - - feature_extractor.push_to_hub("test-dynamic-feature-extractor", use_auth_token=self._token) - - # This has added the proper auto_map field to the config - self.assertDictEqual( - feature_extractor.auto_map, - {"AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor"}, - ) - - new_feature_extractor = AutoFeatureExtractor.from_pretrained( - f"{USER}/test-dynamic-feature-extractor", trust_remote_code=True - ) - # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module - self.assertEqual(new_feature_extractor.__class__.__name__, "CustomFeatureExtractor") diff --git a/tests/test_feature_extraction_utils.py b/tests/test_feature_extraction_utils.py new file mode 100644 index 00000000000000..b17c48ff120d4f --- /dev/null +++ b/tests/test_feature_extraction_utils.py @@ -0,0 +1,144 @@ +# coding=utf-8 +# Copyright 2021 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +import tempfile +import unittest +import unittest.mock as mock +from pathlib import Path + +from huggingface_hub import HfFolder, delete_repo +from requests.exceptions import HTTPError + +from transformers import AutoFeatureExtractor, Wav2Vec2FeatureExtractor +from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test + + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from test_module.custom_feature_extraction import CustomFeatureExtractor # noqa E402 + + +SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR = get_tests_dir("fixtures") + + +class FeatureExtractorUtilTester(unittest.TestCase): + def test_cached_files_are_used_when_internet_is_down(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = Wav2Vec2FeatureExtractor.from_pretrained("hf-internal-testing/tiny-random-wav2vec2") + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = Wav2Vec2FeatureExtractor.from_pretrained("hf-internal-testing/tiny-random-wav2vec2") + # This check we did call the fake head request + mock_head.assert_called() + + def test_legacy_load_from_url(self): + # This test is for deprecated behavior and can be removed in v5 + _ = Wav2Vec2FeatureExtractor.from_pretrained( + "https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json" + ) + + +@is_staging_test +class FeatureExtractorPushToHubTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-feature-extractor") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-feature-extractor-org") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="test-dynamic-feature-extractor") + except HTTPError: + pass + + def test_push_to_hub(self): + feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) + feature_extractor.push_to_hub("test-feature-extractor", use_auth_token=self._token) + + new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor") + for k, v in feature_extractor.__dict__.items(): + self.assertEqual(v, getattr(new_feature_extractor, k)) + + # Reset repo + delete_repo(token=self._token, repo_id="test-feature-extractor") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + feature_extractor.save_pretrained( + tmp_dir, repo_id="test-feature-extractor", push_to_hub=True, use_auth_token=self._token + ) + + new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(f"{USER}/test-feature-extractor") + for k, v in feature_extractor.__dict__.items(): + self.assertEqual(v, getattr(new_feature_extractor, k)) + + def test_push_to_hub_in_organization(self): + feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) + feature_extractor.push_to_hub("valid_org/test-feature-extractor", use_auth_token=self._token) + + new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("valid_org/test-feature-extractor") + for k, v in feature_extractor.__dict__.items(): + self.assertEqual(v, getattr(new_feature_extractor, k)) + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-feature-extractor") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + feature_extractor.save_pretrained( + tmp_dir, repo_id="valid_org/test-feature-extractor-org", push_to_hub=True, use_auth_token=self._token + ) + + new_feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("valid_org/test-feature-extractor-org") + for k, v in feature_extractor.__dict__.items(): + self.assertEqual(v, getattr(new_feature_extractor, k)) + + def test_push_to_hub_dynamic_feature_extractor(self): + CustomFeatureExtractor.register_for_auto_class() + feature_extractor = CustomFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR) + + feature_extractor.push_to_hub("test-dynamic-feature-extractor", use_auth_token=self._token) + + # This has added the proper auto_map field to the config + self.assertDictEqual( + feature_extractor.auto_map, + {"AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor"}, + ) + + new_feature_extractor = AutoFeatureExtractor.from_pretrained( + f"{USER}/test-dynamic-feature-extractor", trust_remote_code=True + ) + # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module + self.assertEqual(new_feature_extractor.__class__.__name__, "CustomFeatureExtractor") diff --git a/tests/test_image_processing_common.py b/tests/test_image_processing_common.py index 15a6688fbb9c4b..166440c4d52514 100644 --- a/tests/test_image_processing_common.py +++ b/tests/test_image_processing_common.py @@ -13,34 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. - import json import os -import sys import tempfile -import unittest -import unittest.mock as mock -from pathlib import Path - -from huggingface_hub import HfFolder, delete_repo -from requests.exceptions import HTTPError - -from transformers import AutoImageProcessor, ViTImageProcessor -from transformers.testing_utils import ( - TOKEN, - USER, - check_json_file_has_correct_format, - get_tests_dir, - is_staging_test, - require_torch, - require_vision, -) -from transformers.utils import is_torch_available, is_vision_available - - -sys.path.append(str(Path(__file__).parent.parent / "utils")) -from test_module.custom_image_processing import CustomImageProcessor # noqa E402 +from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_vision +from transformers.utils import is_torch_available, is_vision_available if is_torch_available(): @@ -51,9 +29,6 @@ from PIL import Image -SAMPLE_IMAGE_PROCESSING_CONFIG_DIR = get_tests_dir("fixtures") - - def prepare_image_inputs(image_processor_tester, equal_resolution=False, numpify=False, torchify=False): """This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True, or a list of PyTorch tensors if one specifies torchify=True. @@ -201,123 +176,3 @@ def test_cast_dtype_device(self): self.assertEqual(encoding.pixel_values.device, torch.device("cpu")) self.assertEqual(encoding.pixel_values.dtype, torch.float16) self.assertEqual(encoding.input_ids.dtype, torch.long) - - -class ImageProcessorUtilTester(unittest.TestCase): - def test_cached_files_are_used_when_internet_is_down(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit") - # Under the mock environment we get a 500 error when trying to reach the model. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit") - # This check we did call the fake head request - mock_head.assert_called() - - def test_legacy_load_from_url(self): - # This test is for deprecated behavior and can be removed in v5 - _ = ViTImageProcessor.from_pretrained( - "https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json" - ) - - -@is_staging_test -class ImageProcessorPushToHubTester(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-image-processor") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-image-processor-org") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="test-dynamic-image-processor") - except HTTPError: - pass - - def test_push_to_hub(self): - image_processor = ViTImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) - image_processor.push_to_hub("test-image-processor", use_auth_token=self._token) - - new_image_processor = ViTImageProcessor.from_pretrained(f"{USER}/test-image-processor") - for k, v in image_processor.__dict__.items(): - self.assertEqual(v, getattr(new_image_processor, k)) - - # Reset repo - delete_repo(token=self._token, repo_id="test-image-processor") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - image_processor.save_pretrained( - tmp_dir, repo_id="test-image-processor", push_to_hub=True, use_auth_token=self._token - ) - - new_image_processor = ViTImageProcessor.from_pretrained(f"{USER}/test-image-processor") - for k, v in image_processor.__dict__.items(): - self.assertEqual(v, getattr(new_image_processor, k)) - - def test_push_to_hub_in_organization(self): - image_processor = ViTImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) - image_processor.push_to_hub("valid_org/test-image-processor", use_auth_token=self._token) - - new_image_processor = ViTImageProcessor.from_pretrained("valid_org/test-image-processor") - for k, v in image_processor.__dict__.items(): - self.assertEqual(v, getattr(new_image_processor, k)) - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-image-processor") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - image_processor.save_pretrained( - tmp_dir, repo_id="valid_org/test-image-processor-org", push_to_hub=True, use_auth_token=self._token - ) - - new_image_processor = ViTImageProcessor.from_pretrained("valid_org/test-image-processor-org") - for k, v in image_processor.__dict__.items(): - self.assertEqual(v, getattr(new_image_processor, k)) - - def test_push_to_hub_dynamic_image_processor(self): - CustomImageProcessor.register_for_auto_class() - image_processor = CustomImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) - - image_processor.push_to_hub("test-dynamic-image-processor", use_auth_token=self._token) - - # This has added the proper auto_map field to the config - self.assertDictEqual( - image_processor.auto_map, - {"ImageProcessor": "custom_image_processing.CustomImageProcessor"}, - ) - - new_image_processor = AutoImageProcessor.from_pretrained( - f"{USER}/test-dynamic-image-processor", trust_remote_code=True - ) - # Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module - self.assertEqual(new_image_processor.__class__.__name__, "CustomImageProcessor") - - def test_image_processor_from_pretrained_subfolder(self): - with self.assertRaises(OSError): - # config is in subfolder, the following should not work without specifying the subfolder - _ = AutoImageProcessor.from_pretrained("hf-internal-testing/stable-diffusion-all-variants") - - config = AutoImageProcessor.from_pretrained( - "hf-internal-testing/stable-diffusion-all-variants", subfolder="feature_extractor" - ) - - self.assertIsNotNone(config) diff --git a/tests/test_image_processing_utils.py b/tests/test_image_processing_utils.py new file mode 100644 index 00000000000000..d74da63249ba62 --- /dev/null +++ b/tests/test_image_processing_utils.py @@ -0,0 +1,154 @@ +# coding=utf-8 +# Copyright 2023 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import tempfile +import unittest +import unittest.mock as mock +from pathlib import Path + +from huggingface_hub import HfFolder, delete_repo +from requests.exceptions import HTTPError + +from transformers import AutoImageProcessor, ViTImageProcessor +from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test + + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from test_module.custom_image_processing import CustomImageProcessor # noqa E402 + + +SAMPLE_IMAGE_PROCESSING_CONFIG_DIR = get_tests_dir("fixtures") + + +class ImageProcessorUtilTester(unittest.TestCase): + def test_cached_files_are_used_when_internet_is_down(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit") + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit") + # This check we did call the fake head request + mock_head.assert_called() + + def test_legacy_load_from_url(self): + # This test is for deprecated behavior and can be removed in v5 + _ = ViTImageProcessor.from_pretrained( + "https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json" + ) + + +@is_staging_test +class ImageProcessorPushToHubTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-image-processor") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-image-processor-org") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="test-dynamic-image-processor") + except HTTPError: + pass + + def test_push_to_hub(self): + image_processor = ViTImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) + image_processor.push_to_hub("test-image-processor", use_auth_token=self._token) + + new_image_processor = ViTImageProcessor.from_pretrained(f"{USER}/test-image-processor") + for k, v in image_processor.__dict__.items(): + self.assertEqual(v, getattr(new_image_processor, k)) + + # Reset repo + delete_repo(token=self._token, repo_id="test-image-processor") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + image_processor.save_pretrained( + tmp_dir, repo_id="test-image-processor", push_to_hub=True, use_auth_token=self._token + ) + + new_image_processor = ViTImageProcessor.from_pretrained(f"{USER}/test-image-processor") + for k, v in image_processor.__dict__.items(): + self.assertEqual(v, getattr(new_image_processor, k)) + + def test_push_to_hub_in_organization(self): + image_processor = ViTImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) + image_processor.push_to_hub("valid_org/test-image-processor", use_auth_token=self._token) + + new_image_processor = ViTImageProcessor.from_pretrained("valid_org/test-image-processor") + for k, v in image_processor.__dict__.items(): + self.assertEqual(v, getattr(new_image_processor, k)) + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-image-processor") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + image_processor.save_pretrained( + tmp_dir, repo_id="valid_org/test-image-processor-org", push_to_hub=True, use_auth_token=self._token + ) + + new_image_processor = ViTImageProcessor.from_pretrained("valid_org/test-image-processor-org") + for k, v in image_processor.__dict__.items(): + self.assertEqual(v, getattr(new_image_processor, k)) + + def test_push_to_hub_dynamic_image_processor(self): + CustomImageProcessor.register_for_auto_class() + image_processor = CustomImageProcessor.from_pretrained(SAMPLE_IMAGE_PROCESSING_CONFIG_DIR) + + image_processor.push_to_hub("test-dynamic-image-processor", use_auth_token=self._token) + + # This has added the proper auto_map field to the config + self.assertDictEqual( + image_processor.auto_map, + {"ImageProcessor": "custom_image_processing.CustomImageProcessor"}, + ) + + new_image_processor = AutoImageProcessor.from_pretrained( + f"{USER}/test-dynamic-image-processor", trust_remote_code=True + ) + # Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module + self.assertEqual(new_image_processor.__class__.__name__, "CustomImageProcessor") + + def test_image_processor_from_pretrained_subfolder(self): + with self.assertRaises(OSError): + # config is in subfolder, the following should not work without specifying the subfolder + _ = AutoImageProcessor.from_pretrained("hf-internal-testing/stable-diffusion-all-variants") + + config = AutoImageProcessor.from_pretrained( + "hf-internal-testing/stable-diffusion-all-variants", subfolder="feature_extractor" + ) + + self.assertIsNotNone(config) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 1a55aa6b754f2f..07a8b16bfef758 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -16,32 +16,22 @@ import collections import copy import gc -import glob import inspect -import json import os import os.path import pickle import random import re -import sys import tempfile -import unittest -import unittest.mock as mock import warnings from collections import defaultdict -from pathlib import Path from typing import Dict, List, Tuple import numpy as np -from huggingface_hub import HfFolder, delete_repo -from huggingface_hub.file_download import http_get from pytest import mark -from requests.exceptions import HTTPError import transformers from transformers import ( - AutoConfig, AutoModel, AutoModelForSequenceClassification, PretrainedConfig, @@ -70,28 +60,20 @@ MODEL_MAPPING_NAMES, ) from transformers.testing_utils import ( - TOKEN, - USER, CaptureLogger, - TestCasePlus, is_pt_flax_cross_test, is_pt_tf_cross_test, - is_staging_test, require_accelerate, require_safetensors, require_torch, require_torch_gpu, require_torch_multi_gpu, - require_usr_bin_time, slow, torch_device, ) from transformers.utils import ( CONFIG_NAME, GENERATION_CONFIG_NAME, - SAFE_WEIGHTS_INDEX_NAME, - SAFE_WEIGHTS_NAME, - WEIGHTS_INDEX_NAME, WEIGHTS_NAME, is_accelerate_available, is_flax_available, @@ -101,65 +83,17 @@ from transformers.utils.generic import ModelOutput -sys.path.append(str(Path(__file__).parent.parent / "utils")) - -from test_module.custom_configuration import CustomConfig, NoSuperInitConfig # noqa E402 - - if is_accelerate_available(): from accelerate.utils import compute_module_sizes if is_torch_available(): import torch - from test_module.custom_modeling import CustomModel, NoSuperInitModel from torch import nn - from transformers import ( - BERT_PRETRAINED_MODEL_ARCHIVE_LIST, - MODEL_MAPPING, - AdaptiveEmbedding, - AutoModelForCausalLM, - AutoTokenizer, - BertConfig, - BertModel, - CLIPTextModel, - PreTrainedModel, - T5Config, - T5ForConditionalGeneration, - ) - from transformers.modeling_utils import shard_checkpoint + from transformers import MODEL_MAPPING, AdaptiveEmbedding from transformers.pytorch_utils import id_tensor_storage - # Fake pretrained models for tests - class BaseModel(PreTrainedModel): - config_class = PretrainedConfig - - def __init__(self, config): - super().__init__(config) - self.linear = nn.Linear(4, 5) - self.linear_2 = nn.Linear(5, 6) - - def forward(self, x): - return self.linear_2(self.linear(x)) - - class ModelWithHead(PreTrainedModel): - base_model_prefix = "base" - config_class = PretrainedConfig - - def _init_weights(self, module): - pass - - def __init__(self, config): - super().__init__(config) - self.base = BaseModel(config) - # linear is a common name between Base and Head on purpose. - self.linear = nn.Linear(6, 3) - self.linear2 = nn.Linear(3, 5) - - def forward(self, x): - return self.linear2(self.linear(self.base(x))) - if is_tf_available(): import tensorflow as tf @@ -187,10 +121,6 @@ def _config_zero_init(config): return configs_no_init -TINY_T5 = "patrickvonplaten/t5-tiny-random" -TINY_BERT_FOR_TOKEN_CLASSIFICATION = "hf-internal-testing/tiny-bert-for-token-classification" - - def _mock_init_weights(self, module): for name, param in module.named_parameters(recurse=False): # Use the first letter of the name to get a value and go from a <> -13 to z <> 12 @@ -2778,865 +2708,3 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None): values.append(rng.random() * scale) return torch.tensor(data=values, dtype=torch.float, device=torch_device).view(shape).contiguous() - - -def check_models_equal(model1, model2): - models_are_equal = True - for model1_p, model2_p in zip(model1.parameters(), model2.parameters()): - if model1_p.data.ne(model2_p.data).sum() > 0: - models_are_equal = False - - return models_are_equal - - -@require_torch -class ModelUtilsTest(TestCasePlus): - @slow - def test_model_from_pretrained(self): - for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - config = BertConfig.from_pretrained(model_name) - self.assertIsNotNone(config) - self.assertIsInstance(config, PretrainedConfig) - - model = BertModel.from_pretrained(model_name) - model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) - self.assertIsNotNone(model) - self.assertIsInstance(model, PreTrainedModel) - - self.assertEqual(len(loading_info["missing_keys"]), 0) - self.assertEqual(len(loading_info["unexpected_keys"]), 8) - self.assertEqual(len(loading_info["mismatched_keys"]), 0) - self.assertEqual(len(loading_info["error_msgs"]), 0) - - config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) - - # Not sure this is the intended behavior. TODO fix Lysandre & Thom - config.name_or_path = model_name - - model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) - self.assertEqual(model.config.output_hidden_states, True) - self.assertEqual(model.config, config) - - def test_model_from_pretrained_subfolder(self): - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - model = BertModel(config) - - subfolder = "bert" - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(os.path.join(tmp_dir, subfolder)) - - with self.assertRaises(OSError): - _ = BertModel.from_pretrained(tmp_dir) - - model_loaded = BertModel.from_pretrained(tmp_dir, subfolder=subfolder) - - self.assertTrue(check_models_equal(model, model_loaded)) - - def test_model_from_pretrained_subfolder_sharded(self): - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - model = BertModel(config) - - subfolder = "bert" - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(os.path.join(tmp_dir, subfolder), max_shard_size="10KB") - - with self.assertRaises(OSError): - _ = BertModel.from_pretrained(tmp_dir) - - model_loaded = BertModel.from_pretrained(tmp_dir, subfolder=subfolder) - - self.assertTrue(check_models_equal(model, model_loaded)) - - def test_model_from_pretrained_hub_subfolder(self): - subfolder = "bert" - model_id = "hf-internal-testing/tiny-random-bert-subfolder" - with self.assertRaises(OSError): - _ = BertModel.from_pretrained(model_id) - - model = BertModel.from_pretrained(model_id, subfolder=subfolder) - - self.assertIsNotNone(model) - - def test_model_from_pretrained_hub_subfolder_sharded(self): - subfolder = "bert" - model_id = "hf-internal-testing/tiny-random-bert-sharded-subfolder" - with self.assertRaises(OSError): - _ = BertModel.from_pretrained(model_id) - - model = BertModel.from_pretrained(model_id, subfolder=subfolder) - - self.assertIsNotNone(model) - - def test_model_from_pretrained_with_different_pretrained_model_name(self): - model = T5ForConditionalGeneration.from_pretrained(TINY_T5) - self.assertIsNotNone(model) - - logger = logging.get_logger("transformers.configuration_utils") - with CaptureLogger(logger) as cl: - BertModel.from_pretrained(TINY_T5) - self.assertTrue("You are using a model of type t5 to instantiate a model of type bert" in cl.out) - - def test_model_from_config_torch_dtype(self): - # test that the model can be instantiated with dtype of user's choice - as long as it's a - # float dtype. To make it happen config.torch_dtype needs to be set before instantiating the - # model from the config object. - - config = T5Config.from_pretrained(TINY_T5) - model = AutoModel.from_config(config) - # XXX: isn't supported - # model = T5ForConditionalGeneration.from_config(config) - self.assertEqual(model.dtype, torch.float32) - - model = AutoModel.from_config(config, torch_dtype=torch.float16) - self.assertEqual(model.dtype, torch.float16) - - # torch.set_default_dtype() supports only float dtypes, so will fail with non-float type - with self.assertRaises(ValueError): - model = AutoModel.from_config(config, torch_dtype=torch.int64) - - def test_model_from_pretrained_torch_dtype(self): - # test that the model can be instantiated with dtype of either - # 1. explicit from_pretrained's torch_dtype argument - # 2. via autodiscovery by looking at model weights (torch_dtype="auto") - # so if a model.half() was saved, we want it to be instantiated as such. - # - # test an explicit model class, but also AutoModel separately as the latter goes through a different code path - model_path = self.get_auto_remove_tmp_dir() - - # baseline - we know TINY_T5 is fp32 model - model = T5ForConditionalGeneration.from_pretrained(TINY_T5) - self.assertEqual(model.dtype, torch.float32) - - def remove_torch_dtype(model_path): - file = f"{model_path}/config.json" - with open(file, "r", encoding="utf-8") as f: - s = json.load(f) - s.pop("torch_dtype") - with open(file, "w", encoding="utf-8") as f: - json.dump(s, f) - - # test the default fp32 save_pretrained => from_pretrained cycle - model.save_pretrained(model_path) - model = T5ForConditionalGeneration.from_pretrained(model_path) - self.assertEqual(model.dtype, torch.float32) - # 1. test torch_dtype="auto" via `config.torch_dtype` - model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") - self.assertEqual(model.dtype, torch.float32) - # 2. test torch_dtype="auto" via auto-derivation - # now remove the torch_dtype entry from config.json and try "auto" again which should - # perform auto-derivation from weights - remove_torch_dtype(model_path) - model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") - self.assertEqual(model.dtype, torch.float32) - - # test forced loading in fp16 (even though the weights are in fp32) - model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.float16) - self.assertEqual(model.dtype, torch.float16) - - # test fp16 save_pretrained, loaded with auto-detection - model = model.half() - model.save_pretrained(model_path) - # 1. test torch_dtype="auto" via `config.torch_dtype` - model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") - self.assertEqual(model.config.torch_dtype, torch.float16) - self.assertEqual(model.dtype, torch.float16) - # tests `config.torch_dtype` saving - with open(f"{model_path}/config.json") as f: - config_dict = json.load(f) - self.assertEqual(config_dict["torch_dtype"], "float16") - # 2. test torch_dtype="auto" via auto-derivation - # now same with using config info - remove_torch_dtype(model_path) - model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") - self.assertEqual(model.dtype, torch.float16) - - # 3. now retest that AutoModel behaves the same wrt torch_dtype="auto" as T5ForConditionalGeneration - model = AutoModel.from_pretrained(model_path, torch_dtype="auto") - self.assertEqual(model.dtype, torch.float16) - - # test fp16 save_pretrained, loaded with the explicit fp16 - model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.float16) - self.assertEqual(model.dtype, torch.float16) - - # test AutoModel separately as it goes through a different path - # test auto-detection - as currently TINY_T5 doesn't have torch_dtype entry - model = AutoModel.from_pretrained(TINY_T5, torch_dtype="auto") - # test that the config object didn't get polluted with torch_dtype="auto" - # there was a bug that after this call we ended up with config.torch_dtype=="auto" - self.assertNotEqual(model.config.torch_dtype, "auto") - # now test the outcome - self.assertEqual(model.dtype, torch.float32) - model = AutoModel.from_pretrained(TINY_T5, torch_dtype=torch.float16) - self.assertEqual(model.dtype, torch.float16) - - # test model whose first param is not of a floating type, but int - model = AutoModel.from_pretrained(TINY_BERT_FOR_TOKEN_CLASSIFICATION, torch_dtype="auto") - self.assertEqual(model.dtype, torch.float32) - - def test_no_super_init_config_and_model(self): - config = NoSuperInitConfig(attribute=32) - model = NoSuperInitModel(config) - - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir) - - new_model = NoSuperInitModel.from_pretrained(tmp_dir) - - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.equal(p1, p2)) - - def test_shard_checkpoint(self): - # This is the model we will use, total size 340,000 bytes. - model = torch.nn.Sequential( - torch.nn.Linear(100, 200, bias=False), # size 80,000 - torch.nn.Linear(200, 200, bias=False), # size 160,000 - torch.nn.Linear(200, 100, bias=False), # size 80,000 - torch.nn.Linear(100, 50, bias=False), # size 20,000 - ) - state_dict = model.state_dict() - - with self.subTest("No shard when max size is bigger than model size"): - shards, index = shard_checkpoint(state_dict) - self.assertIsNone(index) - self.assertDictEqual(shards, {WEIGHTS_NAME: state_dict}) - - with self.subTest("Test sharding, no weights bigger than max size"): - shards, index = shard_checkpoint(state_dict, max_shard_size="300kB") - # Split is first two layers then last two. - self.assertDictEqual( - index, - { - "metadata": {"total_size": 340000}, - "weight_map": { - "0.weight": "pytorch_model-00001-of-00002.bin", - "1.weight": "pytorch_model-00001-of-00002.bin", - "2.weight": "pytorch_model-00002-of-00002.bin", - "3.weight": "pytorch_model-00002-of-00002.bin", - }, - }, - ) - - shard1 = {"0.weight": state_dict["0.weight"], "1.weight": state_dict["1.weight"]} - shard2 = {"2.weight": state_dict["2.weight"], "3.weight": state_dict["3.weight"]} - self.assertDictEqual( - shards, {"pytorch_model-00001-of-00002.bin": shard1, "pytorch_model-00002-of-00002.bin": shard2} - ) - - with self.subTest("Test sharding with weights bigger than max size"): - shards, index = shard_checkpoint(state_dict, max_shard_size="100kB") - # Split is first layer, second layer then last 2. - self.assertDictEqual( - index, - { - "metadata": {"total_size": 340000}, - "weight_map": { - "0.weight": "pytorch_model-00001-of-00003.bin", - "1.weight": "pytorch_model-00002-of-00003.bin", - "2.weight": "pytorch_model-00003-of-00003.bin", - "3.weight": "pytorch_model-00003-of-00003.bin", - }, - }, - ) - - shard1 = {"0.weight": state_dict["0.weight"]} - shard2 = {"1.weight": state_dict["1.weight"]} - shard3 = {"2.weight": state_dict["2.weight"], "3.weight": state_dict["3.weight"]} - self.assertDictEqual( - shards, - { - "pytorch_model-00001-of-00003.bin": shard1, - "pytorch_model-00002-of-00003.bin": shard2, - "pytorch_model-00003-of-00003.bin": shard3, - }, - ) - - def test_checkpoint_sharding_local(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - with tempfile.TemporaryDirectory() as tmp_dir: - # We use the same folder for various sizes to make sure a new save erases the old checkpoint. - for max_size in ["50kB", "50kiB", "100kB", "100kiB", "200kB", "200kiB"]: - model.save_pretrained(tmp_dir, max_shard_size=max_size) - - # Get each shard file and its size - shard_to_size = {} - for shard in os.listdir(tmp_dir): - if shard.endswith(".bin"): - shard_file = os.path.join(tmp_dir, shard) - shard_to_size[shard_file] = os.path.getsize(shard_file) - - index_file = os.path.join(tmp_dir, WEIGHTS_INDEX_NAME) - # Check there is an index but no regular weight file - self.assertTrue(os.path.isfile(index_file)) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) - - # Check a file is bigger than max_size only when it has a single weight - for shard_file, size in shard_to_size.items(): - if max_size.endswith("kiB"): - max_size_int = int(max_size[:-3]) * 2**10 - else: - max_size_int = int(max_size[:-2]) * 10**3 - # Note: pickle adds some junk so the weight of the file can end up being slightly bigger than - # the size asked for (since we count parameters) - if size >= max_size_int + 50000: - state_dict = torch.load(shard_file) - self.assertEqual(len(state_dict), 1) - - # Check the index and the shard files found match - with open(index_file, "r", encoding="utf-8") as f: - index = json.loads(f.read()) - - all_shards = set(index["weight_map"].values()) - shards_found = {f for f in os.listdir(tmp_dir) if f.endswith(".bin")} - self.assertSetEqual(all_shards, shards_found) - - # Finally, check the model can be reloaded - new_model = BertModel.from_pretrained(tmp_dir) - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - def test_checkpoint_sharding_from_hub(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded") - # the model above is the same as the model below, just a sharded version. - ref_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - for p1, p2 in zip(model.parameters(), ref_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - def test_checkpoint_variant_local(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, variant="v2") - - weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["bin"]) - - weights_file = os.path.join(tmp_dir, weights_name) - self.assertTrue(os.path.isfile(weights_file)) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) - - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained(tmp_dir) - - new_model = BertModel.from_pretrained(tmp_dir, variant="v2") - - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - def test_checkpoint_variant_local_sharded(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, variant="v2", max_shard_size="50kB") - - weights_index_name = ".".join(WEIGHTS_INDEX_NAME.split(".")[:-1] + ["v2"] + ["json"]) - weights_index_file = os.path.join(tmp_dir, weights_index_name) - self.assertTrue(os.path.isfile(weights_index_file)) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_INDEX_NAME))) - - for i in range(1, 6): - weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + [f"v2-0000{i}-of-00006"] + ["bin"]) - weights_name_file = os.path.join(tmp_dir, weights_name) - self.assertTrue(os.path.isfile(weights_name_file)) - - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained(tmp_dir) - - new_model = BertModel.from_pretrained(tmp_dir, variant="v2") - - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - @require_safetensors - def test_checkpoint_variant_local_safe(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, variant="v2", safe_serialization=True) - - weights_name = ".".join(SAFE_WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["safetensors"]) - - weights_file = os.path.join(tmp_dir, weights_name) - self.assertTrue(os.path.isfile(weights_file)) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) - - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained(tmp_dir) - - new_model = BertModel.from_pretrained(tmp_dir, variant="v2") - - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - @require_safetensors - def test_checkpoint_variant_local_sharded_safe(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, variant="v2", max_shard_size="50kB", safe_serialization=True) - - weights_index_name = ".".join(SAFE_WEIGHTS_INDEX_NAME.split(".")[:-1] + ["v2"] + ["json"]) - weights_index_file = os.path.join(tmp_dir, weights_index_name) - self.assertTrue(os.path.isfile(weights_index_file)) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_INDEX_NAME))) - - for i in range(1, 6): - weights_name = ".".join(SAFE_WEIGHTS_NAME.split(".")[:-1] + [f"v2-0000{i}-of-00006"] + ["safetensors"]) - weights_name_file = os.path.join(tmp_dir, weights_name) - self.assertTrue(os.path.isfile(weights_name_file)) - - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained(tmp_dir) - - new_model = BertModel.from_pretrained(tmp_dir, variant="v2") - - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - def test_checkpoint_variant_hub(self): - with tempfile.TemporaryDirectory() as tmp_dir: - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-variant", cache_dir=tmp_dir) - model = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant", cache_dir=tmp_dir, variant="v2" - ) - self.assertIsNotNone(model) - - def test_checkpoint_variant_hub_sharded(self): - with tempfile.TemporaryDirectory() as tmp_dir: - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant-sharded", cache_dir=tmp_dir - ) - model = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant-sharded", cache_dir=tmp_dir, variant="v2" - ) - self.assertIsNotNone(model) - - @require_safetensors - def test_checkpoint_variant_hub_safe(self): - with tempfile.TemporaryDirectory() as tmp_dir: - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-variant-safe", cache_dir=tmp_dir) - model = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant-safe", cache_dir=tmp_dir, variant="v2" - ) - self.assertIsNotNone(model) - - @require_safetensors - def test_checkpoint_variant_hub_sharded_safe(self): - with tempfile.TemporaryDirectory() as tmp_dir: - with self.assertRaises(EnvironmentError): - _ = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant-sharded-safe", cache_dir=tmp_dir - ) - model = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant-sharded-safe", cache_dir=tmp_dir, variant="v2" - ) - self.assertIsNotNone(model) - - def test_checkpoint_variant_save_load(self): - with tempfile.TemporaryDirectory() as tmp_dir: - model = BertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert-variant", cache_dir=tmp_dir, variant="v2" - ) - weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["bin"]) - - model.save_pretrained(tmp_dir, variant="v2") - # saving will create a variant checkpoint - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, weights_name))) - - model.save_pretrained(tmp_dir) - # saving shouldn't delete variant checkpoints - weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["bin"]) - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, weights_name))) - - # there should be a normal checkpoint - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) - - self.assertIsNotNone(model) - - @require_accelerate - @mark.accelerate_tests - def test_from_pretrained_low_cpu_mem_usage_functional(self): - # test that we can use `from_pretrained(..., low_cpu_mem_usage=True)` with normal and - # sharded models - - mnames = [ - "hf-internal-testing/tiny-random-bert-sharded", - "hf-internal-testing/tiny-random-bert", - ] - for mname in mnames: - _ = BertModel.from_pretrained(mname, low_cpu_mem_usage=True) - - @require_usr_bin_time - @require_accelerate - @mark.accelerate_tests - def test_from_pretrained_low_cpu_mem_usage_measured(self): - # test that `from_pretrained(..., low_cpu_mem_usage=True)` uses less cpu memory than default - - mname = "bert-base-cased" - - preamble = "from transformers import AutoModel" - one_liner_str = f'{preamble}; AutoModel.from_pretrained("{mname}", low_cpu_mem_usage=False)' - max_rss_normal = self.python_one_liner_max_rss(one_liner_str) - # print(f"{max_rss_normal=}") - - one_liner_str = f'{preamble}; AutoModel.from_pretrained("{mname}", low_cpu_mem_usage=True)' - max_rss_low_mem = self.python_one_liner_max_rss(one_liner_str) - # print(f"{max_rss_low_mem=}") - - diff_bytes = max_rss_normal - max_rss_low_mem - diff_percent = diff_bytes / max_rss_low_mem - # print(f"{diff_bytes=}, {diff_percent=}") - # ideally we would compare that the diff is close to ~1x checkpoint size in bytes, but - # measuring cpu memory on linux is very tricky and inconsistent, so instead let's check that - # it's at least 15% less cpu memory consumed - - self.assertGreater( - diff_percent, - 0.15, - "should use less CPU memory for low_cpu_mem_usage=True, " - f"but got max_rss_normal={max_rss_normal} and max_rss_low_mem={max_rss_low_mem}", - ) - - # if you want to compare things manually, let's first look at the size of the model in bytes - # model = BertModel.from_pretrained(mname, low_cpu_mem_usage=False) - # total_numel = sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values()) - # total_bytes = total_numel * 4 # 420MB - # Now the diff_bytes should be very close to total_bytes, but the reports are inconsistent. - # The easiest way to test this is to switch the model and torch.load to do all the work on - # gpu - that way one can measure exactly the total and peak memory used. Perhaps once we add - # functionality to load models directly on gpu, this test can be rewritten to use torch's - # cuda memory tracking and then we should be able to do a much more precise test. - - @require_accelerate - @mark.accelerate_tests - @require_torch_multi_gpu - @slow - def test_model_parallelism_gpt2(self): - device_map = {"transformer.wte": 0, "transformer.wpe": 0, "lm_head": 0, "transformer.ln_f": 1} - for i in range(12): - device_map[f"transformer.h.{i}"] = 0 if i <= 5 else 1 - - model = AutoModelForCausalLM.from_pretrained("gpt2", device_map=device_map) - - tokenizer = AutoTokenizer.from_pretrained("gpt2") - inputs = tokenizer("Hello, my name is", return_tensors="pt") - output = model.generate(inputs["input_ids"].to(0)) - - text_output = tokenizer.decode(output[0].tolist()) - self.assertEqual(text_output, "Hello, my name is John. I'm a writer, and I'm a writer. I'm") - - @require_accelerate - @mark.accelerate_tests - @require_torch_gpu - def test_from_pretrained_disk_offload_task_model(self): - model = AutoModel.from_pretrained("hf-internal-testing/tiny-random-gpt2") - device_map = { - "transformer.wte": 0, - "transformer.wpe": 0, - "transformer.h.0": "cpu", - "transformer.h.1": "cpu", - "transformer.h.2": "cpu", - "transformer.h.3": "disk", - "transformer.h.4": "disk", - "transformer.ln_f": 0, - "lm_head": 0, - } - with tempfile.TemporaryDirectory() as tmp_dir: - inputs = torch.tensor([[1, 2, 3]]).to(0) - - model.save_pretrained(tmp_dir) - new_model = AutoModelForCausalLM.from_pretrained(tmp_dir).to(0) - outputs1 = new_model.to(0)(inputs) - - offload_folder = os.path.join(tmp_dir, "offload") - new_model_with_offload = AutoModelForCausalLM.from_pretrained( - tmp_dir, device_map=device_map, offload_folder=offload_folder - ) - outputs2 = new_model_with_offload(inputs) - - self.assertTrue(torch.allclose(outputs1.logits.cpu(), outputs2.logits.cpu())) - - # With state dict temp offload - offload_folder = os.path.join(tmp_dir, "offload") - new_model_with_offload = AutoModelForCausalLM.from_pretrained( - tmp_dir, - device_map=device_map, - offload_folder=offload_folder, - offload_state_dict=True, - ) - outputs2 = new_model_with_offload(inputs) - - self.assertTrue(torch.allclose(outputs1.logits.cpu(), outputs2.logits.cpu())) - - def test_cached_files_are_used_when_internet_is_down(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Under the mock environment we get a 500 error when trying to reach the model. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - # This check we did call the fake head request - mock_head.assert_called() - - def test_load_from_one_file(self): - try: - tmp_file = tempfile.mktemp() - with open(tmp_file, "wb") as f: - http_get( - "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", f - ) - - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - _ = BertModel.from_pretrained(tmp_file, config=config) - finally: - os.remove(tmp_file) - - def test_legacy_load_from_url(self): - # This test is for deprecated behavior and can be removed in v5 - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - _ = BertModel.from_pretrained( - "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", config=config - ) - - @require_safetensors - def test_use_safetensors(self): - # test nice error message if no safetensor files available - with self.assertRaises(OSError) as env_error: - AutoModel.from_pretrained("hf-internal-testing/tiny-random-RobertaModel", use_safetensors=True) - - self.assertTrue( - "model.safetensors or model.safetensors.index.json and thus cannot be loaded with `safetensors`" - in str(env_error.exception) - ) - - # test that error if only safetensors is available - with self.assertRaises(OSError) as env_error: - BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors", use_safetensors=False) - - self.assertTrue("does not appear to have a file named pytorch_model.bin" in str(env_error.exception)) - - # test that only safetensors if both available and use_safetensors=False - with tempfile.TemporaryDirectory() as tmp_dir: - CLIPTextModel.from_pretrained( - "hf-internal-testing/diffusers-stable-diffusion-tiny-all", - subfolder="text_encoder", - use_safetensors=False, - cache_dir=tmp_dir, - ) - - all_downloaded_files = glob.glob(os.path.join(tmp_dir, "*", "snapshots", "*", "*", "*")) - self.assertTrue(any(f.endswith("bin") for f in all_downloaded_files)) - self.assertFalse(any(f.endswith("safetensors") for f in all_downloaded_files)) - - # test that no safetensors if both available and use_safetensors=True - with tempfile.TemporaryDirectory() as tmp_dir: - CLIPTextModel.from_pretrained( - "hf-internal-testing/diffusers-stable-diffusion-tiny-all", - subfolder="text_encoder", - use_safetensors=True, - cache_dir=tmp_dir, - ) - - all_downloaded_files = glob.glob(os.path.join(tmp_dir, "*", "snapshots", "*", "*", "*")) - self.assertTrue(any(f.endswith("safetensors") for f in all_downloaded_files)) - self.assertFalse(any(f.endswith("bin") for f in all_downloaded_files)) - - @require_safetensors - def test_safetensors_save_and_load(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, safe_serialization=True) - # No pytorch_model.bin file, only a model.safetensors - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) - - new_model = BertModel.from_pretrained(tmp_dir) - - # Check models are equal - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - @require_safetensors - def test_safetensors_load_from_hub(self): - safetensors_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors") - pytorch_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Check models are equal - for p1, p2 in zip(safetensors_model.parameters(), pytorch_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - @require_safetensors - def test_safetensors_save_and_load_sharded(self): - model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, safe_serialization=True, max_shard_size="100kB") - # No pytorch_model.bin index file, only a model.safetensors index - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_INDEX_NAME))) - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_INDEX_NAME))) - # No regular weights file - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) - - new_model = BertModel.from_pretrained(tmp_dir) - - # Check models are equal - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - @require_safetensors - def test_safetensors_load_from_hub_sharded(self): - safetensors_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded-safetensors") - pytorch_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded") - - # Check models are equal - for p1, p2 in zip(safetensors_model.parameters(), pytorch_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - def test_base_model_to_head_model_load(self): - base_model = BaseModel(PretrainedConfig()) - with tempfile.TemporaryDirectory() as tmp_dir: - base_model.save_pretrained(tmp_dir) - - # Can load a base model in a model with head - model = ModelWithHead.from_pretrained(tmp_dir) - for p1, p2 in zip(model.base.parameters(), base_model.parameters()): - self.assertTrue(torch.allclose(p1, p2)) - - # It doesn't work if the state dict has a mix of keys of the head and base without prefix though. - base_state_dict = base_model.state_dict() - head_state_dict = model.state_dict() - base_state_dict["linear2.weight"] = head_state_dict["linear2.weight"] - base_state_dict["linear2.bias"] = head_state_dict["linear2.bias"] - torch.save(base_state_dict, os.path.join(tmp_dir, WEIGHTS_NAME)) - - with self.assertRaisesRegex( - ValueError, "The state dictionary of the model you are trying to load is corrupted." - ): - _ = ModelWithHead.from_pretrained(tmp_dir) - - @require_torch_gpu - @slow - def test_pretrained_low_mem_new_config(self): - # Checking for 1 model(the same one which was described in the issue) . - model_ids = ["gpt2"] - - for model_id in model_ids: - model_config = AutoConfig.from_pretrained(pretrained_model_name_or_path=model_id) - model_config.n_layer = 48 - model_config.n_head = 25 - model_config.n_embd = 1600 - model = AutoModelForCausalLM.from_pretrained( - pretrained_model_name_or_path=model_id, - config=model_config, - ignore_mismatched_sizes=True, - torch_dtype=torch.float16, - low_cpu_mem_usage=True, - ) - model_ref = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path=model_id) - - self.assertEqual(model.__class__.__name__, model_ref.__class__.__name__) - - -@require_torch -@is_staging_test -class ModelPushToHubTester(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-model") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-model-org") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="test-dynamic-model") - except HTTPError: - pass - - def test_push_to_hub(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = BertModel(config) - model.push_to_hub("test-model", use_auth_token=self._token) - - new_model = BertModel.from_pretrained(f"{USER}/test-model") - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.equal(p1, p2)) - - # Reset repo - delete_repo(token=self._token, repo_id="test-model") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, repo_id="test-model", push_to_hub=True, use_auth_token=self._token) - - new_model = BertModel.from_pretrained(f"{USER}/test-model") - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.equal(p1, p2)) - - def test_push_to_hub_in_organization(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = BertModel(config) - model.push_to_hub("valid_org/test-model-org", use_auth_token=self._token) - - new_model = BertModel.from_pretrained("valid_org/test-model-org") - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.equal(p1, p2)) - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-model-org") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained( - tmp_dir, push_to_hub=True, use_auth_token=self._token, repo_id="valid_org/test-model-org" - ) - - new_model = BertModel.from_pretrained("valid_org/test-model-org") - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.equal(p1, p2)) - - def test_push_to_hub_dynamic_model(self): - CustomConfig.register_for_auto_class() - CustomModel.register_for_auto_class() - - config = CustomConfig(hidden_size=32) - model = CustomModel(config) - - model.push_to_hub("test-dynamic-model", use_auth_token=self._token) - # checks - self.assertDictEqual( - config.auto_map, - {"AutoConfig": "custom_configuration.CustomConfig", "AutoModel": "custom_modeling.CustomModel"}, - ) - - new_model = AutoModel.from_pretrained(f"{USER}/test-dynamic-model", trust_remote_code=True) - # Can't make an isinstance check because the new_model is from the CustomModel class of a dynamic module - self.assertEqual(new_model.__class__.__name__, "CustomModel") - for p1, p2 in zip(model.parameters(), new_model.parameters()): - self.assertTrue(torch.equal(p1, p2)) - - config = AutoConfig.from_pretrained(f"{USER}/test-dynamic-model", trust_remote_code=True) - new_model = AutoModel.from_config(config, trust_remote_code=True) - self.assertEqual(new_model.__class__.__name__, "CustomModel") diff --git a/tests/test_modeling_flax_common.py b/tests/test_modeling_flax_common.py index 058d7906113dcd..58ada0226a51bc 100644 --- a/tests/test_modeling_flax_common.py +++ b/tests/test_modeling_flax_common.py @@ -17,25 +17,14 @@ import json import random import tempfile -import unittest from typing import List, Tuple import numpy as np -from huggingface_hub import HfFolder, delete_repo -from requests.exceptions import HTTPError import transformers -from transformers import BertConfig, is_flax_available, is_torch_available +from transformers import is_flax_available, is_torch_available from transformers.models.auto import get_values -from transformers.testing_utils import ( - TOKEN, - USER, - CaptureLogger, - is_pt_flax_cross_test, - is_staging_test, - require_flax, - torch_device, -) +from transformers.testing_utils import CaptureLogger, is_pt_flax_cross_test, require_flax, torch_device from transformers.utils import CONFIG_NAME, GENERATION_CONFIG_NAME, logging from transformers.utils.generic import ModelOutput @@ -69,14 +58,6 @@ import torch -def _config_zero_init(config): - configs_no_init = copy.deepcopy(config) - for key in configs_no_init.__dict__.keys(): - if "_range" in key or "_std" in key or "initializer_factor" in key: - setattr(configs_no_init, key, 1e-10) - return configs_no_init - - def ids_tensor(shape, vocab_size, rng=None): """Creates a random int32 tensor of the shape within the vocab size.""" if rng is None: @@ -1164,155 +1145,3 @@ def test_gradient_checkpointing(self): # ensure that the outputs remain precisely equal for output, remat_output in zip(outputs, remat_outputs): self.assertTrue((output == remat_output).all()) - - -@require_flax -@is_staging_test -class FlaxModelPushToHubTester(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-model-flax") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-model-flax-org") - except HTTPError: - pass - - def test_push_to_hub(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = FlaxBertModel(config) - model.push_to_hub("test-model-flax", use_auth_token=self._token) - - new_model = FlaxBertModel.from_pretrained(f"{USER}/test-model-flax") - - base_params = flatten_dict(unfreeze(model.params)) - new_params = flatten_dict(unfreeze(new_model.params)) - - for key in base_params.keys(): - max_diff = (base_params[key] - new_params[key]).sum().item() - self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - - # Reset repo - delete_repo(token=self._token, repo_id="test-model-flax") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, repo_id="test-model-flax", push_to_hub=True, use_auth_token=self._token) - - new_model = FlaxBertModel.from_pretrained(f"{USER}/test-model-flax") - - base_params = flatten_dict(unfreeze(model.params)) - new_params = flatten_dict(unfreeze(new_model.params)) - - for key in base_params.keys(): - max_diff = (base_params[key] - new_params[key]).sum().item() - self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - - def test_push_to_hub_in_organization(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = FlaxBertModel(config) - model.push_to_hub("valid_org/test-model-flax-org", use_auth_token=self._token) - - new_model = FlaxBertModel.from_pretrained("valid_org/test-model-flax-org") - - base_params = flatten_dict(unfreeze(model.params)) - new_params = flatten_dict(unfreeze(new_model.params)) - - for key in base_params.keys(): - max_diff = (base_params[key] - new_params[key]).sum().item() - self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-model-flax-org") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained( - tmp_dir, repo_id="valid_org/test-model-flax-org", push_to_hub=True, use_auth_token=self._token - ) - - new_model = FlaxBertModel.from_pretrained("valid_org/test-model-flax-org") - - base_params = flatten_dict(unfreeze(model.params)) - new_params = flatten_dict(unfreeze(new_model.params)) - - for key in base_params.keys(): - max_diff = (base_params[key] - new_params[key]).sum().item() - self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - - -def check_models_equal(model1, model2): - models_are_equal = True - flat_params_1 = flatten_dict(model1.params) - flat_params_2 = flatten_dict(model2.params) - for key in flat_params_1.keys(): - if np.sum(np.abs(flat_params_1[key] - flat_params_2[key])) > 1e-4: - models_are_equal = False - - return models_are_equal - - -@require_flax -class FlaxModelUtilsTest(unittest.TestCase): - def test_model_from_pretrained_subfolder(self): - config = BertConfig.from_pretrained("hf-internal-testing/tiny-bert-flax-only") - model = FlaxBertModel(config) - - subfolder = "bert" - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(os.path.join(tmp_dir, subfolder)) - - with self.assertRaises(OSError): - _ = FlaxBertModel.from_pretrained(tmp_dir) - - model_loaded = FlaxBertModel.from_pretrained(tmp_dir, subfolder=subfolder) - - self.assertTrue(check_models_equal(model, model_loaded)) - - def test_model_from_pretrained_subfolder_sharded(self): - config = BertConfig.from_pretrained("hf-internal-testing/tiny-bert-flax-only") - model = FlaxBertModel(config) - - subfolder = "bert" - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(os.path.join(tmp_dir, subfolder), max_shard_size="10KB") - - with self.assertRaises(OSError): - _ = FlaxBertModel.from_pretrained(tmp_dir) - - model_loaded = FlaxBertModel.from_pretrained(tmp_dir, subfolder=subfolder) - - self.assertTrue(check_models_equal(model, model_loaded)) - - def test_model_from_pretrained_hub_subfolder(self): - subfolder = "bert" - model_id = "hf-internal-testing/tiny-random-bert-subfolder" - - with self.assertRaises(OSError): - _ = FlaxBertModel.from_pretrained(model_id) - - model = FlaxBertModel.from_pretrained(model_id, subfolder=subfolder) - - self.assertIsNotNone(model) - - def test_model_from_pretrained_hub_subfolder_sharded(self): - subfolder = "bert" - model_id = "hf-internal-testing/tiny-random-bert-sharded-subfolder" - with self.assertRaises(OSError): - _ = FlaxBertModel.from_pretrained(model_id) - - model = FlaxBertModel.from_pretrained(model_id, subfolder=subfolder) - - self.assertIsNotNone(model) diff --git a/tests/test_modeling_flax_utils.py b/tests/test_modeling_flax_utils.py new file mode 100644 index 00000000000000..d8fb71a6104c20 --- /dev/null +++ b/tests/test_modeling_flax_utils.py @@ -0,0 +1,186 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +import numpy as np +from huggingface_hub import HfFolder, delete_repo +from requests.exceptions import HTTPError + +from transformers import BertConfig, is_flax_available +from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax + + +if is_flax_available(): + import os + + from flax.core.frozen_dict import unfreeze + from flax.traverse_util import flatten_dict + + from transformers import FlaxBertModel + + os.environ["XLA_PYTHON_CLIENT_MEM_FRACTION"] = "0.12" # assumed parallelism: 8 + + +@require_flax +@is_staging_test +class FlaxModelPushToHubTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-model-flax") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-model-flax-org") + except HTTPError: + pass + + def test_push_to_hub(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = FlaxBertModel(config) + model.push_to_hub("test-model-flax", use_auth_token=self._token) + + new_model = FlaxBertModel.from_pretrained(f"{USER}/test-model-flax") + + base_params = flatten_dict(unfreeze(model.params)) + new_params = flatten_dict(unfreeze(new_model.params)) + + for key in base_params.keys(): + max_diff = (base_params[key] - new_params[key]).sum().item() + self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") + + # Reset repo + delete_repo(token=self._token, repo_id="test-model-flax") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, repo_id="test-model-flax", push_to_hub=True, use_auth_token=self._token) + + new_model = FlaxBertModel.from_pretrained(f"{USER}/test-model-flax") + + base_params = flatten_dict(unfreeze(model.params)) + new_params = flatten_dict(unfreeze(new_model.params)) + + for key in base_params.keys(): + max_diff = (base_params[key] - new_params[key]).sum().item() + self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") + + def test_push_to_hub_in_organization(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = FlaxBertModel(config) + model.push_to_hub("valid_org/test-model-flax-org", use_auth_token=self._token) + + new_model = FlaxBertModel.from_pretrained("valid_org/test-model-flax-org") + + base_params = flatten_dict(unfreeze(model.params)) + new_params = flatten_dict(unfreeze(new_model.params)) + + for key in base_params.keys(): + max_diff = (base_params[key] - new_params[key]).sum().item() + self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-model-flax-org") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained( + tmp_dir, repo_id="valid_org/test-model-flax-org", push_to_hub=True, use_auth_token=self._token + ) + + new_model = FlaxBertModel.from_pretrained("valid_org/test-model-flax-org") + + base_params = flatten_dict(unfreeze(model.params)) + new_params = flatten_dict(unfreeze(new_model.params)) + + for key in base_params.keys(): + max_diff = (base_params[key] - new_params[key]).sum().item() + self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") + + +def check_models_equal(model1, model2): + models_are_equal = True + flat_params_1 = flatten_dict(model1.params) + flat_params_2 = flatten_dict(model2.params) + for key in flat_params_1.keys(): + if np.sum(np.abs(flat_params_1[key] - flat_params_2[key])) > 1e-4: + models_are_equal = False + + return models_are_equal + + +@require_flax +class FlaxModelUtilsTest(unittest.TestCase): + def test_model_from_pretrained_subfolder(self): + config = BertConfig.from_pretrained("hf-internal-testing/tiny-bert-flax-only") + model = FlaxBertModel(config) + + subfolder = "bert" + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(os.path.join(tmp_dir, subfolder)) + + with self.assertRaises(OSError): + _ = FlaxBertModel.from_pretrained(tmp_dir) + + model_loaded = FlaxBertModel.from_pretrained(tmp_dir, subfolder=subfolder) + + self.assertTrue(check_models_equal(model, model_loaded)) + + def test_model_from_pretrained_subfolder_sharded(self): + config = BertConfig.from_pretrained("hf-internal-testing/tiny-bert-flax-only") + model = FlaxBertModel(config) + + subfolder = "bert" + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(os.path.join(tmp_dir, subfolder), max_shard_size="10KB") + + with self.assertRaises(OSError): + _ = FlaxBertModel.from_pretrained(tmp_dir) + + model_loaded = FlaxBertModel.from_pretrained(tmp_dir, subfolder=subfolder) + + self.assertTrue(check_models_equal(model, model_loaded)) + + def test_model_from_pretrained_hub_subfolder(self): + subfolder = "bert" + model_id = "hf-internal-testing/tiny-random-bert-subfolder" + + with self.assertRaises(OSError): + _ = FlaxBertModel.from_pretrained(model_id) + + model = FlaxBertModel.from_pretrained(model_id, subfolder=subfolder) + + self.assertIsNotNone(model) + + def test_model_from_pretrained_hub_subfolder_sharded(self): + subfolder = "bert" + model_id = "hf-internal-testing/tiny-random-bert-sharded-subfolder" + with self.assertRaises(OSError): + _ = FlaxBertModel.from_pretrained(model_id) + + model = FlaxBertModel.from_pretrained(model_id, subfolder=subfolder) + + self.assertIsNotNone(model) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 9ebbe86bf03189..a4de6e8f471e3e 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -23,42 +23,24 @@ import random import tempfile import unittest -import unittest.mock as mock from importlib import import_module from math import isnan from typing import List, Tuple from datasets import Dataset -from huggingface_hub import HfFolder, Repository, delete_repo -from huggingface_hub.file_download import http_get -from requests.exceptions import HTTPError from transformers import is_tf_available, is_torch_available -from transformers.configuration_utils import PretrainedConfig from transformers.models.auto import get_values from transformers.testing_utils import ( # noqa: F401 - TOKEN, - USER, CaptureLogger, - CaptureStdout, _tf_gpu_memory_limit, is_pt_tf_cross_test, - is_staging_test, - require_safetensors, require_tf, require_tf2onnx, slow, - tooslow, torch_device, ) -from transformers.utils import ( - CONFIG_NAME, - GENERATION_CONFIG_NAME, - SAFE_WEIGHTS_NAME, - TF2_WEIGHTS_INDEX_NAME, - TF2_WEIGHTS_NAME, - logging, -) +from transformers.utils import CONFIG_NAME, GENERATION_CONFIG_NAME, logging from transformers.utils.generic import ModelOutput @@ -66,7 +48,6 @@ if is_tf_available(): - import h5py import numpy as np import tensorflow as tf @@ -85,17 +66,8 @@ TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING, TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, - BertConfig, - PreTrainedModel, - PushToHubCallback, - RagRetriever, TFAutoModel, TFAutoModelForSequenceClassification, - TFBertForMaskedLM, - TFBertForSequenceClassification, - TFBertModel, - TFPreTrainedModel, - TFRagModel, TFSharedEmbeddings, ) from transformers.generation import ( @@ -108,8 +80,6 @@ TFSampleDecoderOnlyOutput, TFSampleEncoderDecoderOutput, ) - from transformers.modeling_tf_utils import tf_shard_checkpoint, unpack_inputs - from transformers.tf_utils import stable_softmax tf.config.experimental.enable_tensor_float_32_execution(False) @@ -130,8 +100,6 @@ if is_torch_available(): import torch - from transformers import BertModel - def _config_zero_init(config): configs_no_init = copy.deepcopy(config) @@ -1995,544 +1963,3 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None, dtype=None): values.append(rng.random() * scale) return tf.reshape(tf.constant(values, dtype=dtype if dtype is not None else tf.float32), shape=shape) - - -@require_tf -class UtilsFunctionsTest(unittest.TestCase): - def test_cached_files_are_used_when_internet_is_down(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Under the mock environment we get a 500 error when trying to reach the model. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - # This check we did call the fake head request - mock_head.assert_called() - - def test_load_from_one_file(self): - try: - tmp_file = tempfile.mktemp() - with open(tmp_file, "wb") as f: - http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/tf_model.h5", f) - - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - _ = TFBertModel.from_pretrained(tmp_file, config=config) - finally: - os.remove(tmp_file) - - def test_legacy_load_from_url(self): - # This test is for deprecated behavior and can be removed in v5 - config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") - _ = TFBertModel.from_pretrained( - "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/tf_model.h5", config=config - ) - - # tests whether the unpack_inputs function behaves as expected - def test_unpack_inputs(self): - class DummyModel: - def __init__(self): - config_kwargs = {"output_attentions": False, "output_hidden_states": False, "return_dict": False} - self.config = PretrainedConfig(**config_kwargs) - self.main_input_name = "input_ids" - - @unpack_inputs - def call( - self, - input_ids=None, - past_key_values=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): - return input_ids, past_key_values, output_attentions, output_hidden_states, return_dict - - @unpack_inputs - def foo(self, pixel_values, output_attentions=None, output_hidden_states=None, return_dict=None): - return pixel_values, output_attentions, output_hidden_states, return_dict - - dummy_model = DummyModel() - input_ids = tf.constant([0, 1, 2, 3], dtype=tf.int32) - past_key_values = tf.constant([4, 5, 6, 7], dtype=tf.int32) - pixel_values = tf.constant([8, 9, 10, 11], dtype=tf.int32) - - # test case 1: Pass inputs as keyword arguments; Booleans are inherited from the config. - output = dummy_model.call(input_ids=input_ids, past_key_values=past_key_values) - tf.debugging.assert_equal(output[0], input_ids) - tf.debugging.assert_equal(output[1], past_key_values) - self.assertFalse(output[2]) - self.assertFalse(output[3]) - self.assertFalse(output[4]) - - # test case 2: Same as above, but with positional arguments. - output = dummy_model.call(input_ids, past_key_values) - tf.debugging.assert_equal(output[0], input_ids) - tf.debugging.assert_equal(output[1], past_key_values) - self.assertFalse(output[2]) - self.assertFalse(output[3]) - self.assertFalse(output[4]) - - # test case 3: We can also pack everything in the first input. - output = dummy_model.call(input_ids={"input_ids": input_ids, "past_key_values": past_key_values}) - tf.debugging.assert_equal(output[0], input_ids) - tf.debugging.assert_equal(output[1], past_key_values) - self.assertFalse(output[2]) - self.assertFalse(output[3]) - self.assertFalse(output[4]) - - # test case 4: Explicit boolean arguments should override the config. - output = dummy_model.call( - input_ids=input_ids, past_key_values=past_key_values, output_attentions=False, return_dict=True - ) - tf.debugging.assert_equal(output[0], input_ids) - tf.debugging.assert_equal(output[1], past_key_values) - self.assertFalse(output[2]) - self.assertFalse(output[3]) - self.assertTrue(output[4]) - - # test case 5: Unexpected arguments should raise an exception. - with self.assertRaises(ValueError): - output = dummy_model.call(input_ids=input_ids, past_key_values=past_key_values, foo="bar") - - # test case 6: the decorator is independent from `main_input_name` -- it treats the first argument of the - # decorated function as its main input. - output = dummy_model.foo(pixel_values=pixel_values) - tf.debugging.assert_equal(output[0], pixel_values) - self.assertFalse(output[1]) - self.assertFalse(output[2]) - self.assertFalse(output[3]) - - # Tests whether the stable softmax is stable on CPU, with and without XLA - def test_xla_stable_softmax(self): - large_penalty = -1e9 - n_tokens = 10 - batch_size = 8 - - def masked_softmax(x, boolean_mask): - numerical_mask = (1.0 - tf.cast(boolean_mask, dtype=tf.float32)) * large_penalty - masked_x = x + numerical_mask - return stable_softmax(masked_x) - - xla_masked_softmax = tf.function(masked_softmax, jit_compile=True) - xla_stable_softmax = tf.function(stable_softmax, jit_compile=True) - x = tf.random.normal((batch_size, n_tokens)) - - # Same outcome regardless of the boolean mask here - masked_tokens = random.randint(0, n_tokens) - boolean_mask = tf.convert_to_tensor([[1] * (n_tokens - masked_tokens) + [0] * masked_tokens], dtype=tf.int32) - - # We can randomly mask a random numerical input OUTSIDE XLA - numerical_mask = (1.0 - tf.cast(boolean_mask, dtype=tf.float32)) * large_penalty - masked_x = x + numerical_mask - xla_out = xla_stable_softmax(masked_x) - out = stable_softmax(masked_x) - assert tf.experimental.numpy.allclose(xla_out, out) - - # The stable softmax has the same output as the original softmax - unstable_out = tf.nn.softmax(masked_x) - assert tf.experimental.numpy.allclose(unstable_out, out) - - # We can randomly mask a random numerical input INSIDE XLA - xla_out = xla_masked_softmax(x, boolean_mask) - out = masked_softmax(x, boolean_mask) - assert tf.experimental.numpy.allclose(xla_out, out) - - def test_checkpoint_sharding_from_hub(self): - model = TFBertModel.from_pretrained("ArthurZ/tiny-random-bert-sharded") - # the model above is the same as the model below, just a sharded version. - ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - for p1, p2 in zip(model.weights, ref_model.weights): - assert np.allclose(p1.numpy(), p2.numpy()) - - def test_sharded_checkpoint_with_prefix(self): - model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert", load_weight_prefix="a/b") - sharded_model = TFBertModel.from_pretrained("ArthurZ/tiny-random-bert-sharded", load_weight_prefix="a/b") - for p1, p2 in zip(model.weights, sharded_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - self.assertTrue(p1.name.startswith("a/b/")) - self.assertTrue(p2.name.startswith("a/b/")) - - def test_sharded_checkpoint_transfer(self): - # If this doesn't throw an error then the test passes - TFBertForSequenceClassification.from_pretrained("ArthurZ/tiny-random-bert-sharded") - - @is_pt_tf_cross_test - def test_checkpoint_sharding_local_from_pt(self): - with tempfile.TemporaryDirectory() as tmp_dir: - _ = Repository(local_dir=tmp_dir, clone_from="hf-internal-testing/tiny-random-bert-sharded") - model = TFBertModel.from_pretrained(tmp_dir, from_pt=True) - # the model above is the same as the model below, just a sharded pytorch version. - ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - for p1, p2 in zip(model.weights, ref_model.weights): - assert np.allclose(p1.numpy(), p2.numpy()) - - @is_pt_tf_cross_test - def test_checkpoint_loading_with_prefix_from_pt(self): - model = TFBertModel.from_pretrained( - "hf-internal-testing/tiny-random-bert", from_pt=True, load_weight_prefix="a/b" - ) - ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert", from_pt=True) - for p1, p2 in zip(model.weights, ref_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - self.assertTrue(p1.name.startswith("a/b/")) - - @is_pt_tf_cross_test - def test_checkpoint_sharding_hub_from_pt(self): - model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded", from_pt=True) - # the model above is the same as the model below, just a sharded pytorch version. - ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - for p1, p2 in zip(model.weights, ref_model.weights): - assert np.allclose(p1.numpy(), p2.numpy()) - - def test_shard_checkpoint(self): - # This is the model we will use, total size 340,000 bytes. - model = tf.keras.Sequential( - [ - tf.keras.layers.Dense(200, use_bias=False), # size 80,000 - tf.keras.layers.Dense(200, use_bias=False), # size 160,000 - tf.keras.layers.Dense(100, use_bias=False), # size 80,000 - tf.keras.layers.Dense(50, use_bias=False), # size 20,000 - ] - ) - inputs = tf.zeros((1, 100), dtype=tf.float32) - model(inputs) - weights = model.weights - weights_dict = {w.name: w for w in weights} - with self.subTest("No shard when max size is bigger than model size"): - shards, index = tf_shard_checkpoint(weights) - self.assertIsNone(index) - self.assertDictEqual(shards, {TF2_WEIGHTS_NAME: weights}) - - with self.subTest("Test sharding, no weights bigger than max size"): - shards, index = tf_shard_checkpoint(weights, max_shard_size="300kB") - # Split is first two layers then last two. - self.assertDictEqual( - index, - { - "metadata": {"total_size": 340000}, - "weight_map": { - "dense/kernel:0": "tf_model-00001-of-00002.h5", - "dense_1/kernel:0": "tf_model-00001-of-00002.h5", - "dense_2/kernel:0": "tf_model-00002-of-00002.h5", - "dense_3/kernel:0": "tf_model-00002-of-00002.h5", - }, - }, - ) - - shard1 = [weights_dict["dense/kernel:0"], weights_dict["dense_1/kernel:0"]] - shard2 = [weights_dict["dense_2/kernel:0"], weights_dict["dense_3/kernel:0"]] - self.assertDictEqual(shards, {"tf_model-00001-of-00002.h5": shard1, "tf_model-00002-of-00002.h5": shard2}) - - with self.subTest("Test sharding with weights bigger than max size"): - shards, index = tf_shard_checkpoint(weights, max_shard_size="100kB") - # Split is first layer, second layer then last 2. - self.assertDictEqual( - index, - { - "metadata": {"total_size": 340000}, - "weight_map": { - "dense/kernel:0": "tf_model-00001-of-00003.h5", - "dense_1/kernel:0": "tf_model-00002-of-00003.h5", - "dense_2/kernel:0": "tf_model-00003-of-00003.h5", - "dense_3/kernel:0": "tf_model-00003-of-00003.h5", - }, - }, - ) - - shard1 = [weights_dict["dense/kernel:0"]] - shard2 = [weights_dict["dense_1/kernel:0"]] - shard3 = [weights_dict["dense_2/kernel:0"], weights_dict["dense_3/kernel:0"]] - self.assertDictEqual( - shards, - { - "tf_model-00001-of-00003.h5": shard1, - "tf_model-00002-of-00003.h5": shard2, - "tf_model-00003-of-00003.h5": shard3, - }, - ) - - @slow - def test_special_layer_name_sharding(self): - retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True) - model = TFRagModel.from_pretrained("facebook/rag-token-nq", retriever=retriever) - - with tempfile.TemporaryDirectory() as tmp_dir: - for max_size in ["150kB", "150kiB", "200kB", "200kiB"]: - model.save_pretrained(tmp_dir, max_shard_size=max_size) - ref_model = TFRagModel.from_pretrained(tmp_dir, retriever=retriever) - for p1, p2 in zip(model.weights, ref_model.weights): - assert np.allclose(p1.numpy(), p2.numpy()) - - def test_checkpoint_sharding_local(self): - model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - with tempfile.TemporaryDirectory() as tmp_dir: - # We use the same folder for various sizes to make sure a new save erases the old checkpoint. - for max_size in ["150kB", "150kiB", "200kB", "200kiB"]: - model.save_pretrained(tmp_dir, max_shard_size=max_size) - - # Get each shard file and its size - shard_to_size = {} - for shard in os.listdir(tmp_dir): - if shard.endswith(".h5"): - shard_file = os.path.join(tmp_dir, shard) - shard_to_size[shard_file] = os.path.getsize(shard_file) - - index_file = os.path.join(tmp_dir, TF2_WEIGHTS_INDEX_NAME) - # Check there is an index but no regular weight file - self.assertTrue(os.path.isfile(index_file)) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, TF2_WEIGHTS_NAME))) - - # Check a file is bigger than max_size only when it has a single weight - for shard_file, size in shard_to_size.items(): - if max_size.endswith("kiB"): - max_size_int = int(max_size[:-3]) * 2**10 - else: - max_size_int = int(max_size[:-2]) * 10**3 - # Note: pickle adds some junk so the weight of the file can end up being slightly bigger than - # the size asked for (since we count parameters) - if size >= max_size_int + 50000: - with h5py.File(shard_file, "r") as state_file: - self.assertEqual(len(state_file), 1) - - # Check the index and the shard files found match - with open(index_file, "r", encoding="utf-8") as f: - index = json.loads(f.read()) - - all_shards = set(index["weight_map"].values()) - shards_found = {f for f in os.listdir(tmp_dir) if f.endswith(".h5")} - self.assertSetEqual(all_shards, shards_found) - - # Finally, check the model can be reloaded - new_model = TFBertModel.from_pretrained(tmp_dir) - - model.build() - new_model.build() - - for p1, p2 in zip(model.weights, new_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - - @slow - def test_save_pretrained_signatures(self): - model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Short custom TF signature function. - # `input_signature` is specific to BERT. - @tf.function( - input_signature=[ - [ - tf.TensorSpec([None, None], tf.int32, name="input_ids"), - tf.TensorSpec([None, None], tf.int32, name="token_type_ids"), - tf.TensorSpec([None, None], tf.int32, name="attention_mask"), - ] - ] - ) - def serving_fn(input): - return model(input) - - # Using default signature (default behavior) overrides 'serving_default' - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, saved_model=True, signatures=None) - model_loaded = tf.keras.models.load_model(f"{tmp_dir}/saved_model/1") - self.assertTrue("serving_default" in list(model_loaded.signatures.keys())) - - # Providing custom signature function - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, saved_model=True, signatures={"custom_signature": serving_fn}) - model_loaded = tf.keras.models.load_model(f"{tmp_dir}/saved_model/1") - self.assertTrue("custom_signature" in list(model_loaded.signatures.keys())) - - # Providing multiple custom signature function - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained( - tmp_dir, - saved_model=True, - signatures={"custom_signature_1": serving_fn, "custom_signature_2": serving_fn}, - ) - model_loaded = tf.keras.models.load_model(f"{tmp_dir}/saved_model/1") - self.assertTrue("custom_signature_1" in list(model_loaded.signatures.keys())) - self.assertTrue("custom_signature_2" in list(model_loaded.signatures.keys())) - - @require_safetensors - def test_safetensors_save_and_load(self): - model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, safe_serialization=True) - # No tf_model.h5 file, only a model.safetensors - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) - self.assertFalse(os.path.isfile(os.path.join(tmp_dir, TF2_WEIGHTS_NAME))) - - new_model = TFBertModel.from_pretrained(tmp_dir) - - # Check models are equal - for p1, p2 in zip(model.weights, new_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - - @is_pt_tf_cross_test - def test_safetensors_save_and_load_pt_to_tf(self): - model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - pt_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - with tempfile.TemporaryDirectory() as tmp_dir: - pt_model.save_pretrained(tmp_dir, safe_serialization=True) - # Check we have a model.safetensors file - self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) - - new_model = TFBertModel.from_pretrained(tmp_dir) - - # Check models are equal - for p1, p2 in zip(model.weights, new_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - - @require_safetensors - def test_safetensors_load_from_hub(self): - tf_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Can load from the TF-formatted checkpoint - safetensors_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors-tf") - - # Check models are equal - for p1, p2 in zip(safetensors_model.weights, tf_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - - # Can load from the PyTorch-formatted checkpoint - safetensors_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors") - - # Check models are equal - for p1, p2 in zip(safetensors_model.weights, tf_model.weights): - self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) - - -@require_tf -@is_staging_test -class TFModelPushToHubTester(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-model-tf") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="test-model-tf-callback") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-model-tf-org") - except HTTPError: - pass - - def test_push_to_hub(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = TFBertModel(config) - # Make sure model is properly initialized - model.build() - - logging.set_verbosity_info() - logger = logging.get_logger("transformers.utils.hub") - with CaptureLogger(logger) as cl: - model.push_to_hub("test-model-tf", use_auth_token=self._token) - logging.set_verbosity_warning() - # Check the model card was created and uploaded. - self.assertIn("Uploading the following files to __DUMMY_TRANSFORMERS_USER__/test-model-tf", cl.out) - - new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") - models_equal = True - for p1, p2 in zip(model.weights, new_model.weights): - if not tf.math.reduce_all(p1 == p2): - models_equal = False - break - self.assertTrue(models_equal) - - # Reset repo - delete_repo(token=self._token, repo_id="test-model-tf") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained(tmp_dir, repo_id="test-model-tf", push_to_hub=True, use_auth_token=self._token) - - new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") - models_equal = True - for p1, p2 in zip(model.weights, new_model.weights): - if not tf.math.reduce_all(p1 == p2): - models_equal = False - break - self.assertTrue(models_equal) - - @is_pt_tf_cross_test - def test_push_to_hub_callback(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = TFBertForMaskedLM(config) - model.compile() - - with tempfile.TemporaryDirectory() as tmp_dir: - push_to_hub_callback = PushToHubCallback( - output_dir=tmp_dir, - hub_model_id="test-model-tf-callback", - hub_token=self._token, - ) - model.fit(model.dummy_inputs, model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) - - new_model = TFBertForMaskedLM.from_pretrained(f"{USER}/test-model-tf-callback") - models_equal = True - for p1, p2 in zip(model.weights, new_model.weights): - if not tf.math.reduce_all(p1 == p2): - models_equal = False - break - self.assertTrue(models_equal) - - tf_push_to_hub_params = dict(inspect.signature(TFPreTrainedModel.push_to_hub).parameters) - tf_push_to_hub_params.pop("base_model_card_args") - pt_push_to_hub_params = dict(inspect.signature(PreTrainedModel.push_to_hub).parameters) - pt_push_to_hub_params.pop("deprecated_kwargs") - self.assertDictEaual(tf_push_to_hub_params, pt_push_to_hub_params) - - def test_push_to_hub_in_organization(self): - config = BertConfig( - vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 - ) - model = TFBertModel(config) - # Make sure model is properly initialized - model.build() - - model.push_to_hub("valid_org/test-model-tf-org", use_auth_token=self._token) - - new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") - models_equal = True - for p1, p2 in zip(model.weights, new_model.weights): - if not tf.math.reduce_all(p1 == p2): - models_equal = False - break - self.assertTrue(models_equal) - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-model-tf-org") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - model.save_pretrained( - tmp_dir, push_to_hub=True, use_auth_token=self._token, repo_id="valid_org/test-model-tf-org" - ) - - new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") - models_equal = True - for p1, p2 in zip(model.weights, new_model.weights): - if not tf.math.reduce_all(p1 == p2): - models_equal = False - break - self.assertTrue(models_equal) diff --git a/tests/test_modeling_tf_utils.py b/tests/test_modeling_tf_utils.py new file mode 100644 index 00000000000000..862a2cffa8a0b2 --- /dev/null +++ b/tests/test_modeling_tf_utils.py @@ -0,0 +1,627 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import inspect +import json +import os +import random +import tempfile +import unittest +import unittest.mock as mock + +from huggingface_hub import HfFolder, Repository, delete_repo +from huggingface_hub.file_download import http_get +from requests.exceptions import HTTPError + +from transformers import is_tf_available, is_torch_available +from transformers.configuration_utils import PretrainedConfig +from transformers.testing_utils import ( # noqa: F401 + TOKEN, + USER, + CaptureLogger, + _tf_gpu_memory_limit, + is_pt_tf_cross_test, + is_staging_test, + require_safetensors, + require_tf, + slow, +) +from transformers.utils import SAFE_WEIGHTS_NAME, TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, logging + + +logger = logging.get_logger(__name__) + + +if is_tf_available(): + import h5py + import numpy as np + import tensorflow as tf + + from transformers import ( + BertConfig, + PreTrainedModel, + PushToHubCallback, + RagRetriever, + TFBertForMaskedLM, + TFBertForSequenceClassification, + TFBertModel, + TFPreTrainedModel, + TFRagModel, + ) + from transformers.modeling_tf_utils import tf_shard_checkpoint, unpack_inputs + from transformers.tf_utils import stable_softmax + + tf.config.experimental.enable_tensor_float_32_execution(False) + + if _tf_gpu_memory_limit is not None: + gpus = tf.config.list_physical_devices("GPU") + for gpu in gpus: + # Restrict TensorFlow to only allocate x GB of memory on the GPUs + try: + tf.config.set_logical_device_configuration( + gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)] + ) + logical_gpus = tf.config.list_logical_devices("GPU") + print("Logical GPUs", logical_gpus) + except RuntimeError as e: + # Virtual devices must be set before GPUs have been initialized + print(e) + +if is_torch_available(): + from transformers import BertModel + + +@require_tf +class TFModelUtilsTest(unittest.TestCase): + def test_cached_files_are_used_when_internet_is_down(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + # This check we did call the fake head request + mock_head.assert_called() + + def test_load_from_one_file(self): + try: + tmp_file = tempfile.mktemp() + with open(tmp_file, "wb") as f: + http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/tf_model.h5", f) + + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + _ = TFBertModel.from_pretrained(tmp_file, config=config) + finally: + os.remove(tmp_file) + + def test_legacy_load_from_url(self): + # This test is for deprecated behavior and can be removed in v5 + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + _ = TFBertModel.from_pretrained( + "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/tf_model.h5", config=config + ) + + # tests whether the unpack_inputs function behaves as expected + def test_unpack_inputs(self): + class DummyModel: + def __init__(self): + config_kwargs = {"output_attentions": False, "output_hidden_states": False, "return_dict": False} + self.config = PretrainedConfig(**config_kwargs) + self.main_input_name = "input_ids" + + @unpack_inputs + def call( + self, + input_ids=None, + past_key_values=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + return input_ids, past_key_values, output_attentions, output_hidden_states, return_dict + + @unpack_inputs + def foo(self, pixel_values, output_attentions=None, output_hidden_states=None, return_dict=None): + return pixel_values, output_attentions, output_hidden_states, return_dict + + dummy_model = DummyModel() + input_ids = tf.constant([0, 1, 2, 3], dtype=tf.int32) + past_key_values = tf.constant([4, 5, 6, 7], dtype=tf.int32) + pixel_values = tf.constant([8, 9, 10, 11], dtype=tf.int32) + + # test case 1: Pass inputs as keyword arguments; Booleans are inherited from the config. + output = dummy_model.call(input_ids=input_ids, past_key_values=past_key_values) + tf.debugging.assert_equal(output[0], input_ids) + tf.debugging.assert_equal(output[1], past_key_values) + self.assertFalse(output[2]) + self.assertFalse(output[3]) + self.assertFalse(output[4]) + + # test case 2: Same as above, but with positional arguments. + output = dummy_model.call(input_ids, past_key_values) + tf.debugging.assert_equal(output[0], input_ids) + tf.debugging.assert_equal(output[1], past_key_values) + self.assertFalse(output[2]) + self.assertFalse(output[3]) + self.assertFalse(output[4]) + + # test case 3: We can also pack everything in the first input. + output = dummy_model.call(input_ids={"input_ids": input_ids, "past_key_values": past_key_values}) + tf.debugging.assert_equal(output[0], input_ids) + tf.debugging.assert_equal(output[1], past_key_values) + self.assertFalse(output[2]) + self.assertFalse(output[3]) + self.assertFalse(output[4]) + + # test case 4: Explicit boolean arguments should override the config. + output = dummy_model.call( + input_ids=input_ids, past_key_values=past_key_values, output_attentions=False, return_dict=True + ) + tf.debugging.assert_equal(output[0], input_ids) + tf.debugging.assert_equal(output[1], past_key_values) + self.assertFalse(output[2]) + self.assertFalse(output[3]) + self.assertTrue(output[4]) + + # test case 5: Unexpected arguments should raise an exception. + with self.assertRaises(ValueError): + output = dummy_model.call(input_ids=input_ids, past_key_values=past_key_values, foo="bar") + + # test case 6: the decorator is independent from `main_input_name` -- it treats the first argument of the + # decorated function as its main input. + output = dummy_model.foo(pixel_values=pixel_values) + tf.debugging.assert_equal(output[0], pixel_values) + self.assertFalse(output[1]) + self.assertFalse(output[2]) + self.assertFalse(output[3]) + + # Tests whether the stable softmax is stable on CPU, with and without XLA + def test_xla_stable_softmax(self): + large_penalty = -1e9 + n_tokens = 10 + batch_size = 8 + + def masked_softmax(x, boolean_mask): + numerical_mask = (1.0 - tf.cast(boolean_mask, dtype=tf.float32)) * large_penalty + masked_x = x + numerical_mask + return stable_softmax(masked_x) + + xla_masked_softmax = tf.function(masked_softmax, jit_compile=True) + xla_stable_softmax = tf.function(stable_softmax, jit_compile=True) + x = tf.random.normal((batch_size, n_tokens)) + + # Same outcome regardless of the boolean mask here + masked_tokens = random.randint(0, n_tokens) + boolean_mask = tf.convert_to_tensor([[1] * (n_tokens - masked_tokens) + [0] * masked_tokens], dtype=tf.int32) + + # We can randomly mask a random numerical input OUTSIDE XLA + numerical_mask = (1.0 - tf.cast(boolean_mask, dtype=tf.float32)) * large_penalty + masked_x = x + numerical_mask + xla_out = xla_stable_softmax(masked_x) + out = stable_softmax(masked_x) + assert tf.experimental.numpy.allclose(xla_out, out) + + # The stable softmax has the same output as the original softmax + unstable_out = tf.nn.softmax(masked_x) + assert tf.experimental.numpy.allclose(unstable_out, out) + + # We can randomly mask a random numerical input INSIDE XLA + xla_out = xla_masked_softmax(x, boolean_mask) + out = masked_softmax(x, boolean_mask) + assert tf.experimental.numpy.allclose(xla_out, out) + + def test_checkpoint_sharding_from_hub(self): + model = TFBertModel.from_pretrained("ArthurZ/tiny-random-bert-sharded") + # the model above is the same as the model below, just a sharded version. + ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + for p1, p2 in zip(model.weights, ref_model.weights): + assert np.allclose(p1.numpy(), p2.numpy()) + + def test_sharded_checkpoint_with_prefix(self): + model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert", load_weight_prefix="a/b") + sharded_model = TFBertModel.from_pretrained("ArthurZ/tiny-random-bert-sharded", load_weight_prefix="a/b") + for p1, p2 in zip(model.weights, sharded_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + self.assertTrue(p1.name.startswith("a/b/")) + self.assertTrue(p2.name.startswith("a/b/")) + + def test_sharded_checkpoint_transfer(self): + # If this doesn't throw an error then the test passes + TFBertForSequenceClassification.from_pretrained("ArthurZ/tiny-random-bert-sharded") + + @is_pt_tf_cross_test + def test_checkpoint_sharding_local_from_pt(self): + with tempfile.TemporaryDirectory() as tmp_dir: + _ = Repository(local_dir=tmp_dir, clone_from="hf-internal-testing/tiny-random-bert-sharded") + model = TFBertModel.from_pretrained(tmp_dir, from_pt=True) + # the model above is the same as the model below, just a sharded pytorch version. + ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + for p1, p2 in zip(model.weights, ref_model.weights): + assert np.allclose(p1.numpy(), p2.numpy()) + + @is_pt_tf_cross_test + def test_checkpoint_loading_with_prefix_from_pt(self): + model = TFBertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert", from_pt=True, load_weight_prefix="a/b" + ) + ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert", from_pt=True) + for p1, p2 in zip(model.weights, ref_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + self.assertTrue(p1.name.startswith("a/b/")) + + @is_pt_tf_cross_test + def test_checkpoint_sharding_hub_from_pt(self): + model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded", from_pt=True) + # the model above is the same as the model below, just a sharded pytorch version. + ref_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + for p1, p2 in zip(model.weights, ref_model.weights): + assert np.allclose(p1.numpy(), p2.numpy()) + + def test_shard_checkpoint(self): + # This is the model we will use, total size 340,000 bytes. + model = tf.keras.Sequential( + [ + tf.keras.layers.Dense(200, use_bias=False), # size 80,000 + tf.keras.layers.Dense(200, use_bias=False), # size 160,000 + tf.keras.layers.Dense(100, use_bias=False), # size 80,000 + tf.keras.layers.Dense(50, use_bias=False), # size 20,000 + ] + ) + inputs = tf.zeros((1, 100), dtype=tf.float32) + model(inputs) + weights = model.weights + weights_dict = {w.name: w for w in weights} + with self.subTest("No shard when max size is bigger than model size"): + shards, index = tf_shard_checkpoint(weights) + self.assertIsNone(index) + self.assertDictEqual(shards, {TF2_WEIGHTS_NAME: weights}) + + with self.subTest("Test sharding, no weights bigger than max size"): + shards, index = tf_shard_checkpoint(weights, max_shard_size="300kB") + # Split is first two layers then last two. + self.assertDictEqual( + index, + { + "metadata": {"total_size": 340000}, + "weight_map": { + "dense/kernel:0": "tf_model-00001-of-00002.h5", + "dense_1/kernel:0": "tf_model-00001-of-00002.h5", + "dense_2/kernel:0": "tf_model-00002-of-00002.h5", + "dense_3/kernel:0": "tf_model-00002-of-00002.h5", + }, + }, + ) + + shard1 = [weights_dict["dense/kernel:0"], weights_dict["dense_1/kernel:0"]] + shard2 = [weights_dict["dense_2/kernel:0"], weights_dict["dense_3/kernel:0"]] + self.assertDictEqual(shards, {"tf_model-00001-of-00002.h5": shard1, "tf_model-00002-of-00002.h5": shard2}) + + with self.subTest("Test sharding with weights bigger than max size"): + shards, index = tf_shard_checkpoint(weights, max_shard_size="100kB") + # Split is first layer, second layer then last 2. + self.assertDictEqual( + index, + { + "metadata": {"total_size": 340000}, + "weight_map": { + "dense/kernel:0": "tf_model-00001-of-00003.h5", + "dense_1/kernel:0": "tf_model-00002-of-00003.h5", + "dense_2/kernel:0": "tf_model-00003-of-00003.h5", + "dense_3/kernel:0": "tf_model-00003-of-00003.h5", + }, + }, + ) + + shard1 = [weights_dict["dense/kernel:0"]] + shard2 = [weights_dict["dense_1/kernel:0"]] + shard3 = [weights_dict["dense_2/kernel:0"], weights_dict["dense_3/kernel:0"]] + self.assertDictEqual( + shards, + { + "tf_model-00001-of-00003.h5": shard1, + "tf_model-00002-of-00003.h5": shard2, + "tf_model-00003-of-00003.h5": shard3, + }, + ) + + @slow + def test_special_layer_name_sharding(self): + retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True) + model = TFRagModel.from_pretrained("facebook/rag-token-nq", retriever=retriever) + + with tempfile.TemporaryDirectory() as tmp_dir: + for max_size in ["150kB", "150kiB", "200kB", "200kiB"]: + model.save_pretrained(tmp_dir, max_shard_size=max_size) + ref_model = TFRagModel.from_pretrained(tmp_dir, retriever=retriever) + for p1, p2 in zip(model.weights, ref_model.weights): + assert np.allclose(p1.numpy(), p2.numpy()) + + def test_checkpoint_sharding_local(self): + model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + with tempfile.TemporaryDirectory() as tmp_dir: + # We use the same folder for various sizes to make sure a new save erases the old checkpoint. + for max_size in ["150kB", "150kiB", "200kB", "200kiB"]: + model.save_pretrained(tmp_dir, max_shard_size=max_size) + + # Get each shard file and its size + shard_to_size = {} + for shard in os.listdir(tmp_dir): + if shard.endswith(".h5"): + shard_file = os.path.join(tmp_dir, shard) + shard_to_size[shard_file] = os.path.getsize(shard_file) + + index_file = os.path.join(tmp_dir, TF2_WEIGHTS_INDEX_NAME) + # Check there is an index but no regular weight file + self.assertTrue(os.path.isfile(index_file)) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, TF2_WEIGHTS_NAME))) + + # Check a file is bigger than max_size only when it has a single weight + for shard_file, size in shard_to_size.items(): + if max_size.endswith("kiB"): + max_size_int = int(max_size[:-3]) * 2**10 + else: + max_size_int = int(max_size[:-2]) * 10**3 + # Note: pickle adds some junk so the weight of the file can end up being slightly bigger than + # the size asked for (since we count parameters) + if size >= max_size_int + 50000: + with h5py.File(shard_file, "r") as state_file: + self.assertEqual(len(state_file), 1) + + # Check the index and the shard files found match + with open(index_file, "r", encoding="utf-8") as f: + index = json.loads(f.read()) + + all_shards = set(index["weight_map"].values()) + shards_found = {f for f in os.listdir(tmp_dir) if f.endswith(".h5")} + self.assertSetEqual(all_shards, shards_found) + + # Finally, check the model can be reloaded + new_model = TFBertModel.from_pretrained(tmp_dir) + + model.build() + new_model.build() + + for p1, p2 in zip(model.weights, new_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + + @slow + def test_save_pretrained_signatures(self): + model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Short custom TF signature function. + # `input_signature` is specific to BERT. + @tf.function( + input_signature=[ + [ + tf.TensorSpec([None, None], tf.int32, name="input_ids"), + tf.TensorSpec([None, None], tf.int32, name="token_type_ids"), + tf.TensorSpec([None, None], tf.int32, name="attention_mask"), + ] + ] + ) + def serving_fn(input): + return model(input) + + # Using default signature (default behavior) overrides 'serving_default' + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, saved_model=True, signatures=None) + model_loaded = tf.keras.models.load_model(f"{tmp_dir}/saved_model/1") + self.assertTrue("serving_default" in list(model_loaded.signatures.keys())) + + # Providing custom signature function + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, saved_model=True, signatures={"custom_signature": serving_fn}) + model_loaded = tf.keras.models.load_model(f"{tmp_dir}/saved_model/1") + self.assertTrue("custom_signature" in list(model_loaded.signatures.keys())) + + # Providing multiple custom signature function + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained( + tmp_dir, + saved_model=True, + signatures={"custom_signature_1": serving_fn, "custom_signature_2": serving_fn}, + ) + model_loaded = tf.keras.models.load_model(f"{tmp_dir}/saved_model/1") + self.assertTrue("custom_signature_1" in list(model_loaded.signatures.keys())) + self.assertTrue("custom_signature_2" in list(model_loaded.signatures.keys())) + + @require_safetensors + def test_safetensors_save_and_load(self): + model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, safe_serialization=True) + # No tf_model.h5 file, only a model.safetensors + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, TF2_WEIGHTS_NAME))) + + new_model = TFBertModel.from_pretrained(tmp_dir) + + # Check models are equal + for p1, p2 in zip(model.weights, new_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + + @is_pt_tf_cross_test + def test_safetensors_save_and_load_pt_to_tf(self): + model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + pt_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + with tempfile.TemporaryDirectory() as tmp_dir: + pt_model.save_pretrained(tmp_dir, safe_serialization=True) + # Check we have a model.safetensors file + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) + + new_model = TFBertModel.from_pretrained(tmp_dir) + + # Check models are equal + for p1, p2 in zip(model.weights, new_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + + @require_safetensors + def test_safetensors_load_from_hub(self): + tf_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Can load from the TF-formatted checkpoint + safetensors_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors-tf") + + # Check models are equal + for p1, p2 in zip(safetensors_model.weights, tf_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + + # Can load from the PyTorch-formatted checkpoint + safetensors_model = TFBertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors") + + # Check models are equal + for p1, p2 in zip(safetensors_model.weights, tf_model.weights): + self.assertTrue(np.allclose(p1.numpy(), p2.numpy())) + + +@require_tf +@is_staging_test +class TFModelPushToHubTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-model-tf") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="test-model-tf-callback") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-model-tf-org") + except HTTPError: + pass + + def test_push_to_hub(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = TFBertModel(config) + # Make sure model is properly initialized + model.build() + + logging.set_verbosity_info() + logger = logging.get_logger("transformers.utils.hub") + with CaptureLogger(logger) as cl: + model.push_to_hub("test-model-tf", use_auth_token=self._token) + logging.set_verbosity_warning() + # Check the model card was created and uploaded. + self.assertIn("Uploading the following files to __DUMMY_TRANSFORMERS_USER__/test-model-tf", cl.out) + + new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") + models_equal = True + for p1, p2 in zip(model.weights, new_model.weights): + if not tf.math.reduce_all(p1 == p2): + models_equal = False + break + self.assertTrue(models_equal) + + # Reset repo + delete_repo(token=self._token, repo_id="test-model-tf") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, repo_id="test-model-tf", push_to_hub=True, use_auth_token=self._token) + + new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf") + models_equal = True + for p1, p2 in zip(model.weights, new_model.weights): + if not tf.math.reduce_all(p1 == p2): + models_equal = False + break + self.assertTrue(models_equal) + + @is_pt_tf_cross_test + def test_push_to_hub_callback(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = TFBertForMaskedLM(config) + model.compile() + + with tempfile.TemporaryDirectory() as tmp_dir: + push_to_hub_callback = PushToHubCallback( + output_dir=tmp_dir, + hub_model_id="test-model-tf-callback", + hub_token=self._token, + ) + model.fit(model.dummy_inputs, model.dummy_inputs, epochs=1, callbacks=[push_to_hub_callback]) + + new_model = TFBertForMaskedLM.from_pretrained(f"{USER}/test-model-tf-callback") + models_equal = True + for p1, p2 in zip(model.weights, new_model.weights): + if not tf.math.reduce_all(p1 == p2): + models_equal = False + break + self.assertTrue(models_equal) + + tf_push_to_hub_params = dict(inspect.signature(TFPreTrainedModel.push_to_hub).parameters) + tf_push_to_hub_params.pop("base_model_card_args") + pt_push_to_hub_params = dict(inspect.signature(PreTrainedModel.push_to_hub).parameters) + pt_push_to_hub_params.pop("deprecated_kwargs") + self.assertDictEaual(tf_push_to_hub_params, pt_push_to_hub_params) + + def test_push_to_hub_in_organization(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = TFBertModel(config) + # Make sure model is properly initialized + model.build() + + model.push_to_hub("valid_org/test-model-tf-org", use_auth_token=self._token) + + new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") + models_equal = True + for p1, p2 in zip(model.weights, new_model.weights): + if not tf.math.reduce_all(p1 == p2): + models_equal = False + break + self.assertTrue(models_equal) + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-model-tf-org") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained( + tmp_dir, push_to_hub=True, use_auth_token=self._token, repo_id="valid_org/test-model-tf-org" + ) + + new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org") + models_equal = True + for p1, p2 in zip(model.weights, new_model.weights): + if not tf.math.reduce_all(p1 == p2): + models_equal = False + break + self.assertTrue(models_equal) diff --git a/tests/test_modeling_utils.py b/tests/test_modeling_utils.py new file mode 100755 index 00000000000000..760d6c6d54d0a6 --- /dev/null +++ b/tests/test_modeling_utils.py @@ -0,0 +1,976 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +import os.path +import sys +import tempfile +import unittest +import unittest.mock as mock +from pathlib import Path + +from huggingface_hub import HfFolder, delete_repo +from huggingface_hub.file_download import http_get +from pytest import mark +from requests.exceptions import HTTPError + +from transformers import ( + AutoConfig, + AutoModel, + PretrainedConfig, + is_torch_available, + logging, +) +from transformers.testing_utils import ( + TOKEN, + USER, + CaptureLogger, + TestCasePlus, + is_staging_test, + require_accelerate, + require_safetensors, + require_torch, + require_torch_gpu, + require_torch_multi_gpu, + require_usr_bin_time, + slow, +) +from transformers.utils import ( + SAFE_WEIGHTS_INDEX_NAME, + SAFE_WEIGHTS_NAME, + WEIGHTS_INDEX_NAME, + WEIGHTS_NAME, +) + + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from test_module.custom_configuration import CustomConfig, NoSuperInitConfig # noqa E402 + + +if is_torch_available(): + import torch + from test_module.custom_modeling import CustomModel, NoSuperInitModel + from torch import nn + + from transformers import ( + BERT_PRETRAINED_MODEL_ARCHIVE_LIST, + AutoModelForCausalLM, + AutoTokenizer, + BertConfig, + BertModel, + CLIPTextModel, + PreTrainedModel, + T5Config, + T5ForConditionalGeneration, + ) + from transformers.modeling_utils import shard_checkpoint + + # Fake pretrained models for tests + class BaseModel(PreTrainedModel): + config_class = PretrainedConfig + + def __init__(self, config): + super().__init__(config) + self.linear = nn.Linear(4, 5) + self.linear_2 = nn.Linear(5, 6) + + def forward(self, x): + return self.linear_2(self.linear(x)) + + class ModelWithHead(PreTrainedModel): + base_model_prefix = "base" + config_class = PretrainedConfig + + def _init_weights(self, module): + pass + + def __init__(self, config): + super().__init__(config) + self.base = BaseModel(config) + # linear is a common name between Base and Head on purpose. + self.linear = nn.Linear(6, 3) + self.linear2 = nn.Linear(3, 5) + + def forward(self, x): + return self.linear2(self.linear(self.base(x))) + + +TINY_T5 = "patrickvonplaten/t5-tiny-random" +TINY_BERT_FOR_TOKEN_CLASSIFICATION = "hf-internal-testing/tiny-bert-for-token-classification" + + +def check_models_equal(model1, model2): + models_are_equal = True + for model1_p, model2_p in zip(model1.parameters(), model2.parameters()): + if model1_p.data.ne(model2_p.data).sum() > 0: + models_are_equal = False + + return models_are_equal + + +@require_torch +class ModelUtilsTest(TestCasePlus): + @slow + def test_model_from_pretrained(self): + for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + config = BertConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, PretrainedConfig) + + model = BertModel.from_pretrained(model_name) + model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) + self.assertIsNotNone(model) + self.assertIsInstance(model, PreTrainedModel) + + self.assertEqual(len(loading_info["missing_keys"]), 0) + self.assertEqual(len(loading_info["unexpected_keys"]), 8) + self.assertEqual(len(loading_info["mismatched_keys"]), 0) + self.assertEqual(len(loading_info["error_msgs"]), 0) + + config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + + # Not sure this is the intended behavior. TODO fix Lysandre & Thom + config.name_or_path = model_name + + model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + self.assertEqual(model.config.output_hidden_states, True) + self.assertEqual(model.config, config) + + def test_model_from_pretrained_subfolder(self): + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + model = BertModel(config) + + subfolder = "bert" + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(os.path.join(tmp_dir, subfolder)) + + with self.assertRaises(OSError): + _ = BertModel.from_pretrained(tmp_dir) + + model_loaded = BertModel.from_pretrained(tmp_dir, subfolder=subfolder) + + self.assertTrue(check_models_equal(model, model_loaded)) + + def test_model_from_pretrained_subfolder_sharded(self): + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + model = BertModel(config) + + subfolder = "bert" + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(os.path.join(tmp_dir, subfolder), max_shard_size="10KB") + + with self.assertRaises(OSError): + _ = BertModel.from_pretrained(tmp_dir) + + model_loaded = BertModel.from_pretrained(tmp_dir, subfolder=subfolder) + + self.assertTrue(check_models_equal(model, model_loaded)) + + def test_model_from_pretrained_hub_subfolder(self): + subfolder = "bert" + model_id = "hf-internal-testing/tiny-random-bert-subfolder" + with self.assertRaises(OSError): + _ = BertModel.from_pretrained(model_id) + + model = BertModel.from_pretrained(model_id, subfolder=subfolder) + + self.assertIsNotNone(model) + + def test_model_from_pretrained_hub_subfolder_sharded(self): + subfolder = "bert" + model_id = "hf-internal-testing/tiny-random-bert-sharded-subfolder" + with self.assertRaises(OSError): + _ = BertModel.from_pretrained(model_id) + + model = BertModel.from_pretrained(model_id, subfolder=subfolder) + + self.assertIsNotNone(model) + + def test_model_from_pretrained_with_different_pretrained_model_name(self): + model = T5ForConditionalGeneration.from_pretrained(TINY_T5) + self.assertIsNotNone(model) + + logger = logging.get_logger("transformers.configuration_utils") + with CaptureLogger(logger) as cl: + BertModel.from_pretrained(TINY_T5) + self.assertTrue("You are using a model of type t5 to instantiate a model of type bert" in cl.out) + + def test_model_from_config_torch_dtype(self): + # test that the model can be instantiated with dtype of user's choice - as long as it's a + # float dtype. To make it happen config.torch_dtype needs to be set before instantiating the + # model from the config object. + + config = T5Config.from_pretrained(TINY_T5) + model = AutoModel.from_config(config) + # XXX: isn't supported + # model = T5ForConditionalGeneration.from_config(config) + self.assertEqual(model.dtype, torch.float32) + + model = AutoModel.from_config(config, torch_dtype=torch.float16) + self.assertEqual(model.dtype, torch.float16) + + # torch.set_default_dtype() supports only float dtypes, so will fail with non-float type + with self.assertRaises(ValueError): + model = AutoModel.from_config(config, torch_dtype=torch.int64) + + def test_model_from_pretrained_torch_dtype(self): + # test that the model can be instantiated with dtype of either + # 1. explicit from_pretrained's torch_dtype argument + # 2. via autodiscovery by looking at model weights (torch_dtype="auto") + # so if a model.half() was saved, we want it to be instantiated as such. + # + # test an explicit model class, but also AutoModel separately as the latter goes through a different code path + model_path = self.get_auto_remove_tmp_dir() + + # baseline - we know TINY_T5 is fp32 model + model = T5ForConditionalGeneration.from_pretrained(TINY_T5) + self.assertEqual(model.dtype, torch.float32) + + def remove_torch_dtype(model_path): + file = f"{model_path}/config.json" + with open(file, "r", encoding="utf-8") as f: + s = json.load(f) + s.pop("torch_dtype") + with open(file, "w", encoding="utf-8") as f: + json.dump(s, f) + + # test the default fp32 save_pretrained => from_pretrained cycle + model.save_pretrained(model_path) + model = T5ForConditionalGeneration.from_pretrained(model_path) + self.assertEqual(model.dtype, torch.float32) + # 1. test torch_dtype="auto" via `config.torch_dtype` + model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") + self.assertEqual(model.dtype, torch.float32) + # 2. test torch_dtype="auto" via auto-derivation + # now remove the torch_dtype entry from config.json and try "auto" again which should + # perform auto-derivation from weights + remove_torch_dtype(model_path) + model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") + self.assertEqual(model.dtype, torch.float32) + + # test forced loading in fp16 (even though the weights are in fp32) + model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.float16) + self.assertEqual(model.dtype, torch.float16) + + # test fp16 save_pretrained, loaded with auto-detection + model = model.half() + model.save_pretrained(model_path) + # 1. test torch_dtype="auto" via `config.torch_dtype` + model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") + self.assertEqual(model.config.torch_dtype, torch.float16) + self.assertEqual(model.dtype, torch.float16) + # tests `config.torch_dtype` saving + with open(f"{model_path}/config.json") as f: + config_dict = json.load(f) + self.assertEqual(config_dict["torch_dtype"], "float16") + # 2. test torch_dtype="auto" via auto-derivation + # now same with using config info + remove_torch_dtype(model_path) + model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype="auto") + self.assertEqual(model.dtype, torch.float16) + + # 3. now retest that AutoModel behaves the same wrt torch_dtype="auto" as T5ForConditionalGeneration + model = AutoModel.from_pretrained(model_path, torch_dtype="auto") + self.assertEqual(model.dtype, torch.float16) + + # test fp16 save_pretrained, loaded with the explicit fp16 + model = T5ForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.float16) + self.assertEqual(model.dtype, torch.float16) + + # test AutoModel separately as it goes through a different path + # test auto-detection - as currently TINY_T5 doesn't have torch_dtype entry + model = AutoModel.from_pretrained(TINY_T5, torch_dtype="auto") + # test that the config object didn't get polluted with torch_dtype="auto" + # there was a bug that after this call we ended up with config.torch_dtype=="auto" + self.assertNotEqual(model.config.torch_dtype, "auto") + # now test the outcome + self.assertEqual(model.dtype, torch.float32) + model = AutoModel.from_pretrained(TINY_T5, torch_dtype=torch.float16) + self.assertEqual(model.dtype, torch.float16) + + # test model whose first param is not of a floating type, but int + model = AutoModel.from_pretrained(TINY_BERT_FOR_TOKEN_CLASSIFICATION, torch_dtype="auto") + self.assertEqual(model.dtype, torch.float32) + + def test_no_super_init_config_and_model(self): + config = NoSuperInitConfig(attribute=32) + model = NoSuperInitModel(config) + + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir) + + new_model = NoSuperInitModel.from_pretrained(tmp_dir) + + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + def test_shard_checkpoint(self): + # This is the model we will use, total size 340,000 bytes. + model = torch.nn.Sequential( + torch.nn.Linear(100, 200, bias=False), # size 80,000 + torch.nn.Linear(200, 200, bias=False), # size 160,000 + torch.nn.Linear(200, 100, bias=False), # size 80,000 + torch.nn.Linear(100, 50, bias=False), # size 20,000 + ) + state_dict = model.state_dict() + + with self.subTest("No shard when max size is bigger than model size"): + shards, index = shard_checkpoint(state_dict) + self.assertIsNone(index) + self.assertDictEqual(shards, {WEIGHTS_NAME: state_dict}) + + with self.subTest("Test sharding, no weights bigger than max size"): + shards, index = shard_checkpoint(state_dict, max_shard_size="300kB") + # Split is first two layers then last two. + self.assertDictEqual( + index, + { + "metadata": {"total_size": 340000}, + "weight_map": { + "0.weight": "pytorch_model-00001-of-00002.bin", + "1.weight": "pytorch_model-00001-of-00002.bin", + "2.weight": "pytorch_model-00002-of-00002.bin", + "3.weight": "pytorch_model-00002-of-00002.bin", + }, + }, + ) + + shard1 = {"0.weight": state_dict["0.weight"], "1.weight": state_dict["1.weight"]} + shard2 = {"2.weight": state_dict["2.weight"], "3.weight": state_dict["3.weight"]} + self.assertDictEqual( + shards, {"pytorch_model-00001-of-00002.bin": shard1, "pytorch_model-00002-of-00002.bin": shard2} + ) + + with self.subTest("Test sharding with weights bigger than max size"): + shards, index = shard_checkpoint(state_dict, max_shard_size="100kB") + # Split is first layer, second layer then last 2. + self.assertDictEqual( + index, + { + "metadata": {"total_size": 340000}, + "weight_map": { + "0.weight": "pytorch_model-00001-of-00003.bin", + "1.weight": "pytorch_model-00002-of-00003.bin", + "2.weight": "pytorch_model-00003-of-00003.bin", + "3.weight": "pytorch_model-00003-of-00003.bin", + }, + }, + ) + + shard1 = {"0.weight": state_dict["0.weight"]} + shard2 = {"1.weight": state_dict["1.weight"]} + shard3 = {"2.weight": state_dict["2.weight"], "3.weight": state_dict["3.weight"]} + self.assertDictEqual( + shards, + { + "pytorch_model-00001-of-00003.bin": shard1, + "pytorch_model-00002-of-00003.bin": shard2, + "pytorch_model-00003-of-00003.bin": shard3, + }, + ) + + def test_checkpoint_sharding_local(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + with tempfile.TemporaryDirectory() as tmp_dir: + # We use the same folder for various sizes to make sure a new save erases the old checkpoint. + for max_size in ["50kB", "50kiB", "100kB", "100kiB", "200kB", "200kiB"]: + model.save_pretrained(tmp_dir, max_shard_size=max_size) + + # Get each shard file and its size + shard_to_size = {} + for shard in os.listdir(tmp_dir): + if shard.endswith(".bin"): + shard_file = os.path.join(tmp_dir, shard) + shard_to_size[shard_file] = os.path.getsize(shard_file) + + index_file = os.path.join(tmp_dir, WEIGHTS_INDEX_NAME) + # Check there is an index but no regular weight file + self.assertTrue(os.path.isfile(index_file)) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) + + # Check a file is bigger than max_size only when it has a single weight + for shard_file, size in shard_to_size.items(): + if max_size.endswith("kiB"): + max_size_int = int(max_size[:-3]) * 2**10 + else: + max_size_int = int(max_size[:-2]) * 10**3 + # Note: pickle adds some junk so the weight of the file can end up being slightly bigger than + # the size asked for (since we count parameters) + if size >= max_size_int + 50000: + state_dict = torch.load(shard_file) + self.assertEqual(len(state_dict), 1) + + # Check the index and the shard files found match + with open(index_file, "r", encoding="utf-8") as f: + index = json.loads(f.read()) + + all_shards = set(index["weight_map"].values()) + shards_found = {f for f in os.listdir(tmp_dir) if f.endswith(".bin")} + self.assertSetEqual(all_shards, shards_found) + + # Finally, check the model can be reloaded + new_model = BertModel.from_pretrained(tmp_dir) + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + def test_checkpoint_sharding_from_hub(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded") + # the model above is the same as the model below, just a sharded version. + ref_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + for p1, p2 in zip(model.parameters(), ref_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + def test_checkpoint_variant_local(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, variant="v2") + + weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["bin"]) + + weights_file = os.path.join(tmp_dir, weights_name) + self.assertTrue(os.path.isfile(weights_file)) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) + + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained(tmp_dir) + + new_model = BertModel.from_pretrained(tmp_dir, variant="v2") + + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + def test_checkpoint_variant_local_sharded(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, variant="v2", max_shard_size="50kB") + + weights_index_name = ".".join(WEIGHTS_INDEX_NAME.split(".")[:-1] + ["v2"] + ["json"]) + weights_index_file = os.path.join(tmp_dir, weights_index_name) + self.assertTrue(os.path.isfile(weights_index_file)) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_INDEX_NAME))) + + for i in range(1, 6): + weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + [f"v2-0000{i}-of-00006"] + ["bin"]) + weights_name_file = os.path.join(tmp_dir, weights_name) + self.assertTrue(os.path.isfile(weights_name_file)) + + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained(tmp_dir) + + new_model = BertModel.from_pretrained(tmp_dir, variant="v2") + + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + @require_safetensors + def test_checkpoint_variant_local_safe(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, variant="v2", safe_serialization=True) + + weights_name = ".".join(SAFE_WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["safetensors"]) + + weights_file = os.path.join(tmp_dir, weights_name) + self.assertTrue(os.path.isfile(weights_file)) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) + + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained(tmp_dir) + + new_model = BertModel.from_pretrained(tmp_dir, variant="v2") + + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + @require_safetensors + def test_checkpoint_variant_local_sharded_safe(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, variant="v2", max_shard_size="50kB", safe_serialization=True) + + weights_index_name = ".".join(SAFE_WEIGHTS_INDEX_NAME.split(".")[:-1] + ["v2"] + ["json"]) + weights_index_file = os.path.join(tmp_dir, weights_index_name) + self.assertTrue(os.path.isfile(weights_index_file)) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_INDEX_NAME))) + + for i in range(1, 6): + weights_name = ".".join(SAFE_WEIGHTS_NAME.split(".")[:-1] + [f"v2-0000{i}-of-00006"] + ["safetensors"]) + weights_name_file = os.path.join(tmp_dir, weights_name) + self.assertTrue(os.path.isfile(weights_name_file)) + + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained(tmp_dir) + + new_model = BertModel.from_pretrained(tmp_dir, variant="v2") + + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + def test_checkpoint_variant_hub(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-variant", cache_dir=tmp_dir) + model = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant", cache_dir=tmp_dir, variant="v2" + ) + self.assertIsNotNone(model) + + def test_checkpoint_variant_hub_sharded(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant-sharded", cache_dir=tmp_dir + ) + model = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant-sharded", cache_dir=tmp_dir, variant="v2" + ) + self.assertIsNotNone(model) + + @require_safetensors + def test_checkpoint_variant_hub_safe(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-variant-safe", cache_dir=tmp_dir) + model = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant-safe", cache_dir=tmp_dir, variant="v2" + ) + self.assertIsNotNone(model) + + @require_safetensors + def test_checkpoint_variant_hub_sharded_safe(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with self.assertRaises(EnvironmentError): + _ = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant-sharded-safe", cache_dir=tmp_dir + ) + model = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant-sharded-safe", cache_dir=tmp_dir, variant="v2" + ) + self.assertIsNotNone(model) + + def test_checkpoint_variant_save_load(self): + with tempfile.TemporaryDirectory() as tmp_dir: + model = BertModel.from_pretrained( + "hf-internal-testing/tiny-random-bert-variant", cache_dir=tmp_dir, variant="v2" + ) + weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["bin"]) + + model.save_pretrained(tmp_dir, variant="v2") + # saving will create a variant checkpoint + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, weights_name))) + + model.save_pretrained(tmp_dir) + # saving shouldn't delete variant checkpoints + weights_name = ".".join(WEIGHTS_NAME.split(".")[:-1] + ["v2"] + ["bin"]) + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, weights_name))) + + # there should be a normal checkpoint + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) + + self.assertIsNotNone(model) + + @require_accelerate + @mark.accelerate_tests + def test_from_pretrained_low_cpu_mem_usage_functional(self): + # test that we can use `from_pretrained(..., low_cpu_mem_usage=True)` with normal and + # sharded models + + mnames = [ + "hf-internal-testing/tiny-random-bert-sharded", + "hf-internal-testing/tiny-random-bert", + ] + for mname in mnames: + _ = BertModel.from_pretrained(mname, low_cpu_mem_usage=True) + + @require_usr_bin_time + @require_accelerate + @mark.accelerate_tests + def test_from_pretrained_low_cpu_mem_usage_measured(self): + # test that `from_pretrained(..., low_cpu_mem_usage=True)` uses less cpu memory than default + + mname = "bert-base-cased" + + preamble = "from transformers import AutoModel" + one_liner_str = f'{preamble}; AutoModel.from_pretrained("{mname}", low_cpu_mem_usage=False)' + max_rss_normal = self.python_one_liner_max_rss(one_liner_str) + # print(f"{max_rss_normal=}") + + one_liner_str = f'{preamble}; AutoModel.from_pretrained("{mname}", low_cpu_mem_usage=True)' + max_rss_low_mem = self.python_one_liner_max_rss(one_liner_str) + # print(f"{max_rss_low_mem=}") + + diff_bytes = max_rss_normal - max_rss_low_mem + diff_percent = diff_bytes / max_rss_low_mem + # print(f"{diff_bytes=}, {diff_percent=}") + # ideally we would compare that the diff is close to ~1x checkpoint size in bytes, but + # measuring cpu memory on linux is very tricky and inconsistent, so instead let's check that + # it's at least 15% less cpu memory consumed + + self.assertGreater( + diff_percent, + 0.15, + "should use less CPU memory for low_cpu_mem_usage=True, " + f"but got max_rss_normal={max_rss_normal} and max_rss_low_mem={max_rss_low_mem}", + ) + + # if you want to compare things manually, let's first look at the size of the model in bytes + # model = BertModel.from_pretrained(mname, low_cpu_mem_usage=False) + # total_numel = sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values()) + # total_bytes = total_numel * 4 # 420MB + # Now the diff_bytes should be very close to total_bytes, but the reports are inconsistent. + # The easiest way to test this is to switch the model and torch.load to do all the work on + # gpu - that way one can measure exactly the total and peak memory used. Perhaps once we add + # functionality to load models directly on gpu, this test can be rewritten to use torch's + # cuda memory tracking and then we should be able to do a much more precise test. + + @require_accelerate + @mark.accelerate_tests + @require_torch_multi_gpu + @slow + def test_model_parallelism_gpt2(self): + device_map = {"transformer.wte": 0, "transformer.wpe": 0, "lm_head": 0, "transformer.ln_f": 1} + for i in range(12): + device_map[f"transformer.h.{i}"] = 0 if i <= 5 else 1 + + model = AutoModelForCausalLM.from_pretrained("gpt2", device_map=device_map) + + tokenizer = AutoTokenizer.from_pretrained("gpt2") + inputs = tokenizer("Hello, my name is", return_tensors="pt") + output = model.generate(inputs["input_ids"].to(0)) + + text_output = tokenizer.decode(output[0].tolist()) + self.assertEqual(text_output, "Hello, my name is John. I'm a writer, and I'm a writer. I'm") + + @require_accelerate + @mark.accelerate_tests + @require_torch_gpu + def test_from_pretrained_disk_offload_task_model(self): + model = AutoModel.from_pretrained("hf-internal-testing/tiny-random-gpt2") + device_map = { + "transformer.wte": 0, + "transformer.wpe": 0, + "transformer.h.0": "cpu", + "transformer.h.1": "cpu", + "transformer.h.2": "cpu", + "transformer.h.3": "disk", + "transformer.h.4": "disk", + "transformer.ln_f": 0, + "lm_head": 0, + } + with tempfile.TemporaryDirectory() as tmp_dir: + inputs = torch.tensor([[1, 2, 3]]).to(0) + + model.save_pretrained(tmp_dir) + new_model = AutoModelForCausalLM.from_pretrained(tmp_dir).to(0) + outputs1 = new_model.to(0)(inputs) + + offload_folder = os.path.join(tmp_dir, "offload") + new_model_with_offload = AutoModelForCausalLM.from_pretrained( + tmp_dir, device_map=device_map, offload_folder=offload_folder + ) + outputs2 = new_model_with_offload(inputs) + + self.assertTrue(torch.allclose(outputs1.logits.cpu(), outputs2.logits.cpu())) + + # With state dict temp offload + offload_folder = os.path.join(tmp_dir, "offload") + new_model_with_offload = AutoModelForCausalLM.from_pretrained( + tmp_dir, + device_map=device_map, + offload_folder=offload_folder, + offload_state_dict=True, + ) + outputs2 = new_model_with_offload(inputs) + + self.assertTrue(torch.allclose(outputs1.logits.cpu(), outputs2.logits.cpu())) + + def test_cached_files_are_used_when_internet_is_down(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + # This check we did call the fake head request + mock_head.assert_called() + + def test_load_from_one_file(self): + try: + tmp_file = tempfile.mktemp() + with open(tmp_file, "wb") as f: + http_get( + "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", f + ) + + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + _ = BertModel.from_pretrained(tmp_file, config=config) + finally: + os.remove(tmp_file) + + def test_legacy_load_from_url(self): + # This test is for deprecated behavior and can be removed in v5 + config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert") + _ = BertModel.from_pretrained( + "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", config=config + ) + + @require_safetensors + def test_use_safetensors(self): + # test nice error message if no safetensor files available + with self.assertRaises(OSError) as env_error: + AutoModel.from_pretrained("hf-internal-testing/tiny-random-RobertaModel", use_safetensors=True) + + self.assertTrue( + "model.safetensors or model.safetensors.index.json and thus cannot be loaded with `safetensors`" + in str(env_error.exception) + ) + + # test that error if only safetensors is available + with self.assertRaises(OSError) as env_error: + BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors", use_safetensors=False) + + self.assertTrue("does not appear to have a file named pytorch_model.bin" in str(env_error.exception)) + + # test that only safetensors if both available and use_safetensors=False + with tempfile.TemporaryDirectory() as tmp_dir: + CLIPTextModel.from_pretrained( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", + subfolder="text_encoder", + use_safetensors=False, + cache_dir=tmp_dir, + ) + + all_downloaded_files = glob.glob(os.path.join(tmp_dir, "*", "snapshots", "*", "*", "*")) + self.assertTrue(any(f.endswith("bin") for f in all_downloaded_files)) + self.assertFalse(any(f.endswith("safetensors") for f in all_downloaded_files)) + + # test that no safetensors if both available and use_safetensors=True + with tempfile.TemporaryDirectory() as tmp_dir: + CLIPTextModel.from_pretrained( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", + subfolder="text_encoder", + use_safetensors=True, + cache_dir=tmp_dir, + ) + + all_downloaded_files = glob.glob(os.path.join(tmp_dir, "*", "snapshots", "*", "*", "*")) + self.assertTrue(any(f.endswith("safetensors") for f in all_downloaded_files)) + self.assertFalse(any(f.endswith("bin") for f in all_downloaded_files)) + + @require_safetensors + def test_safetensors_save_and_load(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, safe_serialization=True) + # No pytorch_model.bin file, only a model.safetensors + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) + + new_model = BertModel.from_pretrained(tmp_dir) + + # Check models are equal + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + @require_safetensors + def test_safetensors_load_from_hub(self): + safetensors_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors") + pytorch_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Check models are equal + for p1, p2 in zip(safetensors_model.parameters(), pytorch_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + @require_safetensors + def test_safetensors_save_and_load_sharded(self): + model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, safe_serialization=True, max_shard_size="100kB") + # No pytorch_model.bin index file, only a model.safetensors index + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_INDEX_NAME))) + self.assertTrue(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_INDEX_NAME))) + # No regular weights file + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, WEIGHTS_NAME))) + self.assertFalse(os.path.isfile(os.path.join(tmp_dir, SAFE_WEIGHTS_NAME))) + + new_model = BertModel.from_pretrained(tmp_dir) + + # Check models are equal + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + @require_safetensors + def test_safetensors_load_from_hub_sharded(self): + safetensors_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded-safetensors") + pytorch_model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded") + + # Check models are equal + for p1, p2 in zip(safetensors_model.parameters(), pytorch_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + def test_base_model_to_head_model_load(self): + base_model = BaseModel(PretrainedConfig()) + with tempfile.TemporaryDirectory() as tmp_dir: + base_model.save_pretrained(tmp_dir) + + # Can load a base model in a model with head + model = ModelWithHead.from_pretrained(tmp_dir) + for p1, p2 in zip(model.base.parameters(), base_model.parameters()): + self.assertTrue(torch.allclose(p1, p2)) + + # It doesn't work if the state dict has a mix of keys of the head and base without prefix though. + base_state_dict = base_model.state_dict() + head_state_dict = model.state_dict() + base_state_dict["linear2.weight"] = head_state_dict["linear2.weight"] + base_state_dict["linear2.bias"] = head_state_dict["linear2.bias"] + torch.save(base_state_dict, os.path.join(tmp_dir, WEIGHTS_NAME)) + + with self.assertRaisesRegex( + ValueError, "The state dictionary of the model you are trying to load is corrupted." + ): + _ = ModelWithHead.from_pretrained(tmp_dir) + + @require_torch_gpu + @slow + def test_pretrained_low_mem_new_config(self): + # Checking for 1 model(the same one which was described in the issue) . + model_ids = ["gpt2"] + + for model_id in model_ids: + model_config = AutoConfig.from_pretrained(pretrained_model_name_or_path=model_id) + model_config.n_layer = 48 + model_config.n_head = 25 + model_config.n_embd = 1600 + model = AutoModelForCausalLM.from_pretrained( + pretrained_model_name_or_path=model_id, + config=model_config, + ignore_mismatched_sizes=True, + torch_dtype=torch.float16, + low_cpu_mem_usage=True, + ) + model_ref = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path=model_id) + + self.assertEqual(model.__class__.__name__, model_ref.__class__.__name__) + + +@require_torch +@is_staging_test +class ModelPushToHubTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-model") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-model-org") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="test-dynamic-model") + except HTTPError: + pass + + def test_push_to_hub(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = BertModel(config) + model.push_to_hub("test-model", use_auth_token=self._token) + + new_model = BertModel.from_pretrained(f"{USER}/test-model") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(token=self._token, repo_id="test-model") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, repo_id="test-model", push_to_hub=True, use_auth_token=self._token) + + new_model = BertModel.from_pretrained(f"{USER}/test-model") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + def test_push_to_hub_in_organization(self): + config = BertConfig( + vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 + ) + model = BertModel(config) + model.push_to_hub("valid_org/test-model-org", use_auth_token=self._token) + + new_model = BertModel.from_pretrained("valid_org/test-model-org") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-model-org") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained( + tmp_dir, push_to_hub=True, use_auth_token=self._token, repo_id="valid_org/test-model-org" + ) + + new_model = BertModel.from_pretrained("valid_org/test-model-org") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + def test_push_to_hub_dynamic_model(self): + CustomConfig.register_for_auto_class() + CustomModel.register_for_auto_class() + + config = CustomConfig(hidden_size=32) + model = CustomModel(config) + + model.push_to_hub("test-dynamic-model", use_auth_token=self._token) + # checks + self.assertDictEqual( + config.auto_map, + {"AutoConfig": "custom_configuration.CustomConfig", "AutoModel": "custom_modeling.CustomModel"}, + ) + + new_model = AutoModel.from_pretrained(f"{USER}/test-dynamic-model", trust_remote_code=True) + # Can't make an isinstance check because the new_model is from the CustomModel class of a dynamic module + self.assertEqual(new_model.__class__.__name__, "CustomModel") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + config = AutoConfig.from_pretrained(f"{USER}/test-dynamic-model", trust_remote_code=True) + new_model = AutoModel.from_config(config, trust_remote_code=True) + self.assertEqual(new_model.__class__.__name__, "CustomModel") diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 53231998ee41fa..0656e382a8ef5d 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -21,28 +21,20 @@ import pickle import re import shutil -import sys import tempfile import traceback import unittest -import unittest.mock as mock from collections import OrderedDict from itertools import takewhile -from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union -from huggingface_hub import HfFolder, delete_repo -from huggingface_hub.file_download import http_get from parameterized import parameterized -from requests.exceptions import HTTPError from transformers import ( AlbertTokenizer, AlbertTokenizerFast, - AutoTokenizer, BertTokenizer, BertTokenizerFast, - GPT2TokenizerFast, PreTrainedTokenizer, PreTrainedTokenizerBase, PreTrainedTokenizerFast, @@ -51,24 +43,20 @@ TrainingArguments, is_flax_available, is_tf_available, - is_tokenizers_available, is_torch_available, logging, ) from transformers.testing_utils import ( - TOKEN, - USER, check_json_file_has_correct_format, get_tests_dir, is_pt_tf_cross_test, - is_staging_test, require_tf, require_tokenizers, require_torch, run_test_in_subprocess, slow, ) -from transformers.tokenization_utils import AddedToken, Trie +from transformers.tokenization_utils import AddedToken if is_torch_available(): @@ -79,15 +67,6 @@ from transformers import PretrainedConfig, PreTrainedModel, TFPreTrainedModel -sys.path.append(str(Path(__file__).parent.parent / "utils")) - -from test_module.custom_tokenization import CustomTokenizer # noqa E402 - - -if is_tokenizers_available(): - from test_module.custom_tokenization_fast import CustomTokenizerFast - - logger = logging.get_logger(__name__) NON_ENGLISH_TAGS = ["chinese", "dutch", "french", "finnish", "german", "multilingual"] @@ -3974,238 +3953,3 @@ def test_clean_up_tokenization_spaces(self): tokenizer.clean_up_tokenization_spaces = True decoded = tokenizer.decode(tokens) assert decoded == "[CLS] this shouldn't be! he'll go. [SEP]" - - -class TokenizerUtilTester(unittest.TestCase): - def test_cached_files_are_used_when_internet_is_down(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") - - # Under the mock environment we get a 500 error when trying to reach the tokenizer. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") - # This check we did call the fake head request - mock_head.assert_called() - - @require_tokenizers - def test_cached_files_are_used_when_internet_is_down_missing_files(self): - # A mock response for an HTTP head request to emulate server down - response_mock = mock.Mock() - response_mock.status_code = 500 - response_mock.headers = {} - response_mock.raise_for_status.side_effect = HTTPError - response_mock.json.return_value = {} - - # Download this model to make sure it's in the cache. - _ = GPT2TokenizerFast.from_pretrained("gpt2") - - # Under the mock environment we get a 500 error when trying to reach the tokenizer. - with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: - _ = GPT2TokenizerFast.from_pretrained("gpt2") - # This check we did call the fake head request - mock_head.assert_called() - - def test_legacy_load_from_one_file(self): - # This test is for deprecated behavior and can be removed in v5 - try: - tmp_file = tempfile.mktemp() - with open(tmp_file, "wb") as f: - http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model", f) - - _ = AlbertTokenizer.from_pretrained(tmp_file) - finally: - os.remove(tmp_file) - - # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in - # the current folder and have the right name. - if os.path.isfile("tokenizer.json"): - # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it. - return - try: - with open("tokenizer.json", "wb") as f: - http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json", f) - tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") - # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000 - self.assertEqual(tokenizer.vocab_size, 1000) - # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file. - - finally: - os.remove("tokenizer.json") - - def test_legacy_load_from_url(self): - # This test is for deprecated behavior and can be removed in v5 - _ = AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model") - - -@is_staging_test -class TokenizerPushToHubTester(unittest.TestCase): - vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"] - - @classmethod - def setUpClass(cls): - cls._token = TOKEN - HfFolder.save_token(TOKEN) - - @classmethod - def tearDownClass(cls): - try: - delete_repo(token=cls._token, repo_id="test-tokenizer") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="valid_org/test-tokenizer-org") - except HTTPError: - pass - - try: - delete_repo(token=cls._token, repo_id="test-dynamic-tokenizer") - except HTTPError: - pass - - def test_push_to_hub(self): - with tempfile.TemporaryDirectory() as tmp_dir: - vocab_file = os.path.join(tmp_dir, "vocab.txt") - with open(vocab_file, "w", encoding="utf-8") as vocab_writer: - vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) - tokenizer = BertTokenizer(vocab_file) - - tokenizer.push_to_hub("test-tokenizer", use_auth_token=self._token) - new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer") - self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) - - # Reset repo - delete_repo(token=self._token, repo_id="test-tokenizer") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - tokenizer.save_pretrained(tmp_dir, repo_id="test-tokenizer", push_to_hub=True, use_auth_token=self._token) - - new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer") - self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) - - def test_push_to_hub_in_organization(self): - with tempfile.TemporaryDirectory() as tmp_dir: - vocab_file = os.path.join(tmp_dir, "vocab.txt") - with open(vocab_file, "w", encoding="utf-8") as vocab_writer: - vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) - tokenizer = BertTokenizer(vocab_file) - - tokenizer.push_to_hub("valid_org/test-tokenizer-org", use_auth_token=self._token) - new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org") - self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) - - # Reset repo - delete_repo(token=self._token, repo_id="valid_org/test-tokenizer-org") - - # Push to hub via save_pretrained - with tempfile.TemporaryDirectory() as tmp_dir: - tokenizer.save_pretrained( - tmp_dir, repo_id="valid_org/test-tokenizer-org", push_to_hub=True, use_auth_token=self._token - ) - - new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org") - self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) - - @require_tokenizers - def test_push_to_hub_dynamic_tokenizer(self): - CustomTokenizer.register_for_auto_class() - with tempfile.TemporaryDirectory() as tmp_dir: - vocab_file = os.path.join(tmp_dir, "vocab.txt") - with open(vocab_file, "w", encoding="utf-8") as vocab_writer: - vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) - tokenizer = CustomTokenizer(vocab_file) - - # No fast custom tokenizer - tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token) - - tokenizer = AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer", trust_remote_code=True) - # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module - self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizer") - - # Fast and slow custom tokenizer - CustomTokenizerFast.register_for_auto_class() - with tempfile.TemporaryDirectory() as tmp_dir: - vocab_file = os.path.join(tmp_dir, "vocab.txt") - with open(vocab_file, "w", encoding="utf-8") as vocab_writer: - vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) - - bert_tokenizer = BertTokenizerFast.from_pretrained(tmp_dir) - bert_tokenizer.save_pretrained(tmp_dir) - tokenizer = CustomTokenizerFast.from_pretrained(tmp_dir) - - tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token) - - tokenizer = AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer", trust_remote_code=True) - # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module - self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizerFast") - tokenizer = AutoTokenizer.from_pretrained( - f"{USER}/test-dynamic-tokenizer", use_fast=False, trust_remote_code=True - ) - # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module - self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizer") - - -class TrieTest(unittest.TestCase): - def test_trie(self): - trie = Trie() - trie.add("Hello 友達") - self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}) - trie.add("Hello") - trie.data - self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}) - - def test_trie_split(self): - trie = Trie() - self.assertEqual(trie.split("[CLS] This is a extra_id_100"), ["[CLS] This is a extra_id_100"]) - trie.add("[CLS]") - trie.add("extra_id_1") - trie.add("extra_id_100") - self.assertEqual(trie.split("[CLS] This is a extra_id_100"), ["[CLS]", " This is a ", "extra_id_100"]) - - def test_trie_single(self): - trie = Trie() - trie.add("A") - self.assertEqual(trie.split("ABC"), ["A", "BC"]) - self.assertEqual(trie.split("BCA"), ["BC", "A"]) - - def test_trie_final(self): - trie = Trie() - trie.add("TOKEN]") - trie.add("[SPECIAL_TOKEN]") - self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"), ["This is something ", "[SPECIAL_TOKEN]"]) - - def test_trie_subtokens(self): - trie = Trie() - trie.add("A") - trie.add("P") - trie.add("[SPECIAL_TOKEN]") - self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"), ["This is something ", "[SPECIAL_TOKEN]"]) - - def test_trie_suffix_tokens(self): - trie = Trie() - trie.add("AB") - trie.add("B") - trie.add("C") - self.assertEqual(trie.split("ABC"), ["AB", "C"]) - - def test_trie_skip(self): - trie = Trie() - trie.add("ABC") - trie.add("B") - trie.add("CD") - self.assertEqual(trie.split("ABCD"), ["ABC", "D"]) - - def test_cut_text_hardening(self): - # Even if the offsets are wrong, we necessarily output correct string - # parts. - trie = Trie() - parts = trie.cut_text("ABC", [0, 0, 2, 1, 2, 3]) - self.assertEqual(parts, ["AB", "C"]) diff --git a/tests/test_tokenization_utils.py b/tests/test_tokenization_utils.py new file mode 100644 index 00000000000000..2984de97fdcb5b --- /dev/null +++ b/tests/test_tokenization_utils.py @@ -0,0 +1,280 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import tempfile +import unittest +import unittest.mock as mock +from pathlib import Path + +from huggingface_hub import HfFolder, delete_repo +from huggingface_hub.file_download import http_get +from requests.exceptions import HTTPError + +from transformers import ( + AlbertTokenizer, + AutoTokenizer, + BertTokenizer, + BertTokenizerFast, + GPT2TokenizerFast, + is_tokenizers_available, +) +from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers +from transformers.tokenization_utils import Trie + + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from test_module.custom_tokenization import CustomTokenizer # noqa E402 + + +if is_tokenizers_available(): + from test_module.custom_tokenization_fast import CustomTokenizerFast + + +class TokenizerUtilTester(unittest.TestCase): + def test_cached_files_are_used_when_internet_is_down(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") + + # Under the mock environment we get a 500 error when trying to reach the tokenizer. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert") + # This check we did call the fake head request + mock_head.assert_called() + + @require_tokenizers + def test_cached_files_are_used_when_internet_is_down_missing_files(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + _ = GPT2TokenizerFast.from_pretrained("gpt2") + + # Under the mock environment we get a 500 error when trying to reach the tokenizer. + with mock.patch("requests.Session.request", return_value=response_mock) as mock_head: + _ = GPT2TokenizerFast.from_pretrained("gpt2") + # This check we did call the fake head request + mock_head.assert_called() + + def test_legacy_load_from_one_file(self): + # This test is for deprecated behavior and can be removed in v5 + try: + tmp_file = tempfile.mktemp() + with open(tmp_file, "wb") as f: + http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model", f) + + _ = AlbertTokenizer.from_pretrained(tmp_file) + finally: + os.remove(tmp_file) + + # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in + # the current folder and have the right name. + if os.path.isfile("tokenizer.json"): + # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it. + return + try: + with open("tokenizer.json", "wb") as f: + http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json", f) + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") + # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000 + self.assertEqual(tokenizer.vocab_size, 1000) + # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file. + + finally: + os.remove("tokenizer.json") + + def test_legacy_load_from_url(self): + # This test is for deprecated behavior and can be removed in v5 + _ = AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model") + + +@is_staging_test +class TokenizerPushToHubTester(unittest.TestCase): + vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"] + + @classmethod + def setUpClass(cls): + cls._token = TOKEN + HfFolder.save_token(TOKEN) + + @classmethod + def tearDownClass(cls): + try: + delete_repo(token=cls._token, repo_id="test-tokenizer") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="valid_org/test-tokenizer-org") + except HTTPError: + pass + + try: + delete_repo(token=cls._token, repo_id="test-dynamic-tokenizer") + except HTTPError: + pass + + def test_push_to_hub(self): + with tempfile.TemporaryDirectory() as tmp_dir: + vocab_file = os.path.join(tmp_dir, "vocab.txt") + with open(vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) + tokenizer = BertTokenizer(vocab_file) + + tokenizer.push_to_hub("test-tokenizer", use_auth_token=self._token) + new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer") + self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) + + # Reset repo + delete_repo(token=self._token, repo_id="test-tokenizer") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + tokenizer.save_pretrained(tmp_dir, repo_id="test-tokenizer", push_to_hub=True, use_auth_token=self._token) + + new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer") + self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) + + def test_push_to_hub_in_organization(self): + with tempfile.TemporaryDirectory() as tmp_dir: + vocab_file = os.path.join(tmp_dir, "vocab.txt") + with open(vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) + tokenizer = BertTokenizer(vocab_file) + + tokenizer.push_to_hub("valid_org/test-tokenizer-org", use_auth_token=self._token) + new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org") + self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) + + # Reset repo + delete_repo(token=self._token, repo_id="valid_org/test-tokenizer-org") + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + tokenizer.save_pretrained( + tmp_dir, repo_id="valid_org/test-tokenizer-org", push_to_hub=True, use_auth_token=self._token + ) + + new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org") + self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) + + @require_tokenizers + def test_push_to_hub_dynamic_tokenizer(self): + CustomTokenizer.register_for_auto_class() + with tempfile.TemporaryDirectory() as tmp_dir: + vocab_file = os.path.join(tmp_dir, "vocab.txt") + with open(vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) + tokenizer = CustomTokenizer(vocab_file) + + # No fast custom tokenizer + tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token) + + tokenizer = AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer", trust_remote_code=True) + # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module + self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizer") + + # Fast and slow custom tokenizer + CustomTokenizerFast.register_for_auto_class() + with tempfile.TemporaryDirectory() as tmp_dir: + vocab_file = os.path.join(tmp_dir, "vocab.txt") + with open(vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) + + bert_tokenizer = BertTokenizerFast.from_pretrained(tmp_dir) + bert_tokenizer.save_pretrained(tmp_dir) + tokenizer = CustomTokenizerFast.from_pretrained(tmp_dir) + + tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token) + + tokenizer = AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer", trust_remote_code=True) + # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module + self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizerFast") + tokenizer = AutoTokenizer.from_pretrained( + f"{USER}/test-dynamic-tokenizer", use_fast=False, trust_remote_code=True + ) + # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module + self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizer") + + +class TrieTest(unittest.TestCase): + def test_trie(self): + trie = Trie() + trie.add("Hello 友達") + self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}) + trie.add("Hello") + trie.data + self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}) + + def test_trie_split(self): + trie = Trie() + self.assertEqual(trie.split("[CLS] This is a extra_id_100"), ["[CLS] This is a extra_id_100"]) + trie.add("[CLS]") + trie.add("extra_id_1") + trie.add("extra_id_100") + self.assertEqual(trie.split("[CLS] This is a extra_id_100"), ["[CLS]", " This is a ", "extra_id_100"]) + + def test_trie_single(self): + trie = Trie() + trie.add("A") + self.assertEqual(trie.split("ABC"), ["A", "BC"]) + self.assertEqual(trie.split("BCA"), ["BC", "A"]) + + def test_trie_final(self): + trie = Trie() + trie.add("TOKEN]") + trie.add("[SPECIAL_TOKEN]") + self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"), ["This is something ", "[SPECIAL_TOKEN]"]) + + def test_trie_subtokens(self): + trie = Trie() + trie.add("A") + trie.add("P") + trie.add("[SPECIAL_TOKEN]") + self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"), ["This is something ", "[SPECIAL_TOKEN]"]) + + def test_trie_suffix_tokens(self): + trie = Trie() + trie.add("AB") + trie.add("B") + trie.add("C") + self.assertEqual(trie.split("ABC"), ["AB", "C"]) + + def test_trie_skip(self): + trie = Trie() + trie.add("ABC") + trie.add("B") + trie.add("CD") + self.assertEqual(trie.split("ABCD"), ["ABC", "D"]) + + def test_cut_text_hardening(self): + # Even if the offsets are wrong, we necessarily output correct string + # parts. + trie = Trie() + parts = trie.cut_text("ABC", [0, 0, 2, 1, 2, 3]) + self.assertEqual(parts, ["AB", "C"]) From 6cd34d451c8bcefb011907485fd6f5ee6828507f Mon Sep 17 00:00:00 2001 From: jiangmingyan <1829166702@qq.com> Date: Thu, 15 Jun 2023 19:33:37 +0800 Subject: [PATCH 10/15] [fix] bug in BatchEncoding.__getitem__ (#24293) Co-authored-by: luchen --- src/transformers/tokenization_utils_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index ba2f8ef3f13b32..eb757d98acfb86 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -242,7 +242,7 @@ def __getitem__(self, item: Union[int, str]) -> Union[Any, EncodingFast]: elif self._encodings is not None: return self._encodings[item] elif isinstance(item, slice): - return {key: self.data[key][slice] for key in self.data.keys()} + return {key: self.data[key][item] for key in self.data.keys()} else: raise KeyError( "Invalid key. Only three types of key are available: " From e6122c3f40d3c1fec5c4966a58340fd62d55cb71 Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:09:31 +0100 Subject: [PATCH 11/15] Fix image segmentation tool bug (#23897) * Image segmentation tool bug * Remove resizing in the tests --- src/transformers/tools/image_segmentation.py | 1 - tests/tools/test_image_segmentation.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/transformers/tools/image_segmentation.py b/src/transformers/tools/image_segmentation.py index 4471b849052785..b6cbf3eb3f7d53 100644 --- a/src/transformers/tools/image_segmentation.py +++ b/src/transformers/tools/image_segmentation.py @@ -44,7 +44,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def encode(self, image: "Image", label: str): - self.pre_processor.image_processor.size = {"width": image.size[0], "height": image.size[1]} return self.pre_processor(text=[label], images=[image], padding=True, return_tensors="pt") def forward(self, inputs): diff --git a/tests/tools/test_image_segmentation.py b/tests/tools/test_image_segmentation.py index 8a741b501b87ba..2f003f2c8b91d1 100644 --- a/tests/tools/test_image_segmentation.py +++ b/tests/tools/test_image_segmentation.py @@ -33,21 +33,21 @@ def setUp(self): self.remote_tool = load_tool("image-segmentation", remote=True) def test_exact_match_arg(self): - image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png").resize((512, 512)) + image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png") result = self.tool(image, "cat") self.assertTrue(isinstance(result, Image.Image)) def test_exact_match_arg_remote(self): - image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png").resize((512, 512)) + image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png") result = self.remote_tool(image, "cat") self.assertTrue(isinstance(result, Image.Image)) def test_exact_match_kwarg(self): - image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png").resize((512, 512)) + image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png") result = self.tool(image=image, label="cat") self.assertTrue(isinstance(result, Image.Image)) def test_exact_match_kwarg_remote(self): - image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png").resize((512, 512)) + image = Image.open(Path(get_tests_dir("fixtures/tests_samples/COCO")) / "000000039769.png") result = self.remote_tool(image=image, label="cat") self.assertTrue(isinstance(result, Image.Image)) From 604a21b1e68267df29e4910f425c92c336973f5d Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 15 Jun 2023 14:29:32 +0200 Subject: [PATCH 12/15] [Docs] Improve docs for MMS loading of other languages (#24292) * Improve docs * Apply suggestions from code review * upload readme * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- docs/source/en/model_doc/mms.mdx | 46 +++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/source/en/model_doc/mms.mdx b/docs/source/en/model_doc/mms.mdx index bd32617370cbb1..32cffbdfcbb4f1 100644 --- a/docs/source/en/model_doc/mms.mdx +++ b/docs/source/en/model_doc/mms.mdx @@ -44,11 +44,51 @@ MMS's architecture is based on the Wav2Vec2 model, so one can refer to [Wav2Vec2 The original code can be found [here](https://github.com/facebookresearch/fairseq/tree/main/examples/mms). -## Inference +## Loading + +By default MMS loads adapter weights for English. If you want to load adapter weights of another language +make sure to specify `target_lang=` as well as `"ignore_mismatched_sizes=True`. +The `ignore_mismatched_sizes=True` keyword has to be passed to allow the language model head to be resized according +to the vocabulary of the specified language. +Similarly, the processor should be loaded with the same target language + +```py +from transformers import Wav2Vec2ForCTC, AutoProcessor + +model_id = "facebook/mms-1b-all" +target_lang = "fra" + +processor = AutoProcessor.from_pretrained(model_id, target_lang=target_lang) +model = Wav2Vec2ForCTC.from_pretrained(model_id, target_lang=target_lang, ignore_mismatched_sizes=True) +``` + + + +You can safely ignore a warning such as: -By default MMS loads adapter weights for English, but those can be easily switched out for another language. -Let's look at an example. +```text +Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/mms-1b-all and are newly initialized because the shapes did not match: +- lm_head.bias: found shape torch.Size([154]) in the checkpoint and torch.Size([314]) in the model instantiated +- lm_head.weight: found shape torch.Size([154, 1280]) in the checkpoint and torch.Size([314, 1280]) in the model instantiated +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +``` + + + +If you want to use the ASR pipeline, you can load your chosen target language as such: + +```py +from transformers import pipeline + +model_id = "facebook/mms-1b-all" +target_lang = "fra" + +pipe = pipeline(model=model_id, model_kwargs={"target_lang": "fra", "ignore_mismatched_sizes": True}) +``` + +## Inference +Next, let's look at how we can run MMS in inference and change adapter layers after having called [`~PretrainedModel.from_pretrained`] First, we load audio data in different languages using the [Datasets](https://github.com/huggingface/datasets). ```py From 6a081c512a3f7f702dd35c9de4ed5019f82c3303 Mon Sep 17 00:00:00 2001 From: Cooper Date: Thu, 15 Jun 2023 20:50:40 +0800 Subject: [PATCH 13/15] Update README_zh-hans.md (#24181) * Update README_zh-hans.md update document link * Update README_zh-hans.md Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --------- Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- README_zh-hans.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_zh-hans.md b/README_zh-hans.md index 469fd867602e94..81912c08fd84c5 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -454,7 +454,7 @@ conda install -c huggingface transformers | 章节 | 描述 | |-|-| -| [文档](https://huggingface.co/transformers/) | 完整的 API 文档和教程 | +| [文档](https://huggingface.co/docs/transformers/) | 完整的 API 文档和教程 | | [任务总结](https://huggingface.co/docs/transformers/task_summary) | 🤗 Transformers 支持的任务 | | [预处理教程](https://huggingface.co/docs/transformers/preprocessing) | 使用 `Tokenizer` 来为模型准备数据 | | [训练和微调](https://huggingface.co/docs/transformers/training) | 在 PyTorch/TensorFlow 的训练循环或 `Trainer` API 中使用 🤗 Transformers 提供的模型 | From 01b55779d3789fa07885a677e2c17b21187eed98 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Thu, 15 Jun 2023 18:47:09 +0530 Subject: [PATCH 14/15] deepspeed init during eval fix (#24298) * deepspeed init during eval fix * commit suggestions Co-Authored-By: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/trainer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 9429b12ff96fac..e9fec6a5466b62 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -340,6 +340,7 @@ def __init__( # Seed must be set before instantiating the model when using model enable_full_determinism(self.args.seed) if self.args.full_determinism else set_seed(self.args.seed) self.hp_name = None + self.deepspeed = None self.is_in_train = False self.create_accelerator_and_postprocess() @@ -3041,7 +3042,7 @@ def evaluation_loop( prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else args.prediction_loss_only # if eval is called w/o train, handle model prep here - if self.is_deepspeed_enabled and self.model_wrapped is self.model: + if self.is_deepspeed_enabled and self.deepspeed is None: _, _ = deepspeed_init(self, num_training_steps=0, inference=True) model = self._wrap_model(self.model, training=False, dataloader=dataloader) @@ -3634,7 +3635,7 @@ def prediction_loop( prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else args.prediction_loss_only # if eval is called w/o train, handle model prep here - if self.is_deepspeed_enabled and self.model_wrapped is self.model: + if self.is_deepspeed_enabled and self.deepspeed is None: _, _ = deepspeed_init(self, num_training_steps=0, inference=True) model = self._wrap_model(self.model, training=False, dataloader=dataloader) From 4124a09f8b3349f338917ad3282ca952bd15ec3a Mon Sep 17 00:00:00 2001 From: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> Date: Thu, 15 Jun 2023 14:36:19 +0100 Subject: [PATCH 15/15] [EnCodec] Changes for 32kHz ckpt (#24296) * [EnCodec] Changes for 32kHz ckpt * Update src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py * Update src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py --- .../models/encodec/configuration_encodec.py | 5 +++++ .../convert_encodec_checkpoint_to_pytorch.py | 17 +++++++++++++++-- .../models/encodec/modeling_encodec.py | 5 ++++- tests/models/encodec/test_modeling_encodec.py | 5 +++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/encodec/configuration_encodec.py b/src/transformers/models/encodec/configuration_encodec.py index 9ea2cfee945b0d..e75711d9264e00 100644 --- a/src/transformers/models/encodec/configuration_encodec.py +++ b/src/transformers/models/encodec/configuration_encodec.py @@ -90,6 +90,9 @@ class EncodecConfig(PretrainedConfig): Number of discret codes that make up VQVAE. codebook_dim (`int`, *optional*): Dimension of the codebook vectors. If not defined, uses `hidden_size`. + use_conv_shortcut (`bool`, *optional*, defaults to `True`): + Whether to use a convolutional layer as the 'skip' connection in the `EncodecResnetBlock` block. If False, + an identity function will be used, giving a generic residual connection. Example: @@ -131,6 +134,7 @@ def __init__( trim_right_ratio=1.0, codebook_size=1024, codebook_dim=None, + use_conv_shortcut=True, **kwargs, ): self.target_bandwidths = target_bandwidths @@ -155,6 +159,7 @@ def __init__( self.trim_right_ratio = trim_right_ratio self.codebook_size = codebook_size self.codebook_dim = codebook_dim if codebook_dim is not None else hidden_size + self.use_conv_shortcut = use_conv_shortcut if self.norm_type not in ["weight_norm", "time_group_norm"]: raise ValueError( diff --git a/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py b/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py index cd7ead3d72741b..3a16a4b7ba0f3b 100644 --- a/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py +++ b/src/transformers/models/encodec/convert_encodec_checkpoint_to_pytorch.py @@ -28,6 +28,7 @@ # checkpoints downloaded from: # https://dl.fbaipublicfiles.com/encodec/v0/encodec_24khz-d7cc33bc.th +# https://huggingface.co/facebook/musicgen-small/resolve/main/compression_state_dict.bin # https://dl.fbaipublicfiles.com/encodec/v0/encodec_48khz-7e698e3e.th @@ -206,7 +207,7 @@ def should_ignore(name, ignore_keys): def recursively_load_weights(orig_dict, hf_model, model_name): unused_weights = [] - if model_name == "encodec_24khz": + if model_name == "encodec_24khz" or "encodec_32khz": MAPPING = MAPPING_24K elif model_name == "encodec_48khz": MAPPING = MAPPING_48K @@ -292,6 +293,15 @@ def convert_checkpoint( if model_name == "encodec_24khz": pass # config is already correct + elif model_name == "encodec_32khz": + config.upsampling_ratios = [8, 5, 4, 4] + config.target_bandwidths = [2.2] + config.num_filters = 64 + config.sampling_rate = 32_000 + config.codebook_size = 2048 + config.use_causal_conv = False + config.normalize = False + config.use_conv_shortcut = False elif model_name == "encodec_48khz": config.upsampling_ratios = [8, 5, 4, 2] config.target_bandwidths = [3.0, 6.0, 12.0, 24.0] @@ -316,6 +326,9 @@ def convert_checkpoint( feature_extractor.save_pretrained(pytorch_dump_folder_path) original_checkpoint = torch.load(checkpoint_path) + if "best_state" in original_checkpoint: + # we might have a training state saved, in which case discard the yaml results and just retain the weights + original_checkpoint = original_checkpoint["best_state"] recursively_load_weights(original_checkpoint, model, model_name) model.save_pretrained(pytorch_dump_folder_path) @@ -331,7 +344,7 @@ def convert_checkpoint( "--model", default="encodec_24khz", type=str, - help="The model to convert. Should be one of 'encodec_24khz', 'encodec_48khz'.", + help="The model to convert. Should be one of 'encodec_24khz', 'encodec_32khz', 'encodec_48khz'.", ) parser.add_argument("--checkpoint_path", required=True, default=None, type=str, help="Path to original checkpoint") parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert") diff --git a/src/transformers/models/encodec/modeling_encodec.py b/src/transformers/models/encodec/modeling_encodec.py index ad1f6a0ee83a9e..697fb3c94fbb1d 100644 --- a/src/transformers/models/encodec/modeling_encodec.py +++ b/src/transformers/models/encodec/modeling_encodec.py @@ -259,7 +259,10 @@ def __init__(self, config: EncodecConfig, dim: int, dilations: List[int]): block += [EncodecConv1d(config, in_chs, out_chs, kernel_size, dilation=dilation)] self.block = nn.ModuleList(block) - self.shortcut = EncodecConv1d(config, dim, dim, kernel_size=1) + if config.use_conv_shortcut: + self.shortcut = EncodecConv1d(config, dim, dim, kernel_size=1) + else: + self.shortcut = nn.Identity() def forward(self, hidden_states): residual = hidden_states diff --git a/tests/models/encodec/test_modeling_encodec.py b/tests/models/encodec/test_modeling_encodec.py index 23b2114a5db378..398da6f5d09ab6 100644 --- a/tests/models/encodec/test_modeling_encodec.py +++ b/tests/models/encodec/test_modeling_encodec.py @@ -385,6 +385,11 @@ def test_initialization(self): msg=f"Parameter {name} of model {model_class} seems not properly initialized", ) + def test_identity_shortcut(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs() + config.use_conv_shortcut = False + self.model_tester.create_and_check_model_forward(config, inputs_dict) + def normalize(arr): norm = np.linalg.norm(arr)