From 469d31f5bde6f743cfecba67e10c0de58f3ffe7b Mon Sep 17 00:00:00 2001 From: Max Marion Date: Thu, 15 Feb 2024 00:05:09 +0000 Subject: [PATCH 01/11] kill generation_length --- .../in_context_learning_evaluation.py | 13 ++++------ composer/models/huggingface.py | 1 - .../test_in_context_learning_datasets.py | 24 +++++++++---------- tests/models/test_hf_model.py | 4 ++-- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/composer/datasets/in_context_learning_evaluation.py b/composer/datasets/in_context_learning_evaluation.py index 2fc75cf899..e781cf9184 100644 --- a/composer/datasets/in_context_learning_evaluation.py +++ b/composer/datasets/in_context_learning_evaluation.py @@ -671,9 +671,7 @@ def __init__(self, self.cot_delimiter = cot_delimiter self.has_cot = False self.max_answer_length = 0 - static_keys = [ - 'mode', 'cot_delimiter', 'generation_length', 'generation_kwargs', 'do_normalization', 'stopping_criteria' - ] + static_keys = ['mode', 'cot_delimiter', 'generation_kwargs', 'do_normalization', 'stopping_criteria'] tensor_keys = ['input_ids', 'attention_mask'] list_keys = ['labels'] super().__init__(padding_side='left', @@ -690,10 +688,10 @@ def __init__(self, 'mode': 'generate', 'labels': [], 'cot_delimiter': self.cot_delimiter, - 'generation_length': self.max_answer_length, 'stopping_criteria': early_stopping_criteria, 'do_normalization': do_normalization, 'generation_kwargs': { + 'max_new_tokens': self.max_answer_length, 'pad_token_id': self.pad_tok_id, 'use_cache': True, 'eos_token_id': self.tokenizer.eos_token_id, @@ -1215,7 +1213,6 @@ class InContextLearningCodeEvalDataset(InContextLearningDataset): - test_outputs: List of test outputs - languages: List of languages - pass_at_k: Passed value for pass_at_k - - generation_length: Derrived maximum generation length - generation_kwargs: Dictionary of kwargs neeeded for generation. Includes the following, which will be individually overwritten by keys in generaiton_kwargs if set (see https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig for more details): @@ -1255,7 +1252,7 @@ def __init__( # Linting complains if these are not set in init self.max_prompt_length = 0 self.max_answer_length = 0 - static_keys = ['mode', 'pass_at_k', 'generation_length', 'generation_kwargs'] + static_keys = ['mode', 'pass_at_k', 'generation_kwargs'] list_keys = ['prompts', 'tests', 'entry_points', 'test_inputs', 'test_outputs', 'languages', 'labels'] tensor_keys = ['input_ids', 'attention_mask'] super().__init__( @@ -1284,14 +1281,14 @@ def __init__( 'test_outputs': [], 'languages': [], 'pass_at_k': pass_at_k, - 'generation_length': min(self.max_answer_length, self.max_seq_len - self.max_prompt_length), 'generation_kwargs': { 'pad_token_id': self.pad_tok_id, 'num_beams': 1, # single beam 'num_return_sequences': generations_per_sample, 'do_sample': True, 'use_cache': True, - 'eos_token_id': self.tokenizer.eos_token_id + 'eos_token_id': self.tokenizer.eos_token_id, + 'max_new_tokens': min(self.max_answer_length, self.max_seq_len - self.max_prompt_length), } } if 'generation_kwargs' in kwargs: diff --git a/composer/models/huggingface.py b/composer/models/huggingface.py index 439f8b50fe..359092b6d9 100644 --- a/composer/models/huggingface.py +++ b/composer/models/huggingface.py @@ -469,7 +469,6 @@ def eval_forward(self, batch, outputs: Optional[Any] = None): self.labels = batch.pop('labels') generation = self.generate(batch['input_ids'], attention_mask=batch['attention_mask'], - max_new_tokens=batch['generation_length'], synced_gpus=dist.get_world_size() > 1, **batch.get('generation_kwargs', {})) diff --git a/tests/datasets/test_in_context_learning_datasets.py b/tests/datasets/test_in_context_learning_datasets.py index 063f7215bc..ccdf0e120f 100644 --- a/tests/datasets/test_in_context_learning_datasets.py +++ b/tests/datasets/test_in_context_learning_datasets.py @@ -296,7 +296,7 @@ def test_update_generation_kwargs_no_kwargs_qa_dataset(tmp_path): continuation_delimiter=': ', destination_path=str(Path(gathered_paths[0]) / 'icl.jsonl'), generation_kwargs=None) - assert len(dl.base_batch['generation_kwargs']) == 3 + assert len(dl.base_batch['generation_kwargs']) == 4 def test_update_generation_kwargs_with_kwargs_qa_dataset(tmp_path): @@ -321,7 +321,7 @@ def test_update_generation_kwargs_with_kwargs_qa_dataset(tmp_path): generation_kwargs={'temperature': 0.9}) assert 'generation_kwargs' in dl.base_batch assert dl.base_batch['generation_kwargs']['temperature'] == 0.9 - assert len(dl.base_batch['generation_kwargs']) == 4 + assert len(dl.base_batch['generation_kwargs']) == 5 @pytest.mark.filterwarnings( @@ -1255,8 +1255,8 @@ def test_qa_split_batch(tiny_opt_tokenizer, dataset_uri, tmp_path): assert len(split2['labels']) == 1 assert all(isinstance(v, list) for v in split1['labels'] + split2['labels']) - assert isinstance(split1['generation_length'], int) - assert isinstance(split2['generation_length'], int) + assert isinstance(split1['generation_kwargs']['max_new_tokens'], int) + assert isinstance(split2['generation_kwargs']['max_new_tokens'], int) assert isinstance(split1['generation_kwargs'], dict) assert isinstance(split2['generation_kwargs'], dict) @@ -1326,7 +1326,7 @@ def test_qa_task_dataloader(dataset_uri, tiny_gpt2_tokenizer, tmp_path, num_fews assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == maximum_answer_length + assert batch['generation_kwargs']['max_new_tokens'] == maximum_answer_length assert all(item[0] == tokenizer.eos_token_id for item in batch['input_ids']) decoded_batch = tokenizer.batch_decode(batch['input_ids']) @@ -1376,7 +1376,7 @@ def test_qa_task_with_cot_dataloader(dataset_uri, tiny_gpt2_tokenizer, tmp_path, assert tuple(batch['attention_mask'].shape) == (batch_size, seqlen - maximum_answer_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == maximum_answer_length + assert batch['generation_kwargs']['max_new_tokens'] == maximum_answer_length assert all(item[0] == tokenizer.eos_token_id for item in batch['input_ids']) decoded_batch = tokenizer.batch_decode(batch['input_ids']) assert all(item.count('Q: ') == num_fewshot + 1 for item in decoded_batch) @@ -1505,8 +1505,8 @@ def test_code_eval_split_batch(dataset_uri, tmp_path): assert isinstance(split1['pass_at_k'], int) assert isinstance(split2['pass_at_k'], int) - assert isinstance(split1['generation_length'], int) - assert isinstance(split2['generation_length'], int) + assert isinstance(split1['generation_kwargs']['max_new_tokens'], int) + assert isinstance(split2['generation_kwargs']['max_new_tokens'], int) assert isinstance(split1['generation_kwargs'], dict) assert isinstance(split2['generation_kwargs'], dict) @@ -1552,7 +1552,7 @@ def test_code_eval_sentpiece_dataloader(dataset_uri, tmp_path, num_fewshot, prom assert tuple(batch['attention_mask'].shape) == (batch_size, max_prompt_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == 129 + assert batch['generation_kwargs']['max_new_tokens'] == 129 assert any(item[0] != tokenizer.eos_token_id for item in batch['input_ids']) # longest should be pushed left decoded_batch = tokenizer.batch_decode(batch['input_ids']) @@ -1618,7 +1618,7 @@ def test_code_eval_test_cases(dataset_uri, tmp_path, tiny_llama_tokenizer): assert tuple(batch['attention_mask'].shape) == (batch_size, max_prompt_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == 129 + assert batch['generation_kwargs']['max_new_tokens'] == 129 assert any(item[0] != tokenizer.eos_token_id for item in batch['input_ids']) # longest should be pushed left mod = types.ModuleType('test_module') @@ -1702,7 +1702,7 @@ def test_code_eval_task_dataloader(dataset_uri, tmp_path, num_fewshot, prompt_st assert tuple(batch['attention_mask'].shape) == (batch_size, max_prompt_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == 122 + assert batch['generation_kwargs']['max_new_tokens'] == 122 assert any(item[0] != tokenizer.eos_token_id for item in batch['input_ids']) # longest should be pushed left decoded_batch = tokenizer.batch_decode(batch['input_ids']) @@ -2456,7 +2456,7 @@ def test_hf_dataloading_custom_parsing(dataset_uri, tiny_gpt2_tokenizer, tmp_pat assert tuple(batch['attention_mask'].shape) == (batch_size, seqlen - maximum_answer_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == maximum_answer_length + assert batch['generation_kwargs']['max_new_tokens'] == maximum_answer_length assert all(item[0] == tokenizer.eos_token_id for item in batch['input_ids']) decoded_batch = tokenizer.batch_decode(batch['input_ids']) diff --git a/tests/models/test_hf_model.py b/tests/models/test_hf_model.py index e677941e9e..50387d5e59 100644 --- a/tests/models/test_hf_model.py +++ b/tests/models/test_hf_model.py @@ -1196,10 +1196,10 @@ def test_eval_forward_generate(device, world_size, hf_model, hf_tokenizer, use_f input_dict[k] = device.tensor_to_device(v) input_dict['mode'] = 'generate' - input_dict['generation_length'] = 5 + input_dict['generation_kwargs']['max_new_tokens'] = 5 input_dict['labels'] = [['answer1'], ['answer2']] generation1 = model.eval_forward(input_dict, None) - input_dict['generation_length'] = 3 + input_dict['generation_kwargs']['max_new_tokens'] = 3 input_dict['labels'] = [['answer1'], ['answer2']] generation2 = model.eval_forward(input_dict, None) From a441c1e741447cd971d77d73292a2900348224b4 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Thu, 15 Feb 2024 22:13:04 +0000 Subject: [PATCH 02/11] fix tests --- tests/datasets/test_in_context_learning_datasets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/datasets/test_in_context_learning_datasets.py b/tests/datasets/test_in_context_learning_datasets.py index 56067fb1b3..7d85be2aab 100644 --- a/tests/datasets/test_in_context_learning_datasets.py +++ b/tests/datasets/test_in_context_learning_datasets.py @@ -1491,10 +1491,11 @@ def test_code_eval_split_batch(dataset_uri, tmp_path): assert len(batch[field]) == size assert all(isinstance(val, type_) for val in batch[field]) - static_keys = {'pass_at_k': (int, list), 'generation_length': int, 'generation_kwargs': dict} + static_keys = {'pass_at_k': (int, list), 'generation_kwargs': dict} for batch in batches: for field, type_ in static_keys.items(): assert isinstance(batch[field], type_) + assert batch['generation_kwargs']['max_new_tokens'] == 122 @pytest.mark.parametrize('dataset_uri', ['human_eval_small.jsonl']) @@ -1544,7 +1545,7 @@ def test_code_eval_sentpiece_dataloader(dataset_uri, tmp_path, num_fewshot, prom assert tuple(batch['attention_mask'].shape) == (bs, max_prompt_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == 129 + assert batch['generation_kwargs']['max_new_tokens'] == 129 has_left_padding.extend([item[0] == tokenizer.eos_token_id for item in batch['input_ids']]) assert not all(has_left_padding) # longest should be pushed left @@ -1703,7 +1704,7 @@ def test_code_eval_task_dataloader(dataset_uri, tmp_path, num_fewshot, prompt_st assert tuple(batch['attention_mask'].shape) == (bs, max_prompt_length) assert batch['mode'] == 'generate' # the maximum generation length from the small test data - assert batch['generation_length'] == 122 + assert batch['generation_kwargs']['max_new_tokens'] == 122 has_left_padding.extend([item[0] == tokenizer.eos_token_id for item in batch['input_ids']]) assert not all(has_left_padding) # longest should be pushed left From 1d0a7feb69531fdbd50aa7c5f6170a70203e2869 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Fri, 16 Feb 2024 19:05:41 +0000 Subject: [PATCH 03/11] fix test --- tests/models/test_hf_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/test_hf_model.py b/tests/models/test_hf_model.py index 50387d5e59..f22143d4d0 100644 --- a/tests/models/test_hf_model.py +++ b/tests/models/test_hf_model.py @@ -1195,6 +1195,7 @@ def test_eval_forward_generate(device, world_size, hf_model, hf_tokenizer, use_f for k, v in input_dict.items(): input_dict[k] = device.tensor_to_device(v) input_dict['mode'] = 'generate' + input_dict['generation_kwargs'] = {} input_dict['generation_kwargs']['max_new_tokens'] = 5 input_dict['labels'] = [['answer1'], ['answer2']] From ea3a63eabdad7505e14272b1556d6c1053a13558 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Fri, 16 Feb 2024 23:13:51 +0000 Subject: [PATCH 04/11] add deprecation warning --- composer/models/huggingface.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/composer/models/huggingface.py b/composer/models/huggingface.py index 359092b6d9..5651173927 100644 --- a/composer/models/huggingface.py +++ b/composer/models/huggingface.py @@ -466,9 +466,19 @@ def eval_forward(self, batch, outputs: Optional[Any] = None): raise ValueError( 'Generation eval cannot be used without providing a tokenizer to the model constructor.') + if 'generation_length' in batch: + warnings.warn( + ('`generation_length` has been deprecated in favor of passing `max_new_tokens` directly into `generation_kwargs`.' + 'It will be removed in v0.21'), + DeprecationWarning, + ) + self.labels = batch.pop('labels') generation = self.generate(batch['input_ids'], attention_mask=batch['attention_mask'], + max_new_tokens=batch.get('generation_kwargs', + {}).pop('max_new_tokens', + batch.get('generation_length', None)), synced_gpus=dist.get_world_size() > 1, **batch.get('generation_kwargs', {})) From 4e98bfadb8e5dde06811e140f27348a18df789d9 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Tue, 20 Feb 2024 19:43:27 +0000 Subject: [PATCH 05/11] fix test --- composer/models/huggingface.py | 9 +++++++++ tests/datasets/test_in_context_learning_datasets.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/composer/models/huggingface.py b/composer/models/huggingface.py index 5651173927..c1a31b123f 100644 --- a/composer/models/huggingface.py +++ b/composer/models/huggingface.py @@ -472,6 +472,10 @@ def eval_forward(self, batch, outputs: Optional[Any] = None): 'It will be removed in v0.21'), DeprecationWarning, ) + max_new_tokens = None + if 'generation_kwargs' in batch: + if 'max_new_tokens' in batch['generation_kwargs']: + max_new_tokens = batch['generation_kwargs']['max_new_tokens'] self.labels = batch.pop('labels') generation = self.generate(batch['input_ids'], @@ -482,6 +486,11 @@ def eval_forward(self, batch, outputs: Optional[Any] = None): synced_gpus=dist.get_world_size() > 1, **batch.get('generation_kwargs', {})) + # Need to pop `max_new_tokens` from batch['generation_kwargs'] to avoid passing in the same kwarg twice + # NOTE: Should be deprecated in v0.21 + if max_new_tokens: + batch['generation_kwargs']['max_new_tokens'] = max_new_tokens + # don't remove prefix space to sentencepiece models if len(self.tokenizer( ' a', add_special_tokens=False)['input_ids']) == 1: # pyright: ignore[reportGeneralTypeIssues] diff --git a/tests/datasets/test_in_context_learning_datasets.py b/tests/datasets/test_in_context_learning_datasets.py index 7d85be2aab..66d14cc76f 100644 --- a/tests/datasets/test_in_context_learning_datasets.py +++ b/tests/datasets/test_in_context_learning_datasets.py @@ -1493,9 +1493,11 @@ def test_code_eval_split_batch(dataset_uri, tmp_path): static_keys = {'pass_at_k': (int, list), 'generation_kwargs': dict} for batch in batches: + assert 'generation_kwargs' in batch + assert 'max_new_tokens' in batch['generation_kwargs'] + assert isinstance(batch['generation_kwargs']['max_new_tokens'], int) for field, type_ in static_keys.items(): assert isinstance(batch[field], type_) - assert batch['generation_kwargs']['max_new_tokens'] == 122 @pytest.mark.parametrize('dataset_uri', ['human_eval_small.jsonl']) From de47894b5f91ea1d216518f047122221e5a77fe0 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Wed, 21 Feb 2024 00:57:28 +0000 Subject: [PATCH 06/11] add gen_len back into static_keys --- composer/datasets/in_context_learning_evaluation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/composer/datasets/in_context_learning_evaluation.py b/composer/datasets/in_context_learning_evaluation.py index 85e21e749e..08371e3fb6 100644 --- a/composer/datasets/in_context_learning_evaluation.py +++ b/composer/datasets/in_context_learning_evaluation.py @@ -689,7 +689,9 @@ def __init__( self.cot_delimiter = cot_delimiter self.has_cot = False self.max_answer_length = 0 - static_keys = ['mode', 'cot_delimiter', 'generation_kwargs', 'do_normalization', 'stopping_criteria'] + static_keys = [ + 'mode', 'cot_delimiter', 'generation_length', 'generation_kwargs', 'do_normalization', 'stopping_criteria' + ] tensor_keys = ['input_ids', 'attention_mask'] list_keys = ['labels'] super().__init__( @@ -1297,6 +1299,7 @@ def __init__( static_keys = [ 'mode', 'pass_at_k', + 'generation_length', 'generation_kwargs', 'generations_per_sample', 'dataset_size', From 4599ae75a5d6959c4153a60920495c0e9935bd82 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Wed, 21 Feb 2024 02:36:57 +0000 Subject: [PATCH 07/11] simplify setting variable in forward and add test --- composer/models/huggingface.py | 16 +++------ tests/models/test_hf_model.py | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/composer/models/huggingface.py b/composer/models/huggingface.py index c1a31b123f..07149e4ce6 100644 --- a/composer/models/huggingface.py +++ b/composer/models/huggingface.py @@ -472,25 +472,17 @@ def eval_forward(self, batch, outputs: Optional[Any] = None): 'It will be removed in v0.21'), DeprecationWarning, ) - max_new_tokens = None - if 'generation_kwargs' in batch: - if 'max_new_tokens' in batch['generation_kwargs']: - max_new_tokens = batch['generation_kwargs']['max_new_tokens'] + if 'generation_kwargs' in batch: + batch['generation_kwargs']['max_new_tokens'] = batch['generation_length'] + else: + batch['generation_kwargs'] = {'max_new_tokens': batch['generation_length']} self.labels = batch.pop('labels') generation = self.generate(batch['input_ids'], attention_mask=batch['attention_mask'], - max_new_tokens=batch.get('generation_kwargs', - {}).pop('max_new_tokens', - batch.get('generation_length', None)), synced_gpus=dist.get_world_size() > 1, **batch.get('generation_kwargs', {})) - # Need to pop `max_new_tokens` from batch['generation_kwargs'] to avoid passing in the same kwarg twice - # NOTE: Should be deprecated in v0.21 - if max_new_tokens: - batch['generation_kwargs']['max_new_tokens'] = max_new_tokens - # don't remove prefix space to sentencepiece models if len(self.tokenizer( ' a', add_special_tokens=False)['input_ids']) == 1: # pyright: ignore[reportGeneralTypeIssues] diff --git a/tests/models/test_hf_model.py b/tests/models/test_hf_model.py index f22143d4d0..8e88bc45da 100644 --- a/tests/models/test_hf_model.py +++ b/tests/models/test_hf_model.py @@ -1209,6 +1209,67 @@ def test_eval_forward_generate(device, world_size, hf_model, hf_tokenizer, use_f assert all(isinstance(decoded_generation, str) for decoded_generation in generation2) +@device('cpu', 'gpu') +@world_size(1, 2) +@pytest.mark.parametrize('use_fsdp', [True, False]) +@pytest.mark.parametrize('hf_model,hf_tokenizer', [(configure_tiny_gpt2_model, configure_tiny_gpt2_tokenizer), + (configure_tiny_t5_model, configure_tiny_t5_tokenizer)]) +def test_eval_forward_generate_adjust_generation_length(device, world_size, hf_model, hf_tokenizer, use_fsdp): + transformers = pytest.importorskip('transformers') + if device == 'cpu' and use_fsdp: + pytest.skip('FSDP is not supported on CPU.') + if world_size == 1 and use_fsdp: + pytest.xfail(( + 'Generation with world size 1 and FSDP fails with ' + '`RuntimeError: The tensor has a non-zero number of elements, ' + 'but its data is not allocated yet. Caffe2 uses a lazy allocation, ' + 'so you will need to call mutable_data() or raw_mutable_data() to actually allocate memory.` ' + 'This issue is resolved with world size > 1 by a dummy call to forward (see HuggingFaceModel.dummy_forward_called), ' + 'but for some reason fails with world size 1.')) + + hf_model = hf_model() + if device == 'cpu' and world_size > 1 and isinstance(hf_model, + transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel): + pytest.xfail( + 'GPT2 is not currently supported with DDP. See https://github.com/huggingface/transformers/issues/22482 for more details.' + ) + + fsdp_config = None + if use_fsdp: + fsdp_config = { + 'sharding_strategy': 'FULL_SHARD', + } + + hf_tokenizer = hf_tokenizer() + + model = HuggingFaceModel(hf_model, tokenizer=hf_tokenizer, use_logits=True) + + # just instantiating Trainer to go through the normal FSDP code path + trainer = Trainer(model=model, fsdp_config=fsdp_config, device=device) + + device = trainer.state.device + + if isinstance(hf_tokenizer, transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast): + hf_tokenizer.padding_side = 'left' + input_dict = hf_tokenizer(['hello', 'goodbyes'], return_tensors='pt', padding=True) + for k, v in input_dict.items(): + input_dict[k] = device.tensor_to_device(v) + input_dict['mode'] = 'generate' + input_dict['generation_kwargs'] = {} + input_dict['generation_length'] = 5 + input_dict['labels'] = [['answer1'], ['answer2']] + with pytest.raises(DeprecationWarning): + generation1 = model.eval_forward(input_dict, None) + + input_dict['generation_length'] = 3 + input_dict['labels'] = [['answer1'], ['answer2']] + generation2 = model.eval_forward(input_dict, None) + + assert len(generation1) == len(generation2) == 2 + assert all(isinstance(decoded_generation, str) for decoded_generation in generation1) + assert all(isinstance(decoded_generation, str) for decoded_generation in generation2) + + @pytest.mark.parametrize('peft_type', ['LORA', 'loRa']) @pytest.mark.parametrize('task_type', ['CAUSAL_LM', 'causal_lm']) def test_peft_init(peft_type: str, task_type: str, tiny_gpt2_model, gpt2_peft_config): From a17d3fbefa044ed2f815ea23c2c61cf36310c17e Mon Sep 17 00:00:00 2001 From: Max Marion Date: Wed, 21 Feb 2024 21:34:29 +0000 Subject: [PATCH 08/11] simply test --- tests/models/test_hf_model.py | 49 +++-------------------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/tests/models/test_hf_model.py b/tests/models/test_hf_model.py index 8e88bc45da..56bc13106b 100644 --- a/tests/models/test_hf_model.py +++ b/tests/models/test_hf_model.py @@ -1209,56 +1209,15 @@ def test_eval_forward_generate(device, world_size, hf_model, hf_tokenizer, use_f assert all(isinstance(decoded_generation, str) for decoded_generation in generation2) -@device('cpu', 'gpu') -@world_size(1, 2) -@pytest.mark.parametrize('use_fsdp', [True, False]) -@pytest.mark.parametrize('hf_model,hf_tokenizer', [(configure_tiny_gpt2_model, configure_tiny_gpt2_tokenizer), - (configure_tiny_t5_model, configure_tiny_t5_tokenizer)]) -def test_eval_forward_generate_adjust_generation_length(device, world_size, hf_model, hf_tokenizer, use_fsdp): - transformers = pytest.importorskip('transformers') - if device == 'cpu' and use_fsdp: - pytest.skip('FSDP is not supported on CPU.') - if world_size == 1 and use_fsdp: - pytest.xfail(( - 'Generation with world size 1 and FSDP fails with ' - '`RuntimeError: The tensor has a non-zero number of elements, ' - 'but its data is not allocated yet. Caffe2 uses a lazy allocation, ' - 'so you will need to call mutable_data() or raw_mutable_data() to actually allocate memory.` ' - 'This issue is resolved with world size > 1 by a dummy call to forward (see HuggingFaceModel.dummy_forward_called), ' - 'but for some reason fails with world size 1.')) - - hf_model = hf_model() - if device == 'cpu' and world_size > 1 and isinstance(hf_model, - transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel): - pytest.xfail( - 'GPT2 is not currently supported with DDP. See https://github.com/huggingface/transformers/issues/22482 for more details.' - ) - - fsdp_config = None - if use_fsdp: - fsdp_config = { - 'sharding_strategy': 'FULL_SHARD', - } - - hf_tokenizer = hf_tokenizer() - - model = HuggingFaceModel(hf_model, tokenizer=hf_tokenizer, use_logits=True) - - # just instantiating Trainer to go through the normal FSDP code path - trainer = Trainer(model=model, fsdp_config=fsdp_config, device=device) - - device = trainer.state.device +def test_eval_forward_generate_adjust_generation_length(tiny_gpt2_model, tiny_gpt2_tokenizer): + model = HuggingFaceModel(tiny_gpt2_model, tokenizer=tiny_gpt2_tokenizer, use_logits=True) + input_dict = tiny_gpt2_tokenizer(['hello', 'goodbyes'], return_tensors='pt', padding=True) - if isinstance(hf_tokenizer, transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast): - hf_tokenizer.padding_side = 'left' - input_dict = hf_tokenizer(['hello', 'goodbyes'], return_tensors='pt', padding=True) - for k, v in input_dict.items(): - input_dict[k] = device.tensor_to_device(v) input_dict['mode'] = 'generate' input_dict['generation_kwargs'] = {} input_dict['generation_length'] = 5 input_dict['labels'] = [['answer1'], ['answer2']] - with pytest.raises(DeprecationWarning): + with pytest.warns(DeprecationWarning): generation1 = model.eval_forward(input_dict, None) input_dict['generation_length'] = 3 From 06c352ae897461927f761ba058cf97fbb7824375 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Wed, 21 Feb 2024 21:36:12 +0000 Subject: [PATCH 09/11] trailing comma --- composer/datasets/in_context_learning_evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer/datasets/in_context_learning_evaluation.py b/composer/datasets/in_context_learning_evaluation.py index 08371e3fb6..e39e61cc95 100644 --- a/composer/datasets/in_context_learning_evaluation.py +++ b/composer/datasets/in_context_learning_evaluation.py @@ -690,7 +690,7 @@ def __init__( self.has_cot = False self.max_answer_length = 0 static_keys = [ - 'mode', 'cot_delimiter', 'generation_length', 'generation_kwargs', 'do_normalization', 'stopping_criteria' + 'mode', 'cot_delimiter', 'generation_length', 'generation_kwargs', 'do_normalization', 'stopping_criteria', ] tensor_keys = ['input_ids', 'attention_mask'] list_keys = ['labels'] From faefc3d2d73e8ae8259a465385babf0a5451e978 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Wed, 21 Feb 2024 21:40:08 +0000 Subject: [PATCH 10/11] trailing comma --- composer/datasets/in_context_learning_evaluation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/composer/datasets/in_context_learning_evaluation.py b/composer/datasets/in_context_learning_evaluation.py index e39e61cc95..38c73bd876 100644 --- a/composer/datasets/in_context_learning_evaluation.py +++ b/composer/datasets/in_context_learning_evaluation.py @@ -690,7 +690,12 @@ def __init__( self.has_cot = False self.max_answer_length = 0 static_keys = [ - 'mode', 'cot_delimiter', 'generation_length', 'generation_kwargs', 'do_normalization', 'stopping_criteria', + 'mode', + 'cot_delimiter', + 'generation_length', + 'generation_kwargs', + 'do_normalization', + 'stopping_criteria', ] tensor_keys = ['input_ids', 'attention_mask'] list_keys = ['labels'] From 2ba9dbddb264b187ef088c874b77db0076b5e426 Mon Sep 17 00:00:00 2001 From: Max Marion Date: Thu, 22 Feb 2024 00:36:15 +0000 Subject: [PATCH 11/11] linting --- composer/utils/object_store/oci_object_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/utils/object_store/oci_object_store.py b/composer/utils/object_store/oci_object_store.py index 72898464cc..cf4b78bdbd 100644 --- a/composer/utils/object_store/oci_object_store.py +++ b/composer/utils/object_store/oci_object_store.py @@ -152,7 +152,8 @@ def download_object( object_size = 0 try: head_object_response = self.client.head_object(self.namespace, self.bucket, object_name) - object_size = int(head_object_response.headers['content-length']) # pyright: ignore[reportOptionalMemberAccess] + object_size = int( + head_object_response.headers['content-length']) # pyright: ignore[reportOptionalMemberAccess] except Exception as e: _reraise_oci_errors(self.get_uri(object_name), e)