Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add dyanmic padding to scripts/eval.py
Browse files Browse the repository at this point in the history
moeiniamir committed Apr 18, 2024
1 parent 4baf419 commit 448a14e
Showing 2 changed files with 30 additions and 1 deletion.
14 changes: 13 additions & 1 deletion scripts/eval/README.md
Original file line number Diff line number Diff line change
@@ -483,4 +483,16 @@ model:
marlin_path: ${marlin_path}
```

And run the gauntlet as you normally would.
And run the gauntlet as you normally would.

# Dynamic Padding
Add `trim_batch` to model's yaml similar to below:

```yaml
models:
-
trim_batch: true
...
```

This will dynamically pad the batch to the maximum sequence length in the batch in contrast to padding to the maximum sequence length in the config. Obtains a speedup up to 10x for small batch sizes.
17 changes: 17 additions & 0 deletions scripts/eval/eval.py
Original file line number Diff line number Diff line change
@@ -75,6 +75,23 @@ def evaluate_model(
icl_seq_len=max_seq_len,
icl_subset_num_batches=icl_subset_num_batches,
)

if model_cfg.get('trim_batch', False):
def _trim_batch(batch):
assert batch['attention_mask'].shape[1] == max_seq_len, 'No padding till max_seq_len found, disable batch_trim'
_max_len = batch['attention_mask'].sum(1).max()
for k in batch:
if isinstance(batch[k], torch.Tensor) and batch[k].shape[1] == max_seq_len:
batch[k] = batch[k][:, :_max_len]
return batch
def _trim_collate_decorator(collate_fn):
def _trim_collate(batch):
return _trim_batch(collate_fn(batch))
return _trim_collate
for evaluator in evaluators:
evaluator.dataloader.dataloader.collate_fn = _trim_collate_decorator(
evaluator.dataloader.dataloader.collate_fn
)

# Callbacks
callbacks: List[Callback] = [

0 comments on commit 448a14e

Please sign in to comment.