Skip to content

Commit

Permalink
fix eval for map dataset
Browse files Browse the repository at this point in the history
  • Loading branch information
DesmonDay committed Nov 21, 2024
1 parent 6813e40 commit 37bec75
Showing 1 changed file with 1 addition and 8 deletions.
9 changes: 1 addition & 8 deletions paddlenlp/trainer/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1588,20 +1588,13 @@ def _get_eval_sampler(self, eval_dataset: Dataset):
drop_last=False,
)
else:
drop_last = False
if self.args.pipeline_parallel_degree > 1:
drop_last = True
logger.warning(
"In parallel mode, the batch_size is strictly checked. set DistributedBatchSampler drop_last=True."
)

return DistributedBatchSampler(
eval_dataset,
num_replicas=self.args.dataset_world_size,
rank=self.args.dataset_rank,
batch_size=self.args.per_device_eval_batch_size,
shuffle=False,
drop_last=drop_last,
drop_last=False,
)

def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoader:
Expand Down

0 comments on commit 37bec75

Please sign in to comment.