From 37bec75769e0af49f70f42a3f4c202db03d8d347 Mon Sep 17 00:00:00 2001 From: DesmonDay <908660116@qq.com> Date: Thu, 21 Nov 2024 17:41:08 +0800 Subject: [PATCH] fix eval for map dataset --- paddlenlp/trainer/trainer.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/paddlenlp/trainer/trainer.py b/paddlenlp/trainer/trainer.py index c58b8e2a93a0..eb5c5803330b 100644 --- a/paddlenlp/trainer/trainer.py +++ b/paddlenlp/trainer/trainer.py @@ -1588,20 +1588,13 @@ def _get_eval_sampler(self, eval_dataset: Dataset): drop_last=False, ) else: - drop_last = False - if self.args.pipeline_parallel_degree > 1: - drop_last = True - logger.warning( - "In parallel mode, the batch_size is strictly checked. set DistributedBatchSampler drop_last=True." - ) - return DistributedBatchSampler( eval_dataset, num_replicas=self.args.dataset_world_size, rank=self.args.dataset_rank, batch_size=self.args.per_device_eval_batch_size, shuffle=False, - drop_last=drop_last, + drop_last=False, ) def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoader: