[shardformer] add all_reduce operation to loss

add all_reduce operation to loss
hpcaitech · Sep 4, 2023 · c9cfa8c · c9cfa8c
1 parent 9ac6dc6
commit c9cfa8c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/language/bert/finetune.py b/examples/language/bert/finetune.py
@@ -118,7 +118,7 @@ def evaluate_subset(dataloader: DataLoader):
                 metric.add_batch(predictions=preds, references=labels)
 
         results = metric.compute()
-
+        dist.all_reduce(accum_loss.div_(len(dataloader)))
         if coordinator.is_master() and results is not None:
             results['loss'] = accum_loss.item() / coordinator.world_size