Merge branch 'feat/fstp' of github.com:yingtongxiong/InternLM into fe…

…at/fstp
yingtongxiong · Oct 17, 2023 · b51cf4e · b51cf4e
2 parents 6682f5d + 4e99a7f
commit b51cf4e
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/internlm/train/training_internlm.py b/internlm/train/training_internlm.py
@@ -576,4 +576,8 @@ def record_current_batch_training_metrics(
             tgs_list.append(tgs_origin)
         if batch_count == gpc.config.data.total_steps - 1:
             print(tgs_list, flush=True)
+            avg_tgs = sum(tgs_list) / len(tgs_list)
+            for tgs in tgs_list.copy():
+                if abs(tgs - avg_tgs) > 1000:
+                    tgs_list.remove(tgs)
             print(f"avg_tgs: {sum(tgs_list)/len(tgs_list)}", flush=True)