add error message

mosaicml · Apr 1, 2024 · 16b5b0d · 16b5b0d
1 parent 039caec
commit 16b5b0d
Showing 1 changed file with 2 additions and 7 deletions.
diff --git a/llmfoundry/models/mpt/modeling_mpt.py b/llmfoundry/models/mpt/modeling_mpt.py
@@ -1022,14 +1022,9 @@ def loss(self, outputs: CausalLMOutputWithPast,
                               targets)
 
         if torch.all(targets == -100):
-            if torch.any(losses != 0.0):
-                raise ValueError(
-                    f'Losses are non-zero even though all the targets are -100.'
-                )
-            warnings.warn(
-                'All targets are -100. This is likely due to padding tokens. Better sequence packing may reduce the number of padding tokens.'
+            raise ValueError(
+                'All targets are -100. This might be due to all tokens in a sequence being padding tokens.'
             )
-            loss = losses.sum()
         else:
             loss = losses.sum() / (targets != self.loss_fn.ignore_index).sum()