add avlue error

mosaicml · Jan 13, 2024 · a121133 · a121133
1 parent 22ec061
commit a121133
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/llmfoundry/models/layers/attention.py b/llmfoundry/models/layers/attention.py
@@ -547,6 +547,8 @@ def __init__(
             raise ValueError(
                 'Each Q head should get the same number of KV heads, so n_heads must be divisible by kv_n_heads.'
             )
+        if qk_ln and qk_gn:
+            raise ValueError('Only one of qk_ln and qk_gn can be set to True.')
 
         self.softmax_scale = softmax_scale
         if self.softmax_scale is None: