adding eps to building norms

mosaicml · Jul 26, 2024 · b8d2f06 · b8d2f06
1 parent 7de4969
commit b8d2f06
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/llmfoundry/models/layers/layer_builders.py b/llmfoundry/models/layers/layer_builders.py
@@ -26,10 +26,12 @@
 def build_norm(
     name: str,
     normalized_shape: Union[int, List[int], torch.Size],
+    eps: Optional[float] = 1e-5,
     device: Optional[str] = None,
 ):
     kwargs = {
         'normalized_shape': normalized_shape,
+        'eps': eps,
         'device': device,
     }