From b8d2f06567e941b9ffcb3fcc55a17faff6a60e25 Mon Sep 17 00:00:00 2001
From: root <abhay.gupta@databricks.com>
Date: Fri, 26 Jul 2024 18:44:56 +0000
Subject: [PATCH] adding eps to building norms

---
 llmfoundry/models/layers/layer_builders.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llmfoundry/models/layers/layer_builders.py b/llmfoundry/models/layers/layer_builders.py
index 69d2059bad..d5fd1d37d4 100644
--- a/llmfoundry/models/layers/layer_builders.py
+++ b/llmfoundry/models/layers/layer_builders.py
@@ -26,10 +26,12 @@
 def build_norm(
     name: str,
     normalized_shape: Union[int, List[int], torch.Size],
+    eps: Optional[float] = 1e-5,
     device: Optional[str] = None,
 ):
     kwargs = {
         'normalized_shape': normalized_shape,
+        'eps': eps,
         'device': device,
     }