change docstring

allenai · May 6, 2024 · 962b983 · 962b983
1 parent 717925e
commit 962b983
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/olmo/config.py b/olmo/config.py
@@ -484,8 +484,7 @@ class OptimizerConfig(BaseConfig):
 
     regularize_embeddings: bool = False
     """
-    Applying weight decay to embeddings may make them too small, potentially causing spikes.
-    Setting this parameter to true is a way of applying "reverse weight decay" to embeddings.
+    Applies a regularizer to the embeddings that tries to pull them towards a standard deviation of 1.
     """
 
     def __post_init__(self):