From 607b982d8b5ac0d1eb842b9e3b57998bb55d26d4 Mon Sep 17 00:00:00 2001
From: Daniel King <43149077+dakinggg@users.noreply.github.com>
Date: Thu, 4 Apr 2024 12:00:09 -0700
Subject: [PATCH] Bump transformers to 4.39.3 (#1086)

---
 llmfoundry/models/layers/attention.py | 7 ++++---
 setup.py                              | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/llmfoundry/models/layers/attention.py b/llmfoundry/models/layers/attention.py
index 1deca69eb2..e3ba488f3f 100644
--- a/llmfoundry/models/layers/attention.py
+++ b/llmfoundry/models/layers/attention.py
@@ -501,9 +501,10 @@ def forward(
                 value = value.view(bsz, seqlen, self.kv_n_heads * self.head_dim)
             elif rotary_emb_w_meta_info['impl'] == 'hf':
                 if is_transformers_version_gte('4.38'):
-                    (cos, sin) = rotary_emb(x=value,
-                                            position_ids=offset_info,
-                                            seq_len=None)
+                    (cos, sin) = rotary_emb(
+                        x=value,
+                        position_ids=offset_info,
+                    )
                 else:
                     (cos, sin) = rotary_emb(x=value, seq_len=seq_len)
                 if is_transformers_version_gte('4.38'):
diff --git a/setup.py b/setup.py
index 7cd3ce919c..79511eeca3 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@
     'mosaicml[libcloud,wandb,oci,gcs]>=0.21.1,<0.22',
     'mlflow>=2.10,<3',
     'accelerate>=0.25,<0.26',  # for HF inference `device_map`
-    'transformers>=4.38.2,<4.39',
+    'transformers>=4.39.3,<4.40',
     'mosaicml-streaming>=0.7.4,<0.8',
     'torch>=2.2.1,<2.3',
     'datasets>=2.16,<2.17',