From 3eb146f4eaf5cb68e09b31a2bd8c908d3f195c59 Mon Sep 17 00:00:00 2001
From: Crystalcareai <162942000+Crystalcareai@users.noreply.github.com>
Date: Wed, 3 Jul 2024 13:45:48 -0500
Subject: [PATCH] Add support for Internlm2 (#362)

---
 mergekit/_data/architectures/internlm2.json | 50 +++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 mergekit/_data/architectures/internlm2.json

diff --git a/mergekit/_data/architectures/internlm2.json b/mergekit/_data/architectures/internlm2.json
new file mode 100644
index 00000000..057bc649
--- /dev/null
+++ b/mergekit/_data/architectures/internlm2.json
@@ -0,0 +1,50 @@
+{
+    "model_type": "internlm2",
+    "architectures": [
+        "InternLM2ForCausalLM"
+    ],
+    "pre_weights": [
+        {
+            "name": "model.tok_embeddings.weight",
+            "is_embed": true
+        }
+    ],
+    "post_weights": [
+        {
+            "name": "model.norm.weight"
+        },
+        {
+            "name": "output.weight",
+            "is_embed": true,
+            "aliases": [
+                "model.tok_embeddings.weight"
+            ]
+        }
+    ],
+    "num_layers_config_key": "num_hidden_layers",
+    "layer_templates": {
+        "weights": [
+            {
+                "name": "model.layers.${layer_index}.attention_norm.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.ffn_norm.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.attention.wqkv.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.attention.wo.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.feed_forward.w1.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.feed_forward.w2.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.feed_forward.w3.weight"
+            }
+        ]
+    }
+}