From 625666f4a5a17daa9e5cf5c05934b79d80cea036 Mon Sep 17 00:00:00 2001
From: binmakeswell <binmakeswell@gmail.com>
Date: Tue, 23 Apr 2024 18:48:07 +0800
Subject: [PATCH] [example] llama3 (#5631)

* release llama3

* [release] llama3

* [release] llama3

* [release] llama3

* [release] llama3
---
 README.md                                          | 14 +++++++++++---
 docs/README-zh-Hans.md                             | 10 +++++++++-
 examples/language/{llama2 => llama}/README.md      |  6 ++++++
 examples/language/{llama2 => llama}/benchmark.py   |  0
 .../language/{llama2 => llama}/requirements.txt    |  0
 .../{llama2 => llama}/scripts/benchmark_70B/3d.sh  |  0
 .../scripts/benchmark_70B/gemini.sh                |  0
 .../scripts/benchmark_70B/gemini_auto.sh           |  0
 .../scripts/benchmark_7B/gemini.sh                 |  0
 .../scripts/benchmark_7B/gemini_auto.sh            |  0
 examples/language/{llama2 => llama}/test_ci.sh     |  0
 11 files changed, 26 insertions(+), 4 deletions(-)
 rename examples/language/{llama2 => llama}/README.md (94%)
 rename examples/language/{llama2 => llama}/benchmark.py (100%)
 rename examples/language/{llama2 => llama}/requirements.txt (100%)
 rename examples/language/{llama2 => llama}/scripts/benchmark_70B/3d.sh (100%)
 rename examples/language/{llama2 => llama}/scripts/benchmark_70B/gemini.sh (100%)
 rename examples/language/{llama2 => llama}/scripts/benchmark_70B/gemini_auto.sh (100%)
 rename examples/language/{llama2 => llama}/scripts/benchmark_7B/gemini.sh (100%)
 rename examples/language/{llama2 => llama}/scripts/benchmark_7B/gemini_auto.sh (100%)
 rename examples/language/{llama2 => llama}/test_ci.sh (100%)

diff --git a/README.md b/README.md
index 26776bdf6d9f..c1e2da0d406f 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@
  <li>
    <a href="#Parallel-Training-Demo">Parallel Training Demo</a>
    <ul>
-     <li><a href="#LLaMA2">LLaMA 1/2</a></li>
+     <li><a href="#LLaMA3">LLaMA 1/2/3 </a></li>
      <li><a href="#MoE">MoE</a></li>
      <li><a href="#GPT-3">GPT-3</a></li>
      <li><a href="#GPT-2">GPT-2</a></li>
@@ -270,13 +270,21 @@ Acceleration of [AlphaFold Protein Structure](https://alphafold.ebi.ac.uk/)
 <p align="right">(<a href="#top">back to top</a>)</p>
 
 ## Parallel Training Demo
+### LLaMA3
+<p align="center">
+<img src="https://raw.githubusercontent.com/hpcaitech/public_assets/main/examples/images/LLaMA3-70B-H100.png" width=600/>
+</p>
+
+- 70 billion parameter LLaMA3 model training accelerated by 18%
+[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama)
+
 ### LLaMA2
 <p align="center">
 <img src="https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/img/llama2_pretraining.png" width=600/>
 </p>
 
 - 70 billion parameter LLaMA2 model training accelerated by 195%
-[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama2)
+[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama)
 [[blog]](https://www.hpc-ai.tech/blog/70b-llama2-training)
 
 ### LLaMA1
@@ -285,7 +293,7 @@ Acceleration of [AlphaFold Protein Structure](https://alphafold.ebi.ac.uk/)
 </p>
 
 - 65-billion-parameter large model pretraining accelerated by 38%
-[[code]](https://github.com/hpcaitech/ColossalAI/tree/example/llama/examples/language/llama)
+[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama)
 [[blog]](https://www.hpc-ai.tech/blog/large-model-pretraining)
 
 ### MoE
diff --git a/docs/README-zh-Hans.md b/docs/README-zh-Hans.md
index 6d243a80852d..7e0ed07fec16 100644
--- a/docs/README-zh-Hans.md
+++ b/docs/README-zh-Hans.md
@@ -51,7 +51,7 @@
  <li>
    <a href="#并行训练样例展示">并行训练样例展示</a>
    <ul>
-     <li><a href="#LLaMA2">LLaMA 1/2</a></li>
+     <li><a href="#LLaMA3">LLaMA 1/2/3</a></li>
      <li><a href="#MoE">MoE</a></li>
      <li><a href="#GPT-3">GPT-3</a></li>
      <li><a href="#GPT-2">GPT-2</a></li>
@@ -261,6 +261,14 @@ Colossal-AI 为您提供了一系列并行组件。我们的目标是让您的
 <p align="right">(<a href="#top">返回顶端</a>)</p>
 
 ## 并行训练样例展示
+### LLaMA3
+<p align="center">
+<img src="https://raw.githubusercontent.com/hpcaitech/public_assets/main/examples/images/LLaMA3-70B-H100.png" width=600/>
+</p>
+
+- 700亿参数LLaMA3训练加速18%
+[[code]](https://github.com/hpcaitech/ColossalAI/tree/main/examples/language/llama)
+
 ### LLaMA2
 <p align="center">
 <img src="https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/img/llama2_pretraining.png" width=600/>
diff --git a/examples/language/llama2/README.md b/examples/language/llama/README.md
similarity index 94%
rename from examples/language/llama2/README.md
rename to examples/language/llama/README.md
index 11b2ee511a6e..fa0c6dc07156 100644
--- a/examples/language/llama2/README.md
+++ b/examples/language/llama/README.md
@@ -1,4 +1,10 @@
 # Pretraining LLaMA-1/2/3: best practices for building LLaMA-1/2/3-like base models
+### LLaMA3
+<p align="center">
+<img src="https://raw.githubusercontent.com/hpcaitech/public_assets/main/examples/images/LLaMA3-70B-H100.png" width=600/>
+</p>
+
+- 70 billion parameter LLaMA3 model training accelerated by 18%
 
 ### LLaMA2
 <p align="center">
diff --git a/examples/language/llama2/benchmark.py b/examples/language/llama/benchmark.py
similarity index 100%
rename from examples/language/llama2/benchmark.py
rename to examples/language/llama/benchmark.py
diff --git a/examples/language/llama2/requirements.txt b/examples/language/llama/requirements.txt
similarity index 100%
rename from examples/language/llama2/requirements.txt
rename to examples/language/llama/requirements.txt
diff --git a/examples/language/llama2/scripts/benchmark_70B/3d.sh b/examples/language/llama/scripts/benchmark_70B/3d.sh
similarity index 100%
rename from examples/language/llama2/scripts/benchmark_70B/3d.sh
rename to examples/language/llama/scripts/benchmark_70B/3d.sh
diff --git a/examples/language/llama2/scripts/benchmark_70B/gemini.sh b/examples/language/llama/scripts/benchmark_70B/gemini.sh
similarity index 100%
rename from examples/language/llama2/scripts/benchmark_70B/gemini.sh
rename to examples/language/llama/scripts/benchmark_70B/gemini.sh
diff --git a/examples/language/llama2/scripts/benchmark_70B/gemini_auto.sh b/examples/language/llama/scripts/benchmark_70B/gemini_auto.sh
similarity index 100%
rename from examples/language/llama2/scripts/benchmark_70B/gemini_auto.sh
rename to examples/language/llama/scripts/benchmark_70B/gemini_auto.sh
diff --git a/examples/language/llama2/scripts/benchmark_7B/gemini.sh b/examples/language/llama/scripts/benchmark_7B/gemini.sh
similarity index 100%
rename from examples/language/llama2/scripts/benchmark_7B/gemini.sh
rename to examples/language/llama/scripts/benchmark_7B/gemini.sh
diff --git a/examples/language/llama2/scripts/benchmark_7B/gemini_auto.sh b/examples/language/llama/scripts/benchmark_7B/gemini_auto.sh
similarity index 100%
rename from examples/language/llama2/scripts/benchmark_7B/gemini_auto.sh
rename to examples/language/llama/scripts/benchmark_7B/gemini_auto.sh
diff --git a/examples/language/llama2/test_ci.sh b/examples/language/llama/test_ci.sh
similarity index 100%
rename from examples/language/llama2/test_ci.sh
rename to examples/language/llama/test_ci.sh