diff --git a/.github/workflows/pr-gpu.yaml b/.github/workflows/pr-gpu.yaml
index 07e811e244..1d8323ba9f 100644
--- a/.github/workflows/pr-gpu.yaml
+++ b/.github/workflows/pr-gpu.yaml
@@ -13,18 +13,18 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 jobs:
-  pytest-gpu:
-    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.5
+  pytest-gpu-1:
+    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.8
     strategy:
       fail-fast: false
       matrix:
         include:
-        - name: "gpu-2.3.0"
+        - name: "gpu-2.3.0-1"
           container: mosaicml/pytorch:2.3.0_cu121-python3.11-ubuntu20.04
           markers: "gpu"
           pytest_command: "coverage run -m pytest"
           pip_deps: "[all]"
-        - name: "gpu-2.3.1"
+        - name: "gpu-2.3.1-1"
           container: mosaicml/llm-foundry:2.3.1_cu121-latest
           markers: "gpu"
           pytest_command: "coverage run -m pytest"
@@ -40,5 +40,66 @@ jobs:
       pytest-command: ${{ matrix.pytest_command }}
       pytest-markers: ${{ matrix.markers }}
       python-version: 3.9
+      gpu_num: 1
+    secrets:
+      mcloud-api-key: ${{ secrets.MCLOUD_API_KEY }}
+  pytest-gpu-2:
+    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.8
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - name: "gpu-2.3.0-2"
+          container: mosaicml/pytorch:2.3.0_cu121-python3.11-ubuntu20.04
+          markers: "gpu"
+          pytest_command: "coverage run -m pytest"
+          pip_deps: "[all]"
+        - name: "gpu-2.3.1-2"
+          container: mosaicml/llm-foundry:2.3.1_cu121-latest
+          markers: "gpu"
+          pytest_command: "coverage run -m pytest"
+          pip_deps: "[all]"
+    name: ${{ matrix.name }}
+    if: github.repository_owner == 'mosaicml'
+    with:
+      container: ${{ matrix.container }}
+      git_repo: mosaicml/llm-foundry
+      mcloud-timeout: 1800
+      name: ${{ matrix.name }}
+      pip_deps: ${{ matrix.pip_deps }}
+      pytest-command: ${{ matrix.pytest_command }}
+      pytest-markers: ${{ matrix.markers }}
+      python-version: 3.9
+      gpu_num: 2
+    secrets:
+      mcloud-api-key: ${{ secrets.MCLOUD_API_KEY }}
+  pytest-gpu-4:
+    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.8
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - name: "gpu-2.3.0-4"
+          container: mosaicml/pytorch:2.3.0_cu121-python3.11-ubuntu20.04
+          markers: "gpu"
+          pytest_command: "coverage run -m pytest"
+          pip_deps: "[all]"
+        - name: "gpu-2.3.1-4"
+          container: mosaicml/llm-foundry:2.3.1_cu121-latest
+          markers: "gpu"
+          pytest_command: "coverage run -m pytest"
+          pip_deps: "[all]"
+    name: ${{ matrix.name }}
+    if: github.repository_owner == 'mosaicml'
+    with:
+      container: ${{ matrix.container }}
+      git_repo: mosaicml/llm-foundry
+      mcloud-timeout: 1800
+      name: ${{ matrix.name }}
+      pip_deps: ${{ matrix.pip_deps }}
+      pytest-command: ${{ matrix.pytest_command }}
+      pytest-markers: ${{ matrix.markers }}
+      python-version: 3.9
+      gpu_num: 4
     secrets:
       mcloud-api-key: ${{ secrets.MCLOUD_API_KEY }}