diff --git a/.github/workflows/_build_torch_with_cuda_and_xla.yml b/.github/workflows/_build_torch_with_cuda_and_xla.yml
index 64c3099eb1c..3dfe1be2492 100644
--- a/.github/workflows/_build_torch_with_cuda_and_xla.yml
+++ b/.github/workflows/_build_torch_with_cuda_and_xla.yml
@@ -54,14 +54,11 @@ jobs:
         shell: bash
         run: |
           cd pytorch/xla/infra/ansible
-          ansible-playbook playbook.yaml -vvv -e "stage=build arch=amd64 accelerator=cuda cuda_compute_capabilities=5.2,7.5 src_root=${GITHUB_WORKSPACE} build_cpp_tests=1 git_versioned_xla_build=1 cache_suffix=-ci build_pytorch_with_cuda=1" --skip-tags=fetch_srcs,install_deps
+          ansible-playbook playbook.yaml -vvv -e "stage=build arch=amd64 accelerator=cuda cuda_compute_capabilities=5.2,7.5 src_root=${GITHUB_WORKSPACE} build_cpp_tests=0 git_versioned_xla_build=1 cache_suffix=-ci build_pytorch_with_cuda=1" --skip-tags=fetch_srcs,install_deps
+          # TODO(xw32): remove the line below
+          ls /dist/
       - name: Upload wheel
         uses: actions/upload-artifact@v4
         with:
-          name: torch-xla-wheels
+          name: torch-with-cuda-xla-with-cuda-wheels
           path: /dist/*.whl
-      - name: Upload CPP test binaries
-        uses: actions/upload-artifact@v4
-        with:
-          name: cpp-test-bin
-          path: /tmp/test/bin
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index 8a454cc075b..c8cca2123ac 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -28,6 +28,11 @@ on:
         type: boolean
         default: false
         description: Whether to install CUDA plugin package
+      require-pytorch-cuda:
+        required: false
+        type: boolean
+        default: false
+        description: Whether the tests require pytorch built with CUDA enabled
 
     secrets:
       gcloud-service-key:
@@ -87,6 +92,13 @@ jobs:
         with:
           name: torch-xla-wheels
           path: /tmp/wheels/
+        if: $! {{ require-pytorch-cuda }}
+      - name: Fetch wheels (pytorch and pytorch/xla with CUDA enabled) if needed.
+        uses: actions/download-artifact@v4
+        with:
+          name: torch-with-cuda-xla-with-cuda-wheels
+          path: /tmp/wheels/
+        if: ${{ require-pytorch-cuda }}
       - name: Fetch CPP test binaries
         uses: actions/download-artifact@v4
         with:
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index f359360c25e..91a3c5b754c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -79,6 +79,7 @@ jobs:
       timeout-minutes: 300
       collect-coverage: false
       install-cuda-plugin: true
+      require-pytorch-cuda: true
     secrets:
       gcloud-service-key: ${{ secrets.GCLOUD_SERVICE_KEY }}