diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index e040884b5ef..60a2eda44cd 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -19,12 +19,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-upstream-image:
-    name: "Build upstream Docker image"
-    uses: ./.github/workflows/_build_upstream_image.yml
-    with:
-      ecr-docker-image-base: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/xla_base
-    if: github.event_name == 'push' && github.event.ref == 'refs/heads/master'
 
   build-torch-xla:
     name: "Build PyTorch/XLA"
diff --git a/.github/workflows/_build_upstream_image.yml b/.github/workflows/build_upstream_image.yml
similarity index 69%
rename from .github/workflows/_build_upstream_image.yml
rename to .github/workflows/build_upstream_image.yml
index ef0975b6abf..446ad366e54 100644
--- a/.github/workflows/_build_upstream_image.yml
+++ b/.github/workflows/build_upstream_image.yml
@@ -1,22 +1,18 @@
-name: xla-buld
+name: Build upstream image
 on:
-  workflow_call:
-    inputs:
-      ecr-docker-image-base:
-        required: true
-        type: string
-        description: Container registry to upload image to
-      runner:
-        required: false
-        type: string
-        description: Runner type for the test
-        default: linux.12xlarge
+  push:
+    branches:
+      - master
+      - r[0-9]+.[0-9]+
+    paths-ignore:
+      - 'experimental/torch_xla2/**'
+  workflow_dispatch:
 jobs:
   build:
-    runs-on: ${{ inputs.runner }}
-    timeout-minutes: 240
+    runs-on: linux.12xlarge
+    timeout-minutes: 30
     env:
-      ECR_DOCKER_IMAGE_BASE: ${{ inputs.ecr-docker-image-base }}
+      ECR_DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/xla_base
       BAZEL_JOBS: 16
     steps:
       # See https://github.com/actions/checkout/issues/1014#issuecomment-1906802802
@@ -28,7 +24,7 @@ jobs:
         uses: pytorch/test-infra/.github/actions/setup-linux@main
       - name: Checkout repo
         uses: actions/checkout@v3
-      - name: Download docker image from GCR
+      - name: Build Docker image
         shell: bash
         run: |
           docker build -t "${ECR_DOCKER_IMAGE_BASE}:v1.2-lite" .github/upstream