diff --git a/.github/actions/with-k-docker/Dockerfile b/.github/actions/with-k-docker/Dockerfile
index f169f9ce435..97df8ea4fb5 100644
--- a/.github/actions/with-k-docker/Dockerfile
+++ b/.github/actions/with-k-docker/Dockerfile
@@ -3,6 +3,7 @@ ARG BASE_DISTRO
 FROM ubuntu:${BASE_DISTRO}
 
 ARG K_DEB_PATH
+ARG INSTALL_BACKEND_DEBS
 ARG LLVM_BACKEND_DEB_PATH
 
 COPY ${K_DEB_PATH} /kframework.deb
@@ -15,9 +16,13 @@ RUN    apt-get -y update    \
          make               \
          python3.10-dev     \
          python3.10-venv    \
-         /llvm-backend.deb  \
-         /kframework.deb    \
-    && apt-get -y clean
+         /kframework.deb
+
+RUN if [ "${INSTALL_BACKEND_DEBS}" = "true" ]; then \
+      apt-get -y install /llvm-backend.deb; \
+    fi
+
+RUN apt-get -y clean
 
 RUN rm /llvm-backend.deb
 RUN rm /kframework.deb
diff --git a/.github/actions/with-k-docker/action.yml b/.github/actions/with-k-docker/action.yml
index 57e52424f6b..62c75bed3e1 100644
--- a/.github/actions/with-k-docker/action.yml
+++ b/.github/actions/with-k-docker/action.yml
@@ -10,6 +10,11 @@ inputs:
     required: true
     type: string
     default: kframework.deb
+  install-backend-debs:
+    description: 'Whether to install a separate package for the LLVM backend'
+    required: true
+    type: boolean
+    default: true
   distro:
     description: 'Distribution to setup Docker for.'
     required: true
@@ -28,26 +33,28 @@ runs:
     env:
       BASE_DISTRO: ${{ inputs.distro }}
       GH_TOKEN: ${{ github.token }}
+      INSTALL_BACKEND_DEBS: ${{ inputs.install-backend-debs }}
     run: |
       set -euxo pipefail
 
-      gh release download                           \
-        --repo runtimeverification/llvm-backend     \
-        --pattern "*ubuntu_${BASE_DISTRO}.deb"  \
-        --output llvm-backend.deb                   \
-        v$(cat deps/llvm-backend_release)
-
       CONTAINER_NAME=${{ inputs.container-name }}
       TAG=runtimeverificationinc/${CONTAINER_NAME}
       DOCKERFILE=${{ github.action_path }}/Dockerfile
       K_DEB_PATH=${{ inputs.k-deb-path }}
       LLVM_BACKEND_DEB_PATH=llvm-backend.deb
 
+      gh release download                           \
+        --repo runtimeverification/llvm-backend     \
+        --pattern "*ubuntu_${BASE_DISTRO}.deb"      \
+        --output "${LLVM_BACKEND_DEB_PATH}"         \
+        v$(cat deps/llvm-backend_release)
+
       docker build . \
         --file ${DOCKERFILE}                                        \
         --tag ${TAG}                                                \
         --build-arg BASE_DISTRO=${BASE_DISTRO}                      \
         --build-arg K_DEB_PATH=${K_DEB_PATH}                        \
+        --build-arg INSTALL_BACKEND_DEBS=${INSTALL_BACKEND_DEBS}    \
         --build-arg LLVM_BACKEND_DEB_PATH=${LLVM_BACKEND_DEB_PATH}
 
   - name: 'Run Docker container'
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 26d079f0994..840a5b07afd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -443,6 +443,7 @@ jobs:
         with:
           container-name: k-pyk-docs-${{ github.sha }}
           k-deb-path: kframework_amd64_ubuntu_jammy.deb
+          install-backend-debs: false
       - name: 'Build documentation in Docker container'
         run: docker exec -u user k-pyk-docs-${{ github.sha }} make docs
       - name: 'Copy documentation from Docker container'