diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index 08eb10cd..d449e0da 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -46,8 +46,7 @@ jobs:
           # branch name is github.head_ref when triggered by pull_request
           # and it is github.ref_name when triggered by workflow_dispatch
           branch_name=${{ github.head_ref || github.ref_name }}
-          # limit CPU core and memory usage to keep runner daemon alive on CI machine
-          taskset -c 16-31 docker build . --memory 80g --shm-size 4g --build-arg TRITONBENCH_BRANCH="${branch_name}" --build-arg FORCE_DATE="${NIGHTLY_DATE}" \
+          docker build . --build-arg TRITONBENCH_BRANCH="${branch_name}" --build-arg FORCE_DATE="${NIGHTLY_DATE}" \
                   -f tritonbench-nightly.dockerfile -t ghcr.io/pytorch-labs/tritonbench:latest
           # Extract pytorch version from the docker
           PYTORCH_VERSION=$(docker run -e SETUP_SCRIPT="${SETUP_SCRIPT}" ghcr.io/pytorch-labs/tritonbench:latest bash -c '. "${SETUP_SCRIPT}"; python -c "import torch; print(torch.__version__)"')
diff --git a/docker/tritonbench-nightly.dockerfile b/docker/tritonbench-nightly.dockerfile
index 71d2c5d9..d69311f0 100644
--- a/docker/tritonbench-nightly.dockerfile
+++ b/docker/tritonbench-nightly.dockerfile
@@ -49,6 +49,9 @@ RUN cd /workspace/tritonbench && \
 # which is from NVIDIA driver
 RUN sudo apt update && sudo apt-get install -y libnvidia-compute-550 patchelf patch
 
+# Workaround: installing Ninja from setup.py hits "Failed to decode METADATA with UTF-8" error
+RUN pip install ninja
+
 # Install Tritonbench
 RUN cd /workspace/tritonbench && \
     bash .ci/tritonbench/install.sh
diff --git a/requirements.txt b/requirements.txt
index 4dc9f4fd..1cc03e35 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,5 +2,4 @@ packaging
 pynvml
 psutil
 tabulate
-ninja
 transformers==4.46.1