diff --git a/Dockerfile b/Dockerfile
index 1063a7e..bba098d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -26,8 +26,14 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa
 # to use.
 COPY docker-apt-install.sh /root/docker-apt-install.sh
 
+# Disable default apt cleaning, as we want to use cache mounts
+# Taken from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
 # install some base tools we want in all images
+# caching from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
      UPGRADE=yes /root/docker-apt-install.sh ca-certificates sysstat lsof net-tools tcpdump vim strace file
 
 # record build info so downstream images know about the base image they were
diff --git a/docker-apt-install.sh b/docker-apt-install.sh
index 36aaa07..653e21f 100755
--- a/docker-apt-install.sh
+++ b/docker-apt-install.sh
@@ -21,11 +21,8 @@ for arg in "$@"; do
     fi
 done
 
+# shellcheck disable=SC2086
 test -n "$PACKAGES" && apt-get install --yes --no-install-recommends $PACKAGES
 
 # clean up if we've upgraded
 test "${UPGRADE:-}" = "yes" && apt-get autoremove --yes
-
-# We do not apt-get clean becuase the default debian docker apt config does that for us.
-# Doing this saves us ~50MB, but means we need to apt-get update before we can install anything again
-rm -rf /var/lib/apt/lists/*