diff --git a/Makefile b/Makefile
index 8480b708bb..f2653c8dd1 100644
--- a/Makefile
+++ b/Makefile
@@ -239,15 +239,10 @@ else
 	@hack/check-go-version.sh
 endif
 
-.PHONY: init-docker-buildx
-init-docker-buildx:
+.PHONY: ensure-buildx
+ensure-buildx:
 ifeq ($(DIND_TASKS),)
-ifneq ($(shell docker buildx 2>&1 >/dev/null; echo $?),)
-	$(error "buildx not available. Docker 19.03 or higher is required with experimental features enabled")
-endif
-	docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
-	docker buildx create --name ingress-nginx --use || true
-	docker buildx inspect --bootstrap
+	./hack/init-buildx.sh
 endif
 
 .PHONY: show-version
@@ -261,7 +256,7 @@ SPACE := $(EMPTY) $(EMPTY)
 COMMA := ,
 
 .PHONY: release # Build a multi-arch docker image
-release: init-docker-buildx clean
+release: ensure-buildx clean
 	echo "Building binaries..."
 	$(foreach PLATFORM,$(PLATFORMS), echo -n "$(PLATFORM)..."; ARCH=$(PLATFORM) make build;)
 
diff --git a/build/images/ingress-controller/build.sh b/build/images/ingress-controller/build.sh
index 879cb5486e..75c365db97 100644
--- a/build/images/ingress-controller/build.sh
+++ b/build/images/ingress-controller/build.sh
@@ -75,16 +75,7 @@ cd ingress-nginx
 
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
-make init-docker-buildx
-docker buildx use ingress-nginx --default --global
+make ensure-buildx
 
-# disable docker in docker tasks
-export DIND_TASKS=0
-
-echo "Building NGINX image..."
-ARCH=amd64 make build image push
-ARCH=arm   make build image push
-ARCH=arm64 make build image push
-
-echo "Creating multi-arch images..."
-make push-manifest
+echo "Building ingress controller image..."
+make release
diff --git a/build/images/nginx/build.sh b/build/images/nginx/build.sh
index 50a034bd46..640794b281 100644
--- a/build/images/nginx/build.sh
+++ b/build/images/nginx/build.sh
@@ -67,8 +67,7 @@ cd ingress-nginx/images/nginx
 
 export TAG=$(git rev-parse HEAD)
 
-make init-docker-buildx
-docker buildx use ingress-nginx --default --global
+make ensure-buildx
 
 echo "Building NGINX images..."
 make image
diff --git a/hack/init-buildx.sh b/hack/init-buildx.sh
new file mode 100755
index 0000000000..87fa0251a2
--- /dev/null
+++ b/hack/init-buildx.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -n "$DEBUG" ]; then
+	set -x
+fi
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+if ! docker buildx 2>&1 >/dev/null; then
+  echo "buildx not available. Docker 19.03 or higher is required with experimental features enabled"
+  exit 1
+fi
+
+# We can skip setup if the current builder already has multi-arch
+# AND if it isn't the docker driver, which doesn't work
+current_builder="$(docker buildx inspect)"
+# linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6
+if ! grep -q "^Driver: docker$"  <<<"${current_builder}" && \
+     grep -q "linux/amd64" <<<"${current_builder}" && \
+     grep -q "linux/arm"   <<<"${current_builder}" && \
+     grep -q "linux/arm64" <<<"${current_builder}" && \
+     grep -q "linux/s390x" <<<"${current_builder}"; then
+  exit 0
+fi
+
+# Ensure qemu is in binfmt_misc
+# Docker desktop already has these in versions recent enough to have buildx
+# We only need to do this setup on linux hosts
+if [ "$(uname)" == 'Linux' ]; then
+  # NOTE: this is pinned to a digest for a reason!
+  docker run --rm --privileged multiarch/qemu-user-static@sha256:c772ee1965aa0be9915ee1b018a0dd92ea361b4fa1bcab5bbc033517749b2af4 --reset -p yes
+fi
+
+# Ensure we use a builder that can leverage it (the default on linux will not)
+docker buildx rm ingress-nginx || true
+docker buildx create --use --name=ingress-nginx
diff --git a/images/nginx/Makefile b/images/nginx/Makefile
index 1df883e985..0e9f50a0c6 100644
--- a/images/nginx/Makefile
+++ b/images/nginx/Makefile
@@ -12,31 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.DEFAULT_GOAL:=image
+.DEFAULT_GOAL:=build
 
 # set default shell
-SHELL=/bin/bash -o pipefail
+SHELL=/bin/bash
 
 # 0.0.0 shouldn't clobber any released builds
-TAG ?= 0.103
-REGISTRY ?= quay.io/kubernetes-ingress-controller
+TAG ?= 0.104
+REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
 
 IMAGE = $(REGISTRY)/nginx
 
-.PHONY: image
-image:
+# required to enable buildx
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+# build with buildx
+PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x
+OUTPUT=
+PROGRESS=plain
+build: ensure-buildx
 	docker buildx build \
+		--platform=${PLATFORMS} $(OUTPUT) \
+		--progress=$(PROGRESS) \
 		--pull \
-		--push \
-		--progress plain \
-		--platform amd64,arm,arm64,s390x \
 		--tag $(IMAGE):$(TAG) rootfs
 
-.PHONY: init-docker-buildx
-init-docker-buildx:
-ifneq ($(shell docker buildx 2>&1 >/dev/null; echo $?),)
-	$(error "buildx not vailable. Docker 19.03 or higher is required")
-endif
-	docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
-	docker buildx create --name ingress-nginx --use || true
-	docker buildx inspect --bootstrap
+# push the cross built image
+push: OUTPUT=--push
+push: build
+
+# enable buildx
+ensure-buildx:
+	./../../hack/init-buildx.sh
+
+.PHONY: build push ensure-buildx