From 6235ab65e7f01f6d6b47e5ea05773385d37a1b25 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Wed, 21 Aug 2024 09:51:44 -0600
Subject: [PATCH 01/22] Update linuxkit to 1.5.0:

This version has volume support that is needed
for embedding images.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index 6a670848..985bea88 100755
--- a/build.sh
+++ b/build.sh
@@ -29,7 +29,7 @@ declare -g HOOK_LK_CONTAINERS_OCI_BASE="${HOOK_LK_CONTAINERS_OCI_BASE:-"quay.io/
 declare -g SKOPEO_IMAGE="${SKOPEO_IMAGE:-"quay.io/skopeo/stable:latest"}"
 
 # See https://github.com/linuxkit/linuxkit/releases
-declare -g -r LINUXKIT_VERSION_DEFAULT="1.2.0" # LinuxKit version to use by default; each flavor can set its own too
+declare -g -r LINUXKIT_VERSION_DEFAULT="1.5.0" # LinuxKit version to use by default; each flavor can set its own too
 
 # Directory to use for storing downloaded artifacts: LinuxKit binary, shellcheck binary, etc.
 declare -g -r CACHE_DIR="${CACHE_DIR:-"cache"}"

From ff15b8a158fd593c8f91c616306eed89a4e8f341 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Wed, 21 Aug 2024 09:52:23 -0600
Subject: [PATCH 02/22] Fix Docker build warnings:

Docker was warning about the case of `FROM`
not matching `AS`.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-bootkit/Dockerfile | 2 +-
 images/hook-docker/Dockerfile  | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/images/hook-bootkit/Dockerfile b/images/hook-bootkit/Dockerfile
index 6dc81241..347e7651 100644
--- a/images/hook-bootkit/Dockerfile
+++ b/images/hook-bootkit/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine as dev
+FROM golang:1.21-alpine AS dev
 COPY . /src/
 WORKDIR /src
 RUN go mod download
diff --git a/images/hook-docker/Dockerfile b/images/hook-docker/Dockerfile
index 3aad7bd1..737a9670 100644
--- a/images/hook-docker/Dockerfile
+++ b/images/hook-docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as dev
+FROM golang:1.20-alpine AS dev
 COPY . /src/
 WORKDIR /src
 RUN CGO_ENABLED=0 go build -a -ldflags '-s -w -extldflags "-static"' -o /hook-docker
@@ -13,4 +13,5 @@ RUN strip /usr/local/bin/docker /usr/local/bin/dockerd /usr/local/bin/docker-pro
 # Purge binutils package after stripping
 RUN apk del binutils
 COPY --from=dev /hook-docker .
+
 ENTRYPOINT ["/hook-docker"]

From 9fc15e8f957ec630712e897809c5fb4b414d055e Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Wed, 21 Aug 2024 09:54:26 -0600
Subject: [PATCH 03/22] Add optional embedding of container images into DinD:

This helps use cases where images already existing
in the DinD cache is needed. Air gap envs, latency
constrained/concerned envs, etc.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 .gitignore                              |  2 +
 README.md                               | 10 ++++-
 bash/hook-lk-containers.sh              |  1 +
 bash/linuxkit.sh                        |  3 +-
 images/hook-embedded/Dockerfile         |  5 +++
 images/hook-embedded/images.txt.example |  9 ++++
 images/hook-embedded/images/.keep       |  0
 images/hook-embedded/pull-images.sh     | 59 +++++++++++++++++++++++++
 linuxkit-templates/hook.template.yaml   | 43 ++++++++++++++----
 9 files changed, 121 insertions(+), 11 deletions(-)
 create mode 100644 images/hook-embedded/Dockerfile
 create mode 100644 images/hook-embedded/images.txt.example
 create mode 100644 images/hook-embedded/images/.keep
 create mode 100755 images/hook-embedded/pull-images.sh

diff --git a/.gitignore b/.gitignore
index 4bba5900..a70e90ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,5 @@ cache/
 *.swp
 .idea
 kernel/Dockerfile.autogen.*
+images/hook-embedded/images/*
+!images/hook-embedded/images/.keep
diff --git a/README.md b/README.md
index c2f88c78..621716d4 100644
--- a/README.md
+++ b/README.md
@@ -160,9 +160,17 @@ The `gha-matrix` CLI command prepares a set of JSON outputs for GitHub Actions m
 - `DOCKER_ARCH` is used by the `linuxkit-containers` command to build the containers for the specified architecture.
 - `DO_PUSH`: `yes` or `no`, will push the built containers to the OCI registry; defaults to `no`.
 
+### Embedding container images into the DinD (docker-in-docker), also known as [hook-docker](images/hook-docker/), container
+
+For use cases where having container images already available in Docker is needed, the following steps can be taken to embed container images into hook-docker (DinD):
+
+1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
+1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
+1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run the [pull-images.sh](images/hook-embedded/pull-images.sh) script. Read the comments at the top of the script for more details.
+1. Change directories to the root of the HookOS repository and run `sudo ./build.sh build ...` to build the HookOS kernel and ramdisk. FYI, `sudo` is needed as DIND changes file ownerships to root.
+
 ### Build system TO-DO list
 
-- [ ] Update to Linuxkit 1.2.0 and new linuxkit pkgs; this might lead into the containerd vs dind;
 - [ ] `make debug` functionality (sshd enabled) was lost in the Makefile -> bash transition;
 
 [formats]: https://github.com/linuxkit/linuxkit/blob/master/README.md#booting-and-testing
diff --git a/bash/hook-lk-containers.sh b/bash/hook-lk-containers.sh
index 1c9a8a87..9c5c8b15 100644
--- a/bash/hook-lk-containers.sh
+++ b/bash/hook-lk-containers.sh
@@ -13,6 +13,7 @@ function build_all_hook_linuxkit_containers() {
 	build_hook_linuxkit_container hook-mdev HOOK_CONTAINER_MDEV_IMAGE
 	build_hook_linuxkit_container hook-containerd HOOK_CONTAINER_CONTAINERD_IMAGE
 	build_hook_linuxkit_container hook-runc HOOK_CONTAINER_RUNC_IMAGE
+	build_hook_linuxkit_container hook-embedded HOOK_CONTAINER_EMBEDDED_IMAGE
 }
 
 function build_hook_linuxkit_container() {
diff --git a/bash/linuxkit.sh b/bash/linuxkit.sh
index bf4dea0d..1e9461fc 100644
--- a/bash/linuxkit.sh
+++ b/bash/linuxkit.sh
@@ -70,7 +70,8 @@ function linuxkit_build() {
 			HOOK_CONTAINER_MDEV_IMAGE="${HOOK_CONTAINER_MDEV_IMAGE}" \
 			HOOK_CONTAINER_CONTAINERD_IMAGE="${HOOK_CONTAINER_CONTAINERD_IMAGE}" \
 			HOOK_CONTAINER_RUNC_IMAGE="${HOOK_CONTAINER_RUNC_IMAGE}" \
-			envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE' \
+			HOOK_CONTAINER_EMBEDDED_IMAGE="${HOOK_CONTAINER_EMBEDDED_IMAGE}" \			
+			envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE $HOOK_CONTAINER_EMBEDDED_IMAGE' \
 			> "hook.${inventory_id}.yaml"
 
 	declare -g linuxkit_bin=""
diff --git a/images/hook-embedded/Dockerfile b/images/hook-embedded/Dockerfile
new file mode 100644
index 00000000..8495d95c
--- /dev/null
+++ b/images/hook-embedded/Dockerfile
@@ -0,0 +1,5 @@
+FROM scratch
+ENTRYPOINT []
+WORKDIR /
+COPY ./images/ /
+CMD []
\ No newline at end of file
diff --git a/images/hook-embedded/images.txt.example b/images/hook-embedded/images.txt.example
new file mode 100644
index 00000000..c532c8a7
--- /dev/null
+++ b/images/hook-embedded/images.txt.example
@@ -0,0 +1,9 @@
+# This is an example file. It explains the required format.
+# For the actual file, you must remove all the comments.
+# The format is source image, a single space, optional additional tag of the source image.
+#<source image> <optional additional tag of the source image>
+# for example:
+quay.io/tinkerbell/tink-worker:v0.10.0
+quay.io/tinkerbell/tink-worker:v0.10.0 tink-worker:v0.10.0
+quay.io/tinkerbell/actions/image2disk embedded/actions/image2disk
+quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec
\ No newline at end of file
diff --git a/images/hook-embedded/images/.keep b/images/hook-embedded/images/.keep
new file mode 100644
index 00000000..e69de29b
diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
new file mode 100755
index 00000000..a225176c
--- /dev/null
+++ b/images/hook-embedded/pull-images.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# This script is used to pull 
+# This script is used to build an image that is embedded in HookOS.
+# The image contains the /var/lib/docker directory which has pulled images
+# from the images.txt file. When HookOS boots up, the DinD container will
+# have all the images in its cache.
+
+# The purpose of doing this is so that EKS Anywhere doesn't have to set registry
+# and registry credentials in each Hardware object.
+
+# In my testing the initramfs for HookOS with these embedded images was about 334MB.
+# The machine booting HookOS needed 6GB of RAM to boot up successfully.
+
+set -euo pipefail
+
+function main() {
+    local dind_container="$1"
+    local images_file="$2"
+    # as this function maybe called multiple times, we need to ensure the container is removed
+    trap "docker rm -f ${dind_container} &> /dev/null" RETURN
+    # we're using set -e so the trap on RETURN will not be executed when a command fails
+    trap "docker rm -f ${dind_container} &> /dev/null" EXIT
+    # start DinD container
+    # In order to avoid the src bind mount directory (./images/) ownership from changing to root
+    # we don't bind mount to /var/lib/docker in the container because the DinD container is running as root and
+    # will change the permissions of the bind mount directory (images/) to root.
+    echo -e "Starting DinD container"
+    echo -e "-----------------------"
+    docker run -d --rm --privileged --name "${dind_container}" -v ${PWD}/images/:/var/lib/docker-embedded/ -d docker:dind
+
+    # wait until the docker daemon is ready
+    until docker exec "${dind_container}" docker info &> /dev/null; do
+        sleep 1
+    done
+
+    # pull images from list
+    # this expects a file named images.txt in the same directory as this script
+    # the format of this file is line separated: <image> <optional tag>
+    #
+    # the || [ -n "$first_image" ] is to handle the last line of the file that doesn't have a newline.
+    while IFS=" " read -r first_image image_tag || [ -n "$first_image" ] ; do
+        echo -e "----------------------- $first_image -----------------------"
+        docker exec "${dind_container}" docker pull $first_image
+        if [[ $image_tag != "" ]]; then
+            docker exec "${dind_container}" docker tag $first_image $image_tag
+        fi
+    done < "${images_file}"
+
+    # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues.
+    docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*"
+    # We need to copy /var/lib/docker to /var/lib/docker-embedded in order for HookOS to use the Docker images in its build.
+    docker exec "${dind_container}" sh -c "cp -a /var/lib/docker/* /var/lib/docker-embedded/"
+}
+
+arch="amd64"
+dind_container_name="hookos-dind-${arch}"
+images_file="images.txt"
+main "${dind_container_name}" "${images_file}" "${arch}"
\ No newline at end of file
diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml
index 26db724d..8eb6f6db 100644
--- a/linuxkit-templates/hook.template.yaml
+++ b/linuxkit-templates/hook.template.yaml
@@ -9,21 +9,36 @@
 # - HOOK_CONTAINER_MDEV_IMAGE: ${HOOK_CONTAINER_MDEV_IMAGE}
 # - HOOK_CONTAINER_CONTAINERD_IMAGE: ${HOOK_CONTAINER_CONTAINERD_IMAGE}
 # - HOOK_CONTAINER_RUNC_IMAGE: ${HOOK_CONTAINER_RUNC_IMAGE}
+# - HOOK_CONTAINER_EMBEDDED_IMAGE: ${HOOK_CONTAINER_EMBEDDED_IMAGE}
 # - Other variables are not replaced: for example this is a literal dollarsign-SOMETHING: $SOMETHING and with braces: ${SOMETHING}
 
 kernel:
   image: "${HOOK_KERNEL_IMAGE}"
-  cmdline: "this_is_not_used=at_at_all_in_hook command_line_is_determined_by=ipxe"
+  cmdline: "this_is_not_used=at_all_in_hook command_line_is_determined_by=ipxe"
 
 init:
-  # this sha is the first with cgroups v2 as the default
-  - linuxkit/init:8a7b6cdb89197dc94eb6db69ef9dc90b750db598
+  # this init container sha has support for volumes
+  - linuxkit/init:872d2e1be745f1acb948762562cf31c367303a3b
   - "${HOOK_CONTAINER_RUNC_IMAGE}"
   - "${HOOK_CONTAINER_CONTAINERD_IMAGE}"
   - linuxkit/ca-certificates:v1.0.0
   - linuxkit/firmware:24402a25359c7bc290f7fc3cd23b6b5f0feb32a5 # "Some" firmware from Linuxkit pkg; see https://github.com/linuxkit/linuxkit/blob/master/pkg/firmware/Dockerfile
 
+volumes:
+  - name: embedded-images
+    image: "${HOOK_CONTAINER_EMBEDDED_IMAGE}"
+
 onboot:
+  - name: embedded-images
+    image: alpine
+    binds:
+      - /var/run/images:/var/run/images
+      - embedded-images:/images
+    command: [ "sh", "-xc", "mv /images/* /var/run/images/" ]
+    runtime:
+      mkdir:
+        - /var/run/images
+
   - name: rngd1
     image: linuxkit/rngd:v1.0.0
     command: [ "/sbin/rngd", "-1" ]
@@ -100,6 +115,8 @@ services:
     devices:
     - path: all
       type: b
+    - path: all
+      type: c
     - path: "/dev/console"
       type: c
       major: 5
@@ -187,6 +204,8 @@ services:
     devices:
     - path: all
       type: b
+    - path: all
+      type: c
 
   - name: hook-bootkit
     image: "${HOOK_CONTAINER_BOOTKIT_IMAGE}"
@@ -219,8 +238,14 @@ services:
       mkdir:
         - /var/lib/dhcpcd
 
-#dbg  - name: sshd
-#dbg    image: linuxkit/sshd:v1.0.0
+#SSH_SERVER  - name: sshd
+#SSH_SERVER    image: linuxkit/sshd:v1.0.0
+#SSH_SERVER    binds.add:
+#SSH_SERVER      - /etc/profile.d/local.sh:/etc/profile.d/local.sh
+#SSH_SERVER      - /root/.ssh/authorized_keys:/root/.ssh/authorized_keys
+#SSH_SERVER      - /usr/bin/nerdctl:/usr/bin/nerdctl
+#SSH_SERVER      - /:/host_root
+
 
 files:
   - path: etc/profile.d/local.sh
@@ -299,10 +324,10 @@ files:
       ttyUSB1
       ttyUSB2
 
-#dbg  - path: root/.ssh/authorized_keys
-#dbg    source: ~/.ssh/id_rsa.pub
-#dbg    mode: "0600"
-#dbg    optional: true
+#SSH_SERVER  - path: root/.ssh/authorized_keys
+#SSH_SERVER    source: ~/.ssh/id_rsa.pub
+#SSH_SERVER    mode: "0600"
+#SSH_SERVER    optional: true
 
 trust:
   org:

From 9318be66bf1b019ae28098b24e688464f1a5871e Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Wed, 21 Aug 2024 09:57:57 -0600
Subject: [PATCH 04/22] Add note on embedded images being optional:

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 621716d4..dcb055d2 100644
--- a/README.md
+++ b/README.md
@@ -164,6 +164,8 @@ The `gha-matrix` CLI command prepares a set of JSON outputs for GitHub Actions m
 
 For use cases where having container images already available in Docker is needed, the following steps can be taken to embed container images into hook-docker (DinD):
 
+> Note: This is optional and no container images will be embedded by default.
+
 1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
 1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
 1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run the [pull-images.sh](images/hook-embedded/pull-images.sh) script. Read the comments at the top of the script for more details.

From f68dc4b8b55c110db0e5207f312da7a994bd5cf9 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Wed, 21 Aug 2024 17:27:39 -0600
Subject: [PATCH 05/22] Add platform option to pull-images.sh:

This makes it so that the correct architecture for
the embedded images are pulled.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 README.md                           |  2 +-
 images/hook-embedded/pull-images.sh | 22 ++++++++++++----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index dcb055d2..5f385076 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ For use cases where having container images already available in Docker is neede
 
 1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
 1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
-1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run the [pull-images.sh](images/hook-embedded/pull-images.sh) script. Read the comments at the top of the script for more details.
+1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run [`pull-images.sh`](images/hook-embedded/pull-images.sh) script when building amd64 images and run [`pull-images.sh arm64`](images/hook-embedded/pull-images.sh) when building arm64 images. Read the comments at the top of the script for more details.
 1. Change directories to the root of the HookOS repository and run `sudo ./build.sh build ...` to build the HookOS kernel and ramdisk. FYI, `sudo` is needed as DIND changes file ownerships to root.
 
 ### Build system TO-DO list
diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index a225176c..88df51df 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -17,10 +17,11 @@ set -euo pipefail
 function main() {
     local dind_container="$1"
     local images_file="$2"
+    local arch="$3"
     # as this function maybe called multiple times, we need to ensure the container is removed
-    trap "docker rm -f ${dind_container} &> /dev/null" RETURN
+    trap "docker rm -f "${dind_container}" &> /dev/null" RETURN
     # we're using set -e so the trap on RETURN will not be executed when a command fails
-    trap "docker rm -f ${dind_container} &> /dev/null" EXIT
+    trap "docker rm -f "${dind_container}" &> /dev/null" EXIT
     # start DinD container
     # In order to avoid the src bind mount directory (./images/) ownership from changing to root
     # we don't bind mount to /var/lib/docker in the container because the DinD container is running as root and
@@ -34,26 +35,27 @@ function main() {
         sleep 1
     done
 
+    # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues.
+    docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*"
+
     # pull images from list
     # this expects a file named images.txt in the same directory as this script
     # the format of this file is line separated: <image> <optional tag>
     #
     # the || [ -n "$first_image" ] is to handle the last line of the file that doesn't have a newline.
-    while IFS=" " read -r first_image image_tag || [ -n "$first_image" ] ; do
+    while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
         echo -e "----------------------- $first_image -----------------------"
-        docker exec "${dind_container}" docker pull $first_image
-        if [[ $image_tag != "" ]]; then
-            docker exec "${dind_container}" docker tag $first_image $image_tag
+        docker exec "${dind_container}" docker pull --platform=linux/"${arch}" "${first_image}"
+        if [[ "${image_tag}" != "" ]]; then
+            docker exec "${dind_container}" docker tag "${first_image}" "${image_tag}"
         fi
     done < "${images_file}"
 
-    # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues.
-    docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*"
     # We need to copy /var/lib/docker to /var/lib/docker-embedded in order for HookOS to use the Docker images in its build.
     docker exec "${dind_container}" sh -c "cp -a /var/lib/docker/* /var/lib/docker-embedded/"
 }
 
-arch="amd64"
+arch="${1-amd64}"
 dind_container_name="hookos-dind-${arch}"
 images_file="images.txt"
-main "${dind_container_name}" "${images_file}" "${arch}"
\ No newline at end of file
+main "${dind_container_name}" "${images_file}" "${arch}"

From d481896f0f6f12ac137b598109a2d3bb509e4289 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Wed, 21 Aug 2024 17:30:22 -0600
Subject: [PATCH 06/22] Don't git track the images.txt file:

There is an example file already. The images.txt
will be something the user creates for themselves.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index a70e90ce..14ffc9ea 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ cache/
 kernel/Dockerfile.autogen.*
 images/hook-embedded/images/*
 !images/hook-embedded/images/.keep
+images/hook-embedded/images.txt

From 4165fe2544f461939b4836fbda057783f527aa9b Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Thu, 22 Aug 2024 15:36:43 -0600
Subject: [PATCH 07/22] Update the embed process:

This removes the need to mv the
directory of embedded images. This is
accomplished by bind mount (rw) the
read only images location onto the /var/lib/docker
directory in the Hook-docker container.

This means that start up doesn't need to wait for
the mv command to complete. So startup doesn't incur
any delay like it was with the mv. This also means that
we can embed a lot more images with having start up issue.
In testing, I found that if enough images, compared to the
amount of memory available, were embedded then HookOS would
not boot up. It would max out on memory. It's possible with
enough time that it would have booted but i didnt wait longer
than about 30min.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 README.md                             |  2 ++
 images/hook-docker/Dockerfile         |  3 ++-
 images/hook-docker/entrypoint.sh      | 13 +++++++++++++
 images/hook-embedded/Dockerfile       |  2 +-
 linuxkit-templates/hook.template.yaml | 16 ++--------------
 5 files changed, 20 insertions(+), 16 deletions(-)
 create mode 100755 images/hook-docker/entrypoint.sh

diff --git a/README.md b/README.md
index 5f385076..b5a14185 100644
--- a/README.md
+++ b/README.md
@@ -166,6 +166,8 @@ For use cases where having container images already available in Docker is neede
 
 > Note: This is optional and no container images will be embedded by default.
 
+> Note: This will increase the overall size of HookOS. As HookOS is an in memory OS, make sure that the size increase works for the machines you are provisioning.
+
 1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
 1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
 1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run [`pull-images.sh`](images/hook-embedded/pull-images.sh) script when building amd64 images and run [`pull-images.sh arm64`](images/hook-embedded/pull-images.sh) when building arm64 images. Read the comments at the top of the script for more details.
diff --git a/images/hook-docker/Dockerfile b/images/hook-docker/Dockerfile
index 737a9670..0a98d2bd 100644
--- a/images/hook-docker/Dockerfile
+++ b/images/hook-docker/Dockerfile
@@ -13,5 +13,6 @@ RUN strip /usr/local/bin/docker /usr/local/bin/dockerd /usr/local/bin/docker-pro
 # Purge binutils package after stripping
 RUN apk del binutils
 COPY --from=dev /hook-docker .
+COPY entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/hook-docker"]
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/images/hook-docker/entrypoint.sh b/images/hook-docker/entrypoint.sh
new file mode 100755
index 00000000..7bca7973
--- /dev/null
+++ b/images/hook-docker/entrypoint.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -xeuo pipefail
+
+# This allows us to embed container images into HookOS.
+# We assume that any images are stored in /etc/embedded-images.
+# The /etc directory in Linuxkit is a read-only filesystem.
+# DinD requires that its data directory is writable.
+# So we bind mount /etc/embedded-images to /var/lib/docker to make it writable.
+mount --bind /etc/embedded-images/ /var/lib/docker
+mount -o remount,rw /var/lib/docker
+
+/hook-docker
\ No newline at end of file
diff --git a/images/hook-embedded/Dockerfile b/images/hook-embedded/Dockerfile
index 8495d95c..aa4ac8c8 100644
--- a/images/hook-embedded/Dockerfile
+++ b/images/hook-embedded/Dockerfile
@@ -1,5 +1,5 @@
 FROM scratch
 ENTRYPOINT []
 WORKDIR /
-COPY ./images/ /
+COPY ./images/ /etc/embedded-images/
 CMD []
\ No newline at end of file
diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml
index 8eb6f6db..6b22856f 100644
--- a/linuxkit-templates/hook.template.yaml
+++ b/linuxkit-templates/hook.template.yaml
@@ -23,22 +23,9 @@ init:
   - "${HOOK_CONTAINER_CONTAINERD_IMAGE}"
   - linuxkit/ca-certificates:v1.0.0
   - linuxkit/firmware:24402a25359c7bc290f7fc3cd23b6b5f0feb32a5 # "Some" firmware from Linuxkit pkg; see https://github.com/linuxkit/linuxkit/blob/master/pkg/firmware/Dockerfile
-
-volumes:
-  - name: embedded-images
-    image: "${HOOK_CONTAINER_EMBEDDED_IMAGE}"
+  - "${HOOK_CONTAINER_EMBEDDED_IMAGE}"
 
 onboot:
-  - name: embedded-images
-    image: alpine
-    binds:
-      - /var/run/images:/var/run/images
-      - embedded-images:/images
-    command: [ "sh", "-xc", "mv /images/* /var/run/images/" ]
-    runtime:
-      mkdir:
-        - /var/run/images
-
   - name: rngd1
     image: linuxkit/rngd:v1.0.0
     command: [ "/sbin/rngd", "-1" ]
@@ -196,6 +183,7 @@ services:
       - /var/run/docker:/var/run
       - /var/run/images:/var/lib/docker
       - /var/run/worker:/worker
+      - /etc/embedded-images/:/etc/embedded-images/
     runtime:
       mkdir:
         - /var/run/images

From 72fde591bbb8b137efca55e352b99d303d442420 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Thu, 22 Aug 2024 20:16:57 -0600
Subject: [PATCH 08/22] Change the way the images are presented to hook-docker:

Instead of having hook-docker know about mounting the
embedded images, the images get mounted with the right
permissions and made available to hook-docker at the
"usual" location. This decouples this embedding process
from hook-docker. This should allow the two to only be
coupled by the mount point of /var/run/images.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-docker/Dockerfile         |  3 +--
 images/hook-docker/entrypoint.sh      | 13 -------------
 images/hook-embedded/.dockerignore    |  3 +++
 images/hook-embedded/Dockerfile       |  4 ++++
 images/hook-embedded/images-mount.sh  | 17 +++++++++++++++++
 linuxkit-templates/hook.template.yaml |  1 -
 6 files changed, 25 insertions(+), 16 deletions(-)
 delete mode 100755 images/hook-docker/entrypoint.sh
 create mode 100644 images/hook-embedded/.dockerignore
 create mode 100755 images/hook-embedded/images-mount.sh

diff --git a/images/hook-docker/Dockerfile b/images/hook-docker/Dockerfile
index 0a98d2bd..737a9670 100644
--- a/images/hook-docker/Dockerfile
+++ b/images/hook-docker/Dockerfile
@@ -13,6 +13,5 @@ RUN strip /usr/local/bin/docker /usr/local/bin/dockerd /usr/local/bin/docker-pro
 # Purge binutils package after stripping
 RUN apk del binutils
 COPY --from=dev /hook-docker .
-COPY entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/hook-docker"]
diff --git a/images/hook-docker/entrypoint.sh b/images/hook-docker/entrypoint.sh
deleted file mode 100755
index 7bca7973..00000000
--- a/images/hook-docker/entrypoint.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-
-set -xeuo pipefail
-
-# This allows us to embed container images into HookOS.
-# We assume that any images are stored in /etc/embedded-images.
-# The /etc directory in Linuxkit is a read-only filesystem.
-# DinD requires that its data directory is writable.
-# So we bind mount /etc/embedded-images to /var/lib/docker to make it writable.
-mount --bind /etc/embedded-images/ /var/lib/docker
-mount -o remount,rw /var/lib/docker
-
-/hook-docker
\ No newline at end of file
diff --git a/images/hook-embedded/.dockerignore b/images/hook-embedded/.dockerignore
new file mode 100644
index 00000000..75ffb5a2
--- /dev/null
+++ b/images/hook-embedded/.dockerignore
@@ -0,0 +1,3 @@
+*
+!images/
+!docker-mount.sh
\ No newline at end of file
diff --git a/images/hook-embedded/Dockerfile b/images/hook-embedded/Dockerfile
index aa4ac8c8..52e4572d 100644
--- a/images/hook-embedded/Dockerfile
+++ b/images/hook-embedded/Dockerfile
@@ -2,4 +2,8 @@ FROM scratch
 ENTRYPOINT []
 WORKDIR /
 COPY ./images/ /etc/embedded-images/
+# the name 001 is important as that is the order in which the scripts are executed
+# we need this mounting to happen before the other init.d scripts run so that
+# the mount points are available to them.
+COPY ./images-mount.sh /etc/init.d/001-images-mount.sh
 CMD []
\ No newline at end of file
diff --git a/images/hook-embedded/images-mount.sh b/images/hook-embedded/images-mount.sh
new file mode 100755
index 00000000..cea4a96f
--- /dev/null
+++ b/images/hook-embedded/images-mount.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+exec 3>&1 4>&2
+trap 'exec 2>&4 1>&3' 0 1 2 3
+exec 1>/var/log/embedded-images.log 2>&1
+
+set -xeuo pipefail
+
+# We can't have a Linuxkit "init" container that dumps its file contents to /var and be writable
+# because the init process overwrites it and the contents are lost.
+# Instead, we have the init container, with all the Docker images, dump its contents to /etc/embedded-images.
+# Then we bind mount /etc/embedded-images to /run/images (/var/run is symlinked to /run) and make sure it's
+# read/write. This allows the DinD container to bind mount /var/run/images to /var/lib/docker and the Docker
+# images are available right away and /var/lib/docker is writable.
+mkdir -p /run/images
+mount -o bind,rw /etc/embedded-images/ /run/images
+mount -o remount,rw /run/images
\ No newline at end of file
diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml
index 6b22856f..5864bdc2 100644
--- a/linuxkit-templates/hook.template.yaml
+++ b/linuxkit-templates/hook.template.yaml
@@ -183,7 +183,6 @@ services:
       - /var/run/docker:/var/run
       - /var/run/images:/var/lib/docker
       - /var/run/worker:/worker
-      - /etc/embedded-images/:/etc/embedded-images/
     runtime:
       mkdir:
         - /var/run/images

From 385941eda816fe67ed0c74452a37f6d96a05dde2 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Thu, 22 Aug 2024 20:21:05 -0600
Subject: [PATCH 09/22] Add newlines to all files without them:

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/.dockerignore      | 2 +-
 images/hook-embedded/Dockerfile         | 2 +-
 images/hook-embedded/images-mount.sh    | 2 +-
 images/hook-embedded/images.txt.example | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/images/hook-embedded/.dockerignore b/images/hook-embedded/.dockerignore
index 75ffb5a2..112729d3 100644
--- a/images/hook-embedded/.dockerignore
+++ b/images/hook-embedded/.dockerignore
@@ -1,3 +1,3 @@
 *
 !images/
-!docker-mount.sh
\ No newline at end of file
+!docker-mount.sh
diff --git a/images/hook-embedded/Dockerfile b/images/hook-embedded/Dockerfile
index 52e4572d..b716d552 100644
--- a/images/hook-embedded/Dockerfile
+++ b/images/hook-embedded/Dockerfile
@@ -6,4 +6,4 @@ COPY ./images/ /etc/embedded-images/
 # we need this mounting to happen before the other init.d scripts run so that
 # the mount points are available to them.
 COPY ./images-mount.sh /etc/init.d/001-images-mount.sh
-CMD []
\ No newline at end of file
+CMD []
diff --git a/images/hook-embedded/images-mount.sh b/images/hook-embedded/images-mount.sh
index cea4a96f..ff939c5c 100755
--- a/images/hook-embedded/images-mount.sh
+++ b/images/hook-embedded/images-mount.sh
@@ -14,4 +14,4 @@ set -xeuo pipefail
 # images are available right away and /var/lib/docker is writable.
 mkdir -p /run/images
 mount -o bind,rw /etc/embedded-images/ /run/images
-mount -o remount,rw /run/images
\ No newline at end of file
+mount -o remount,rw /run/images
diff --git a/images/hook-embedded/images.txt.example b/images/hook-embedded/images.txt.example
index c532c8a7..c0c6bb88 100644
--- a/images/hook-embedded/images.txt.example
+++ b/images/hook-embedded/images.txt.example
@@ -6,4 +6,4 @@
 quay.io/tinkerbell/tink-worker:v0.10.0
 quay.io/tinkerbell/tink-worker:v0.10.0 tink-worker:v0.10.0
 quay.io/tinkerbell/actions/image2disk embedded/actions/image2disk
-quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec
\ No newline at end of file
+quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec

From d0d5e0fb1f9030dafe81352ebab4527de2f4fe2e Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Thu, 22 Aug 2024 20:27:01 -0600
Subject: [PATCH 10/22] Remove .dockerignore:

It adds an unnecessary place to need to be aware of
when adding or renaming files.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/.dockerignore | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 images/hook-embedded/.dockerignore

diff --git a/images/hook-embedded/.dockerignore b/images/hook-embedded/.dockerignore
deleted file mode 100644
index 112729d3..00000000
--- a/images/hook-embedded/.dockerignore
+++ /dev/null
@@ -1,3 +0,0 @@
-*
-!images/
-!docker-mount.sh

From 04dc1bc80762cccf308443c48f190056ea526c61 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Fri, 23 Aug 2024 09:24:37 -0600
Subject: [PATCH 11/22] Make the docker:dind image configurable:

Allows users to specify specific versions of
docker:dind to use.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/pull-images.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index 88df51df..15737066 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -18,6 +18,7 @@ function main() {
     local dind_container="$1"
     local images_file="$2"
     local arch="$3"
+    local dind_container_image="$4"
     # as this function maybe called multiple times, we need to ensure the container is removed
     trap "docker rm -f "${dind_container}" &> /dev/null" RETURN
     # we're using set -e so the trap on RETURN will not be executed when a command fails
@@ -28,7 +29,7 @@ function main() {
     # will change the permissions of the bind mount directory (images/) to root.
     echo -e "Starting DinD container"
     echo -e "-----------------------"
-    docker run -d --rm --privileged --name "${dind_container}" -v ${PWD}/images/:/var/lib/docker-embedded/ -d docker:dind
+    docker run -d --rm --privileged --name "${dind_container}" -v ${PWD}/images/:/var/lib/docker-embedded/ -d "${dind_container_image}"
 
     # wait until the docker daemon is ready
     until docker exec "${dind_container}" docker info &> /dev/null; do
@@ -58,4 +59,5 @@ function main() {
 arch="${1-amd64}"
 dind_container_name="hookos-dind-${arch}"
 images_file="images.txt"
-main "${dind_container_name}" "${images_file}" "${arch}"
+dind_container_image="${2-docker:dind}"
+main "${dind_container_name}" "${images_file}" "${arch}" "${dind_container_image}"

From 4bd7ee60fa8bbce1daabc0d48e8d48b736943ecf Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 09:35:49 -0600
Subject: [PATCH 12/22] Refactor script to pull images:

This pulls images from the local docker
client instead of from the DinD container.
This will allow for registries that need logged
into and any proxying that might be needed to occur
during an image pull.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 .gitignore                            |  4 ++
 images/hook-embedded/docker/.keep     |  0
 images/hook-embedded/images_tar/.keep |  0
 images/hook-embedded/pull-images.sh   | 61 +++++++++++++++++++++++----
 4 files changed, 56 insertions(+), 9 deletions(-)
 create mode 100644 images/hook-embedded/docker/.keep
 create mode 100644 images/hook-embedded/images_tar/.keep

diff --git a/.gitignore b/.gitignore
index 14ffc9ea..42c2ec5c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,7 @@ kernel/Dockerfile.autogen.*
 images/hook-embedded/images/*
 !images/hook-embedded/images/.keep
 images/hook-embedded/images.txt
+images/hook-embedded/docker/*
+!images/hook-embedded/docker/.keep
+images/hook-embedded/images_tar/*
+!images/hook-embedded/images_tar/.keep
diff --git a/images/hook-embedded/docker/.keep b/images/hook-embedded/docker/.keep
new file mode 100644
index 00000000..e69de29b
diff --git a/images/hook-embedded/images_tar/.keep b/images/hook-embedded/images_tar/.keep
new file mode 100644
index 00000000..e69de29b
diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index 15737066..72fd2f7c 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -14,46 +14,89 @@
 
 set -euo pipefail
 
+function docker_save_image() {
+    local image="$1"
+    local output_dir="$2"
+    local output_file="${output_dir}/$(echo "${image}" | tr '/' '-')"
+
+    docker save -o "${output_file}" "${image}"
+}
+
+function docker_load_image() {
+    local image_file="$1"
+    local socket_location="$2"
+
+    sudo -E DOCKER_HOST=unix://"${socket_location}" docker load -i "${image_file}"
+}
+
+function docker_pull_image() {
+    local image="$1"
+    local arch="${2-amd64}"
+
+    docker pull --platform=linux/"${arch}" "${image}"
+}
+
 function main() {
     local dind_container="$1"
     local images_file="$2"
     local arch="$3"
     local dind_container_image="$4"
+
+    # Pull the images
+    while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
+        echo -e "----------------------- $first_image -----------------------"
+        docker_pull_image "${first_image}"
+    done < "${images_file}"
+
+    # Save the images
+    local output_dir="${PWD}/images_tar"
+    mkdir -p "${output_dir}"
+    while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
+        docker_save_image "${first_image}" "${output_dir}"
+    done < "${images_file}"
+
     # as this function maybe called multiple times, we need to ensure the container is removed
     trap "docker rm -f "${dind_container}" &> /dev/null" RETURN
     # we're using set -e so the trap on RETURN will not be executed when a command fails
     trap "docker rm -f "${dind_container}" &> /dev/null" EXIT
+
     # start DinD container
     # In order to avoid the src bind mount directory (./images/) ownership from changing to root
     # we don't bind mount to /var/lib/docker in the container because the DinD container is running as root and
     # will change the permissions of the bind mount directory (images/) to root.
     echo -e "Starting DinD container"
     echo -e "-----------------------"
-    docker run -d --rm --privileged --name "${dind_container}" -v ${PWD}/images/:/var/lib/docker-embedded/ -d "${dind_container_image}"
+    docker run -d --privileged --name "${dind_container}" -v ${PWD}/docker:/run -v ${PWD}/images/:/var/lib/docker-embedded/ -d "${dind_container_image}"
 
     # wait until the docker daemon is ready
     until docker exec "${dind_container}" docker info &> /dev/null; do
         sleep 1
+        if [[ $(docker inspect -f '{{.State.Status}}' "${dind_container}") == "exited" ]]; then
+            echo "DinD container exited unexpectedly"
+            exit 1
+        fi
     done
 
     # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues.
     docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*"
 
-    # pull images from list
-    # this expects a file named images.txt in the same directory as this script
-    # the format of this file is line separated: <image> <optional tag>
-    #
-    # the || [ -n "$first_image" ] is to handle the last line of the file that doesn't have a newline.
+    # Load the images
+    for image_file in "${output_dir}"/*; do
+        docker_load_image "${image_file}" "${PWD}/docker/docker.sock"
+    done
+
+    # clean up tar files
+    rm -rf "${output_dir}"/*
+
+    # Create any tags for the images
     while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
-        echo -e "----------------------- $first_image -----------------------"
-        docker exec "${dind_container}" docker pull --platform=linux/"${arch}" "${first_image}"
         if [[ "${image_tag}" != "" ]]; then
             docker exec "${dind_container}" docker tag "${first_image}" "${image_tag}"
         fi
     done < "${images_file}"
 
     # We need to copy /var/lib/docker to /var/lib/docker-embedded in order for HookOS to use the Docker images in its build.
-    docker exec "${dind_container}" sh -c "cp -a /var/lib/docker/* /var/lib/docker-embedded/"
+    docker exec "${dind_container}" sh -c "cp -a /var/lib/docker/* /var/lib/docker-embedded/"    
 }
 
 arch="${1-amd64}"

From 882aa217aa9efbf0bb9a044adc540b4ed79de4d9 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 10:03:52 -0600
Subject: [PATCH 13/22] Fix multi arch:

When existing images in the local Docker image
cache existed for an image the `--platform` arg
doesn't matter. This means that when an existing
amd64 image is already in the cache the arm64
pulls will not happen. To fix this we always delete
the image before pulling.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/pull-images.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index 72fd2f7c..c9a2cd75 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -36,6 +36,12 @@ function docker_pull_image() {
     docker pull --platform=linux/"${arch}" "${image}"
 }
 
+function docker_remove_image() {
+    local image="$1"
+
+    docker rmi "${image}" || true
+}
+
 function main() {
     local dind_container="$1"
     local images_file="$2"
@@ -45,7 +51,8 @@ function main() {
     # Pull the images
     while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
         echo -e "----------------------- $first_image -----------------------"
-        docker_pull_image "${first_image}"
+        docker_remove_image "${first_image}"
+        docker_pull_image "${first_image}" "${arch}"
     done < "${images_file}"
 
     # Save the images
@@ -73,6 +80,7 @@ function main() {
         sleep 1
         if [[ $(docker inspect -f '{{.State.Status}}' "${dind_container}") == "exited" ]]; then
             echo "DinD container exited unexpectedly"
+            docker logs "${dind_container}"
             exit 1
         fi
     done

From b490bac80e10ce27a7e9a700f425c4b6b77f097f Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 11:03:12 -0600
Subject: [PATCH 14/22] Load images from inside DinD:

This resolves issues with needing to
mount the docker.sock and needing sudo.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/pull-images.sh | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index c9a2cd75..4f294539 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -19,14 +19,7 @@ function docker_save_image() {
     local output_dir="$2"
     local output_file="${output_dir}/$(echo "${image}" | tr '/' '-')"
 
-    docker save -o "${output_file}" "${image}"
-}
-
-function docker_load_image() {
-    local image_file="$1"
-    local socket_location="$2"
-
-    sudo -E DOCKER_HOST=unix://"${socket_location}" docker load -i "${image_file}"
+    docker save -o "${output_file}".tar "${image}"
 }
 
 function docker_pull_image() {
@@ -73,7 +66,7 @@ function main() {
     # will change the permissions of the bind mount directory (images/) to root.
     echo -e "Starting DinD container"
     echo -e "-----------------------"
-    docker run -d --privileged --name "${dind_container}" -v ${PWD}/docker:/run -v ${PWD}/images/:/var/lib/docker-embedded/ -d "${dind_container_image}"
+    docker run -d --privileged --name "${dind_container}" -v "${PWD}/images_tar":/images_tar -v "${PWD}"/images/:/var/lib/docker-embedded/ -d "${dind_container_image}"
 
     # wait until the docker daemon is ready
     until docker exec "${dind_container}" docker info &> /dev/null; do
@@ -90,7 +83,8 @@ function main() {
 
     # Load the images
     for image_file in "${output_dir}"/*; do
-        docker_load_image "${image_file}" "${PWD}/docker/docker.sock"
+        echo -e "Loading image: ${image_file}"
+        docker exec "${dind_container}" docker load -i "/images_tar/$(basename ${image_file})"
     done
 
     # clean up tar files

From 4a8f7ce98cf595aa6fea91817f1f2d510b5326f6 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 12:36:20 -0600
Subject: [PATCH 15/22] Add note on Docker storage driver:

Because hook-docker uses the overlay2
storage driver, the local docker client
using pull-images.sh must also use the
overlay2 storage driver.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index b5a14185..b7b4c08b 100644
--- a/README.md
+++ b/README.md
@@ -168,6 +168,8 @@ For use cases where having container images already available in Docker is neede
 
 > Note: This will increase the overall size of HookOS. As HookOS is an in memory OS, make sure that the size increase works for the machines you are provisioning.
 
+> Note: Hook-docker uses the `overlay2` storage driver. For the `pull-images.sh` script to work, your local Docker storage driver must be `overlay2`.
+
 1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
 1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
 1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run [`pull-images.sh`](images/hook-embedded/pull-images.sh) script when building amd64 images and run [`pull-images.sh arm64`](images/hook-embedded/pull-images.sh) when building arm64 images. Read the comments at the top of the script for more details.

From 58eb3941589d4d010ec5fb0fa755f385d0f52699 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 14:27:14 -0600
Subject: [PATCH 16/22] Remove note on host Docker storage driver:

The host Docker storage driver actually doesn't
matter at all because we use DinD. This was my
mistake.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index b7b4c08b..b5a14185 100644
--- a/README.md
+++ b/README.md
@@ -168,8 +168,6 @@ For use cases where having container images already available in Docker is neede
 
 > Note: This will increase the overall size of HookOS. As HookOS is an in memory OS, make sure that the size increase works for the machines you are provisioning.
 
-> Note: Hook-docker uses the `overlay2` storage driver. For the `pull-images.sh` script to work, your local Docker storage driver must be `overlay2`.
-
 1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
 1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
 1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run [`pull-images.sh`](images/hook-embedded/pull-images.sh) script when building amd64 images and run [`pull-images.sh arm64`](images/hook-embedded/pull-images.sh) when building arm64 images. Read the comments at the top of the script for more details.

From de8a3160292bd44eaec77d8c5e074ffa7a2e312f Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 15:34:30 -0600
Subject: [PATCH 17/22] Update images.txt format:

This adds the ability to remove the "source
image" tag from the final embedded images.
This leaves only the "additional tag".

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/images.txt.example | 8 ++++----
 images/hook-embedded/pull-images.sh     | 7 +++++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/images/hook-embedded/images.txt.example b/images/hook-embedded/images.txt.example
index c0c6bb88..c21c8267 100644
--- a/images/hook-embedded/images.txt.example
+++ b/images/hook-embedded/images.txt.example
@@ -1,9 +1,9 @@
 # This is an example file. It explains the required format.
 # For the actual file, you must remove all the comments.
-# The format is source image, a single space, optional additional tag of the source image.
-#<source image> <optional additional tag of the source image>
+# The format is source image, a single space, optional additional tag of the source image, a single space, true or false to remove the original tag.
+#<source image> <optional additional tag of the source image> <remove original tag>
 # for example:
 quay.io/tinkerbell/tink-worker:v0.10.0
-quay.io/tinkerbell/tink-worker:v0.10.0 tink-worker:v0.10.0
+quay.io/tinkerbell/tink-worker:v0.10.0 tink-worker:v0.10.0 true
 quay.io/tinkerbell/actions/image2disk embedded/actions/image2disk
-quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec
+quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec true
diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index 4f294539..cbea5a20 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -90,10 +90,13 @@ function main() {
     # clean up tar files
     rm -rf "${output_dir}"/*
 
-    # Create any tags for the images
-    while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
+    # Create any tags for the images and remove any original tags
+    while IFS=" " read -r first_image image_tag remove_original || [ -n "${first_image}" ] ; do
         if [[ "${image_tag}" != "" ]]; then
             docker exec "${dind_container}" docker tag "${first_image}" "${image_tag}"
+            if [[ "${remove_original}" == "true" ]]; then
+                docker exec "${dind_container}" docker rmi "${first_image}"
+            fi
         fi
     done < "${images_file}"
 

From 67a2d6cd43d5d99b7285d3583a29257c3a05c56a Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 16:21:37 -0600
Subject: [PATCH 18/22] Add some code comments

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/pull-images.sh | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index cbea5a20..b58ba408 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -1,16 +1,7 @@
 #!/bin/bash
 
-# This script is used to pull 
-# This script is used to build an image that is embedded in HookOS.
-# The image contains the /var/lib/docker directory which has pulled images
-# from the images.txt file. When HookOS boots up, the DinD container will
-# have all the images in its cache.
-
-# The purpose of doing this is so that EKS Anywhere doesn't have to set registry
-# and registry credentials in each Hardware object.
-
-# In my testing the initramfs for HookOS with these embedded images was about 334MB.
-# The machine booting HookOS needed 6GB of RAM to boot up successfully.
+# This script is used to build container images that are embedded in HookOS.
+# When HookOS boots up, the DinD container will have all the images in its cache.
 
 set -euo pipefail
 
@@ -44,6 +35,7 @@ function main() {
     # Pull the images
     while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
         echo -e "----------------------- $first_image -----------------------"
+        # Remove the image if it exists so that the image pulls the correct architecture
         docker_remove_image "${first_image}"
         docker_pull_image "${first_image}" "${arch}"
     done < "${images_file}"

From 845d6b232a8524c635b03da263f8724277e9cf48 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Mon, 26 Aug 2024 18:19:35 -0600
Subject: [PATCH 19/22] Check that DinD uses the overlay2 storage driver:

As hook-docker uses the overlay2 storage driver
the DinD image must use the overlay2 storage driver too.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-embedded/pull-images.sh | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh
index b58ba408..45bf18da 100755
--- a/images/hook-embedded/pull-images.sh
+++ b/images/hook-embedded/pull-images.sh
@@ -26,6 +26,16 @@ function docker_remove_image() {
     docker rmi "${image}" || true
 }
 
+function trap_handler() {
+    local dind_container="$1"
+
+    if [[ "${remove_dind_container}" == "true" ]]; then
+        docker rm -f "${dind_container}" &> /dev/null
+    else
+        echo "DinD container NOT removed, please remove it manually"
+    fi
+}
+
 function main() {
     local dind_container="$1"
     local images_file="$2"
@@ -47,10 +57,11 @@ function main() {
         docker_save_image "${first_image}" "${output_dir}"
     done < "${images_file}"
 
+    export remove_dind_container="true"
     # as this function maybe called multiple times, we need to ensure the container is removed
-    trap "docker rm -f "${dind_container}" &> /dev/null" RETURN
+    trap "trap_handler ${dind_container}" RETURN
     # we're using set -e so the trap on RETURN will not be executed when a command fails
-    trap "docker rm -f "${dind_container}" &> /dev/null" EXIT
+    trap "trap_handler ${dind_container}" EXIT
 
     # start DinD container
     # In order to avoid the src bind mount directory (./images/) ownership from changing to root
@@ -70,6 +81,16 @@ function main() {
         fi
     done
 
+    # As hook-docker uses the overlay2 storage driver the DinD must use the overlay2 storage driver too.
+    # make sure the overlay2 storage driver is used by the DinD container.
+    # The VFS storage driver might get used if /var/lib/docker in the DinD container cannot be used by overlay2.
+    storage_driver=$(docker exec "${dind_container}" docker info --format '{{.Driver}}')
+    if [[ "${storage_driver}" != "overlay2" ]]; then
+        export remove_dind_container="false"
+        echo "DinD container is not using overlay2 storage driver, storage driver detected: ${storage_driver}"
+        exit 1
+    fi
+
     # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues.
     docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*"
 
@@ -97,7 +118,7 @@ function main() {
 }
 
 arch="${1-amd64}"
-dind_container_name="hookos-dind-${arch}"
+dind_container_name="hookos-dind"
 images_file="images.txt"
 dind_container_image="${2-docker:dind}"
 main "${dind_container_name}" "${images_file}" "${arch}" "${dind_container_image}"

From 6be6b096fdcea3d1d6146a2b4917b542aee0ae53 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Tue, 27 Aug 2024 10:14:20 -0600
Subject: [PATCH 20/22] Add ssl certs to SSH container:

This resolves an issue with things
like apk not working in the ssh container.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 linuxkit-templates/hook.template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml
index 5864bdc2..56187f09 100644
--- a/linuxkit-templates/hook.template.yaml
+++ b/linuxkit-templates/hook.template.yaml
@@ -231,6 +231,7 @@ services:
 #SSH_SERVER      - /etc/profile.d/local.sh:/etc/profile.d/local.sh
 #SSH_SERVER      - /root/.ssh/authorized_keys:/root/.ssh/authorized_keys
 #SSH_SERVER      - /usr/bin/nerdctl:/usr/bin/nerdctl
+#SSH_SERVER      - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt
 #SSH_SERVER      - /:/host_root
 
 

From 50cdc1da47e738bf32bab17c306adb586b276557 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Tue, 27 Aug 2024 10:48:05 -0600
Subject: [PATCH 21/22] Fix build process:

Remove tabs that were causing the build
to fail.

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 bash/linuxkit.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bash/linuxkit.sh b/bash/linuxkit.sh
index 1e9461fc..5f053256 100644
--- a/bash/linuxkit.sh
+++ b/bash/linuxkit.sh
@@ -70,7 +70,7 @@ function linuxkit_build() {
 			HOOK_CONTAINER_MDEV_IMAGE="${HOOK_CONTAINER_MDEV_IMAGE}" \
 			HOOK_CONTAINER_CONTAINERD_IMAGE="${HOOK_CONTAINER_CONTAINERD_IMAGE}" \
 			HOOK_CONTAINER_RUNC_IMAGE="${HOOK_CONTAINER_RUNC_IMAGE}" \
-			HOOK_CONTAINER_EMBEDDED_IMAGE="${HOOK_CONTAINER_EMBEDDED_IMAGE}" \			
+			HOOK_CONTAINER_EMBEDDED_IMAGE="${HOOK_CONTAINER_EMBEDDED_IMAGE}" \
 			envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE $HOOK_CONTAINER_EMBEDDED_IMAGE' \
 			> "hook.${inventory_id}.yaml"
 

From 78f2850cbed2cd86d594ccfa788e6894fe901205 Mon Sep 17 00:00:00 2001
From: Jacob Weinstock <jakobweinstock@gmail.com>
Date: Tue, 27 Aug 2024 13:10:11 -0600
Subject: [PATCH 22/22] Update hook-bootkit:

Update go.mod dependencies. Check for tink-worker
image and don't fail the image pull if it doesn't
exist. With embedded images, the tink worker could
potentially already exist in the local Docker image
cache. And the image name could be something
unreachable via the network
(for example: 127.0.0.1/embedded/tink-worker).

Signed-off-by: Jacob Weinstock <jakobweinstock@gmail.com>
---
 images/hook-bootkit/Dockerfile |  2 +-
 images/hook-bootkit/go.mod     |  4 +++-
 images/hook-bootkit/main.go    | 39 ++++++++++++++++++++++------------
 3 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/images/hook-bootkit/Dockerfile b/images/hook-bootkit/Dockerfile
index 347e7651..125e57a4 100644
--- a/images/hook-bootkit/Dockerfile
+++ b/images/hook-bootkit/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine AS dev
+FROM golang:1.22.6-alpine AS dev
 COPY . /src/
 WORKDIR /src
 RUN go mod download
diff --git a/images/hook-bootkit/go.mod b/images/hook-bootkit/go.mod
index 76a259a7..bec5c940 100644
--- a/images/hook-bootkit/go.mod
+++ b/images/hook-bootkit/go.mod
@@ -1,6 +1,8 @@
 module github.com/tinkerbell/hook/hook-bootkit
 
-go 1.17
+go 1.22
+
+toolchain go1.22.6
 
 require (
 	github.com/cenkalti/backoff/v4 v4.3.0
diff --git a/images/hook-bootkit/main.go b/images/hook-bootkit/main.go
index 8a8d6d97..a752ed0f 100644
--- a/images/hook-bootkit/main.go
+++ b/images/hook-bootkit/main.go
@@ -15,8 +15,8 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/client"
@@ -127,13 +127,22 @@ func run(ctx context.Context, log logr.Logger) error {
 
 	authStr := base64.URLEncoding.EncodeToString(encodedJSON)
 
-	pullOpts := types.ImagePullOptions{
+	pullOpts := image.PullOptions{
 		RegistryAuth: authStr,
 	}
 	var out io.ReadCloser
 	imagePullOperation := func() error {
+		// with embedded images, the tink worker could potentially already exist
+		// in the local Docker image cache. And the image name could be something
+		// unreachable via the network (for example: 127.0.0.1/embedded/tink-worker).
+		// Because of this we check if the image already exists and don't return an
+		// error if the image does not exist and the pull fails.
+		var imageExists bool
+		if _, _, err := cli.ImageInspectWithRaw(ctx, imageName); err == nil {
+			imageExists = true
+		}
 		out, err = cli.ImagePull(ctx, imageName, pullOpts)
-		if err != nil {
+		if err != nil && !imageExists {
 			log.Error(err, "image pull failure", "imageName", imageName)
 			return err
 		}
@@ -143,18 +152,20 @@ func run(ctx context.Context, log logr.Logger) error {
 		return err
 	}
 
-	buf := bufio.NewScanner(out)
-	for buf.Scan() {
-		structured := make(map[string]interface{})
-		if err := json.Unmarshal(buf.Bytes(), &structured); err != nil {
-			log.Info("image pull logs", "output", buf.Text())
-		} else {
-			log.Info("image pull logs", "logs", structured)
-		}
+	if out != nil {
+		buf := bufio.NewScanner(out)
+		for buf.Scan() {
+			structured := make(map[string]interface{})
+			if err := json.Unmarshal(buf.Bytes(), &structured); err != nil {
+				log.Info("image pull logs", "output", buf.Text())
+			} else {
+				log.Info("image pull logs", "logs", structured)
+			}
 
-	}
-	if err := out.Close(); err != nil {
-		log.Error(err, "closing image pull logs failed")
+		}
+		if err := out.Close(); err != nil {
+			log.Error(err, "closing image pull logs failed")
+		}
 	}
 
 	log.Info("Removing any existing tink-worker container")