PE-5343: add earthly entrypoint script to fix build with proxy and ce…

…rt (#300) * fix: add earthly entrypoint script to fix build with proxy and cert Signed-off-by: Nianyu Shen <[email protected]> * fix: install-k8s should respect cert Signed-off-by: Nianyu Shen <[email protected]> --------- Signed-off-by: Nianyu Shen <[email protected]>
spectrocloud · Oct 18, 2024 · fb045ee · fb045ee
1 parent f0ff666
commit fb045ee
Show file tree

Hide file tree

Showing 3 changed files with 131 additions and 2 deletions.
diff --git a/Earthfile b/Earthfile
@@ -230,6 +230,7 @@ kairos-agent:
 
 install-k8s:
     FROM --platform=linux/${ARCH} $ALPINE_IMG
+    DO +BASE_ALPINE
     COPY (+third-party/luet --binary=luet) /usr/bin/luet
 
     IF [ "$K8S_DISTRIBUTION" = "kubeadm" ] || [ "$K8S_DISTRIBUTION" = "kubeadm-fips" ] || [ "$K8S_DISTRIBUTION" = "nodeadm" ]

diff --git a/earthly-entrypoint.sh b/earthly-entrypoint.sh
@@ -0,0 +1,95 @@
+#!/bin/sh
+# uncomment the line below to enable debug mode
+set -ex
+cp /workspace/sc.crt /usr/local/share/ca-certificates/sc.crt
+update-ca-certificates
+
+# reference: https://github.com/earthly/earthly/blob/main/earthly-entrypoint.sh
+EARTHLY_DEBUG=${EARTHLY_DEBUG:-false}
+if [ "$EARTHLY_DEBUG" = "true" ]; then
+    set -x
+    export EARTHLY_DEBUG
+fi
+
+earthly_config="/etc/.earthly/config.yml"
+if [ ! -f "$earthly_config" ]; then
+    # Missing config, generate it and use the env vars
+    # Do not do both, since that would write to the mounted config
+    mkdir -p "$(dirname $earthly_config)" && touch "$earthly_config"
+
+    # Apply global configuration
+    if [ -n "$GLOBAL_CONFIG" ]; then
+        earthly --config "$earthly_config" config global "$GLOBAL_CONFIG"
+    fi
+
+    # Apply git configuration
+    if [ -n "$GIT_CONFIG" ]; then
+        earthly --config $earthly_config config git "$GIT_CONFIG"
+    fi
+fi
+
+# If no host specified, start an internal buildkit. If it is specified, rely on external setup
+if [ -z "$NO_BUILDKIT" ]; then
+    if [ -z "$BUILDKIT_HOST" ]; then
+        if ! captest --text | grep sys_admin >/dev/null; then
+            echo 1>&2 "Container appears to be running unprivileged. Currently, privileged mode is required when buildkit runs inside the container."
+            echo 1>&2 "To run this image without buildkit, set the environment variable NO_BUILDKIT=1"
+            exit 1
+        fi
+
+        if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then
+            echo >&2 "detected cgroups v2; earthly-entrypoint.sh running under pid=$$ with controllers \"$(cat /sys/fs/cgroup/cgroup.controllers)\" in group $(cat /proc/self/cgroup)"
+            test "$(cat /sys/fs/cgroup/cgroup.type)" = "domain" || (echo >&2 "WARNING: invalid root cgroup type: $(cat /sys/fs/cgroup/cgroup.type)")
+        fi
+
+        # generate certificates
+        earthly --config "$earthly_config" --buildkit-host=tcp://127.0.0.1:8372 bootstrap --certs-hostname="$(hostname)" --no-buildkit --force-certificate-generation
+
+        if [ ! -f /etc/ca.pem ]; then
+            ln -s /root/.earthly/certs/ca_cert.pem /etc/ca.pem
+        fi
+
+        if [ ! -f /etc/cert.pem ]; then
+            ln -s /root/.earthly/certs/buildkit_cert.pem /etc/cert.pem
+        fi
+
+        if [ ! -f /etc/key.pem ]; then
+            ln -s /root/.earthly/certs/buildkit_key.pem /etc/key.pem
+        fi
+
+        export BUILDKIT_TCP_TRANSPORT_ENABLED=true
+        export BUILDKIT_TLS_ENABLED=true
+
+        /usr/bin/entrypoint.sh \
+            buildkitd \
+            --config=/etc/buildkitd.toml \
+            >/var/log/buildkitd.log 2>&1 \
+            &
+
+        if [ "$BUILDKIT_DEBUG" = "true" ]; then
+            tail -f /var/log/buildkitd.log &
+        fi
+
+        EARTHLY_BUILDKIT_HOST="tcp://$(hostname):8372" # hostname is not recognized as local for this reason
+        export EARTHLY_BUILDKIT_HOST
+    else
+        export EARTHLY_BUILDKIT_HOST="$BUILDKIT_HOST"
+    fi
+    ! "$EARTHLY_DEBUG" || echo 1>&2 "Using $EARTHLY_BUILDKIT_HOST as buildkit daemon"
+fi
+
+if [ -n "$SRC_DIR" ]; then
+    echo 1>&2 'Please note that SRC_DIR is deprecated. This script will no longer automatically switch to it in the future.'
+    echo 1>&2 'Please change the container'"'"'s working directory instead (e.g. via docker run -w)'
+    cd "$SRC_DIR"
+fi
+
+if [ -n "$EARTHLY_EXEC_CMD" ]; then
+    export earthly_config
+    exec "$EARTHLY_EXEC_CMD"
+    exit 1 # this should never be reached
+fi
+
+# Run earthly with given args.
+# Exec so we don't have to trap and manage signal propagation
+exec earthly --config "$earthly_config" "$@"
diff --git a/earthly.sh b/earthly.sh
@@ -11,12 +11,45 @@ function build_with_proxy() {
         docker stop earthly-buildkitd
     fi
     # start earthly buildkitd
-    docker run -d --privileged --name earthly-buildkitd -v ~/.docker/config.json:/root/.docker/config.json -v /var/run/docker.sock:/var/run/docker.sock --rm -t -e GLOBAL_CONFIG="$global_config" -e BUILDKIT_TCP_TRANSPORT_ENABLED=true -e http_proxy=$HTTP_PROXY -e https_proxy=$HTTPS_PROXY -e HTTPS_PROXY=$HTTPS_PROXY -e HTTP_PROXY=$HTTP_PROXY -e NO_PROXY=$NO_PROXY -e no_proxy=$no_proxy -e EARTHLY_GIT_CONFIG=$gitconfig -v "$PROXY_CERT_PATH:/usr/local/share/ca-certificates/sc.crt:ro" -v earthly-tmp:/tmp/earthly:rw -p 8372:8372 $SPECTRO_PUB_REPO/third-party/edge/earthly/buildkitd:$EARTHLY_VERSION
+    docker run -d --privileged \
+        --name earthly-buildkitd \
+        -v ~/.docker/config.json:/root/.docker/config.json \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        --rm -t \
+        -e GLOBAL_CONFIG="$global_config" \
+        -e BUILDKIT_TCP_TRANSPORT_ENABLED=true \
+        -e http_proxy=$HTTP_PROXY \
+        -e https_proxy=$HTTPS_PROXY \
+        -e HTTPS_PROXY=$HTTPS_PROXY \
+        -e HTTP_PROXY=$HTTP_PROXY \
+        -e NO_PROXY=$NO_PROXY \
+        -e no_proxy=$NO_PROXY \
+        -e EARTHLY_GIT_CONFIG=$gitconfig \
+        -v "$PROXY_CERT_PATH:/usr/local/share/ca-certificates/sc.crt:ro" \
+        -v earthly-tmp:/tmp/earthly:rw \
+        -p 8372:8372 \
+        $SPECTRO_PUB_REPO/third-party/edge/earthly/buildkitd:$EARTHLY_VERSION
     # Update the CA certificates in the container
     docker exec -it earthly-buildkitd update-ca-certificates
 
     # Run Earthly in Docker to create artifacts  Variables are passed from the .arg file
-    docker run --privileged -v ~/.docker/config.json:/root/.docker/config.json -v /var/run/docker.sock:/var/run/docker.sock --rm --env EARTHLY_BUILD_ARGS -t -e GLOBAL_CONFIG="$global_config" -e EARTHLY_BUILDKIT_HOST=tcp://0.0.0.0:8372 -e BUILDKIT_TLS_ENABLED=false -v "$(pwd)":/workspace -v "$PROXY_CERT_PATH:/workspace/sc.crt:ro" $SPECTRO_PUB_REPO/third-party/edge/earthly/earthly:$EARTHLY_VERSION --allow-privileged "$@"
+    docker run --privileged \
+        -v ~/.docker/config.json:/root/.docker/config.json \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        --rm --env EARTHLY_BUILD_ARGS -t \
+        -e GLOBAL_CONFIG="$global_config" \
+        -e EARTHLY_BUILDKIT_HOST=tcp://0.0.0.0:8372 \
+        -e BUILDKIT_TLS_ENABLED=false \
+        -e http_proxy=$HTTP_PROXY \
+        -e https_proxy=$HTTPS_PROXY \
+        -e HTTPS_PROXY=$HTTPS_PROXY \
+        -e HTTP_PROXY=$HTTP_PROXY \
+        -e NO_PROXY=$NO_PROXY \
+        -e no_proxy=$NO_PROXY \
+        -v "$(pwd)":/workspace \
+        -v "$PROXY_CERT_PATH:/workspace/sc.crt:ro" \
+        --entrypoint /workspace/earthly-entrypoint.sh \
+        $SPECTRO_PUB_REPO/third-party/edge/earthly/earthly:$EARTHLY_VERSION --allow-privileged "$@"
 }
 
 function build_without_proxy() {