diff --git a/.github/workflows/test-kube.yml b/.github/workflows/test-kube.yml new file mode 100644 index 00000000000..c114196a045 --- /dev/null +++ b/.github/workflows/test-kube.yml @@ -0,0 +1,29 @@ +# This pipeline purpose is solely meant to run a subset of our test suites against a kube cluster +name: kube + +on: + push: + branches: + - main + - 'release/**' + pull_request: + paths-ignore: + - '**.md' + +env: + ROOTFUL: true + +jobs: + linux: + runs-on: "ubuntu-24.04" + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4.1.7 + with: + fetch-depth: 1 + - name: "Prepare integration test environment" + run: | + ./hack/build-integration-kube.sh + - name: "Run Kube integration tests" + run: | + sudo ./_output/nerdctl exec -ti nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test ./cmd/nerdctl/ -test.kube' diff --git a/cmd/nerdctl/container_commit_test.go b/cmd/nerdctl/container_commit_test.go index b569ebfbcc2..e02e47dc8b8 100644 --- a/cmd/nerdctl/container_commit_test.go +++ b/cmd/nerdctl/container_commit_test.go @@ -20,6 +20,8 @@ import ( "fmt" "testing" + "gotest.tools/v3/assert" + "github.com/containerd/nerdctl/v2/pkg/testutil" ) @@ -53,3 +55,12 @@ func TestCommit(t *testing.T) { base.Cmd("rmi", testImage).Run() } } + +func TestKubeCommit(t *testing.T) { + t.Parallel() + _ = testutil.NewBaseForKube(t) + + t.Run("not an actual test", func(t *testing.T) { + assert.Assert(t, true, "this is a non-test") + }) +} diff --git a/hack/build-integration-canary.sh b/hack/build-integration-canary.sh index 8091a192034..4a75a0d427b 100755 --- a/hack/build-integration-canary.sh +++ b/hack/build-integration-canary.sh @@ -16,6 +16,10 @@ # shellcheck disable=SC2034,SC2015 set -o errexit -o errtrace -o functrace -o nounset -o pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)" +readonly root +# shellcheck source=/dev/null +. "$root/scripts/lib.sh" ###################### # Definitions @@ -67,141 +71,6 @@ STARGZ_SNAPSHOTTER_CHECKSUM=linux # We specifically want the static ones TINI_CHECKSUM=static - -###################### -# Lib -###################### - -# Simple logger -readonly LOG_LEVEL_DEBUG=0 -readonly LOG_LEVEL_INFO=1 -readonly LOG_LEVEL_WARNING=2 -readonly LOG_LEVEL_ERROR=3 - -readonly LOG_COLOR_BLACK=0 -readonly LOG_COLOR_RED=1 -readonly LOG_COLOR_GREEN=2 -readonly LOG_COLOR_YELLOW=3 -readonly LOG_COLOR_BLUE=4 -readonly LOG_COLOR_MAGENTA=5 -readonly LOG_COLOR_CYAN=6 -readonly LOG_COLOR_WHITE=7 -readonly LOG_COLOR_DEFAULT=9 - -readonly LOG_STYLE_DEBUG=( setaf "$LOG_COLOR_WHITE" ) -readonly LOG_STYLE_INFO=( setaf "$LOG_COLOR_GREEN" ) -readonly LOG_STYLE_WARNING=( setaf "$LOG_COLOR_YELLOW" ) -readonly LOG_STYLE_ERROR=( setaf "$LOG_COLOR_RED" ) - -_log::log(){ - local level - local style - local numeric_level - local message="$2" - - level="$(printf "%s" "$1" | tr '[:lower:]' '[:upper:]')" - numeric_level="$(printf "LOG_LEVEL_%s" "$level")" - style="LOG_STYLE_${level}[@]" - - [ "${!numeric_level}" -ge "$LOG_LEVEL" ] || return 0 - - [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput "${!style}" 2>/dev/null || true - >&2 printf "[%s] %s: %s\n" "$(date 2>/dev/null || true)" "$(printf "%s" "$level" | tr '[:lower:]' '[:upper:]')" "$message" - [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput op 2>/dev/null || true -} - -log::init(){ - local _ll - # Default log to warning if unspecified - _ll="$(printf "LOG_LEVEL_%s" "${NERDCTL_CI_LOG_LEVEL:-warning}" | tr '[:lower:]' '[:upper:]')" - # Default to 3 (warning) if unrecognized - LOG_LEVEL="${!_ll:-3}" -} - -log::debug(){ - _log::log debug "$@" -} - -log::info(){ - _log::log info "$@" -} - -log::warning(){ - _log::log warning "$@" -} - -log::error(){ - _log::log error "$@" -} - -# Helpers -host::require(){ - local binary="$1" - command -v "$binary" >/dev/null || { - log::error "You need $binary for this script to work, and it cannot be found in your path" - exit 1 - } -} - -fs::mktemp(){ - mktemp -dq "${TMPDIR:-/tmp}/$prefix.XXXXXX" 2>/dev/null || mktemp -dq || { - log::error "Failed to create temporary directory" - exit 1 - } -} - -http::get(){ - local args=(curl --proto '=https' --tlsv1.2 -fsSL) - args+=("$@") - - log::debug "${args[*]}" - "${args[@]}" -} - -http::checksum(){ - local urls=("$@") - local url - - local prefix="nerdctl-checksum" - - local temp - temp="$(fs::mktemp)" - - for url in "${urls[@]}"; do - http::get -o "$temp/${url##*/}" "$url" - done - - cd "$temp" - shasum -a 256 ./* - cd - >/dev/null || true -} - -# Github API helpers -# Set GITHUB_TOKEN to use authenticated requests to workaround limitations -github::request(){ - local endpoint="$1" - local args=( - -H "Accept: application/vnd.github+json" - -H "X-GitHub-Api-Version: 2022-11-28" - ) - - [ "${GITHUB_TOKEN:-}" == "" ] || args+=(-H "Authorization: Bearer $GITHUB_TOKEN") - - http::get "${args[@]}" https://api.github.com/"$endpoint" -} - -github::tags::getlatest(){ - local repo="$1" - github::request "repos/$repo/tags" | - jq -rc .[0].name -} - -github::releases::latest(){ - local repo="$1" - github::request "repos/$repo/releases" | - jq -rc .[] -} - version::compare(){ local raw_version_fd="$1" local parsed @@ -304,7 +173,7 @@ latest::release(){ higher_data="$line" higher_readable="$(echo "$line" | jq -rc .name | sed -E 's/(.*[ ])?(v?[0-9][0-9.a-z-]+).*/\2/')" fi - done < <(github::releases::latest "$repo") + done < <(github::releases "$repo") log::info " >>> latest release detected: $higher_readable" } @@ -314,7 +183,7 @@ latest::tag(){ local repo="$1" log::info "Analyzing tags for $repo" - github::tags::getlatest "$repo" + github::tags::latest "$repo" } # Once a latest release has been retrieved for a given project, you can get the url to the asset matching OS and ARCH @@ -342,13 +211,6 @@ assets::get(){ } } -log::init -host::require jq -host::require curl -host::require shasum -host::require docker -host::require tput - ###################### # Script ###################### diff --git a/hack/build-integration-kube.sh b/hack/build-integration-kube.sh new file mode 100755 index 00000000000..ab0d8c813bf --- /dev/null +++ b/hack/build-integration-kube.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2034,SC2015 +set -o errexit -o errtrace -o functrace -o nounset -o pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)" +readonly root +# shellcheck source=/dev/null +. "$root/scripts/lib.sh" + +GO_VERSION=1.22 +KIND_VERSION=v0.23.0 + +[ "$(uname -m)" == "aarch64" ] && GOARCH=arm64 || GOARCH=amd64 + +_rootful= + +configure::rootful(){ + log::debug "Configuring rootful to: ${1:+true}" + _rootful="${1:+true}" +} + +install::kind(){ + local version="$1" + local temp + temp="$(fs::mktemp "install")" + + http::get "$temp"/kind "https://kind.sigs.k8s.io/dl/$version/kind-linux-${GOARCH:-amd64}" + host::install "$temp"/kind +} + +# shellcheck disable=SC2120 +install::kubectl(){ + local version="${1:-}" + [ "$version" ] || version="$(http::get /dev/stdout https://dl.k8s.io/release/stable.txt)" + local temp + temp="$(fs::mktemp "install")" + + http::get "$temp"/kubectl "https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/${GOARCH:-amd64}/kubectl" + host::install "$temp"/kubectl +} + +exec::kind(){ + local args=() + [ ! "$_rootful" ] || args=(sudo -E) + args+=(kind) + + log::debug "${args[*]} $*" + "${args[@]}" "$@" +} + +exec::nerdctl(){ + local args=() + [ ! "$_rootful" ] || args=(sudo -E) + args+=("$(pwd)"/_output/nerdctl) + + log::debug "${args[*]} $*" + "${args[@]}" "$@" +} + +# Install dependencies +main(){ + log::info "Configuring rootful" + configure::rootful "${ROOTFUL:-}" + + log::info "Installing host dependencies if necessary" + host::require kind 2>/dev/null || install::kind "$KIND_VERSION" + host::require kubectl 2>/dev/null || install::kubectl + + # Build nerdctl to use for kind + make binaries + PATH=$(pwd)/_output:"$PATH" + export PATH + + # Hack to get go into kind control plane + exec::nerdctl rm -f go-kind 2>/dev/null || true + exec::nerdctl run -d --name go-kind golang:"$GO_VERSION" sleep Inf + exec::nerdctl cp go-kind:/usr/local/go /tmp/go + + # Create fresh cluster + log::info "Creating new cluster" + export KIND_EXPERIMENTAL_PROVIDER=nerdctl + exec::kind delete cluster --name nerdctl-test 2>/dev/null || true + exec::kind create cluster --name nerdctl-test --config=./hack/kind.yaml +} + +main "$@" \ No newline at end of file diff --git a/hack/kind.yaml b/hack/kind.yaml new file mode 100644 index 00000000000..1695fafdb88 --- /dev/null +++ b/hack/kind.yaml @@ -0,0 +1,12 @@ +# https://pkg.go.dev/sigs.k8s.io/kind/pkg/apis/config/v1alpha4#Cluster +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - hostPath: _output/nerdctl + containerPath: /usr/local/bin/nerdctl + - hostPath: /tmp/go + containerPath: /usr/local/go + - hostPath: . + containerPath: /nerdctl-source diff --git a/hack/scripts/lib.sh b/hack/scripts/lib.sh new file mode 100755 index 00000000000..de0fa173bae --- /dev/null +++ b/hack/scripts/lib.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2034,SC2015 +set -o errexit -o errtrace -o functrace -o nounset -o pipefail + +## This is a library of generic helpers that can be used across different projects + +# Simple logger +readonly LOG_LEVEL_DEBUG=0 +readonly LOG_LEVEL_INFO=1 +readonly LOG_LEVEL_WARNING=2 +readonly LOG_LEVEL_ERROR=3 + +readonly LOG_COLOR_BLACK=0 +readonly LOG_COLOR_RED=1 +readonly LOG_COLOR_GREEN=2 +readonly LOG_COLOR_YELLOW=3 +readonly LOG_COLOR_BLUE=4 +readonly LOG_COLOR_MAGENTA=5 +readonly LOG_COLOR_CYAN=6 +readonly LOG_COLOR_WHITE=7 +readonly LOG_COLOR_DEFAULT=9 + +readonly LOG_STYLE_DEBUG=( setaf "$LOG_COLOR_WHITE" ) +readonly LOG_STYLE_INFO=( setaf "$LOG_COLOR_GREEN" ) +readonly LOG_STYLE_WARNING=( setaf "$LOG_COLOR_YELLOW" ) +readonly LOG_STYLE_ERROR=( setaf "$LOG_COLOR_RED" ) + +_log::log(){ + local level + local style + local numeric_level + local message="$2" + + level="$(printf "%s" "$1" | tr '[:lower:]' '[:upper:]')" + numeric_level="$(printf "LOG_LEVEL_%s" "$level")" + style="LOG_STYLE_${level}[@]" + + [ "${!numeric_level}" -ge "$LOG_LEVEL" ] || return 0 + + [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput "${!style}" 2>/dev/null || true + >&2 printf "[%s] %s: %s\n" "$(date 2>/dev/null || true)" "$(printf "%s" "$level" | tr '[:lower:]' '[:upper:]')" "$message" + [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput op 2>/dev/null || true +} + +log::init(){ + local _ll + # Default log to warning if unspecified + _ll="$(printf "LOG_LEVEL_%s" "${LOG_LEVEL:-warning}" | tr '[:lower:]' '[:upper:]')" + # Default to 3 (warning) if unrecognized + LOG_LEVEL="${!_ll:-3}" +} + +log::debug(){ + _log::log debug "$@" +} + +log::info(){ + _log::log info "$@" +} + +log::warning(){ + _log::log warning "$@" +} + +log::error(){ + _log::log error "$@" +} + +# Helpers +host::require(){ + local binary="$1" + + log::debug "Checking presence of $binary" + command -v "$binary" >/dev/null || { + log::error "You need $binary for this script to work, and it cannot be found in your path" + return 1 + } +} + +host::install(){ + local binary + + for binary in "$@"; do + log::debug "sudo install -D -m 755 $binary /usr/local/bin/$(basename "$binary")" + sudo install -D -m 755 "$binary" /usr/local/bin/"$(basename "$binary")" + done +} + +fs::mktemp(){ + local prefix="${1:-temporary}" + + mktemp -dq "${TMPDIR:-/tmp}/$prefix.XXXXXX" 2>/dev/null || mktemp -dq || { + log::error "Failed to create temporary directory" + return 1 + } +} + +tar::expand(){ + local dir="$1" + local arc="$2" + + log::debug "tar -C $dir -xzf $arc" + tar -C "$dir" -xzf "$arc" +} + +_http::get(){ + local url="$1" + local output="$2" + local retry="$3" + local delay="$4" + local user="${5:-}" + local password="${6:-}" + shift + shift + shift + shift + shift + shift + + local header + local command=(curl -fsSL --retry "$retry" --retry-delay "$delay" -o "$output") + # Add a basic auth user if necessary + [ "$user" == "" ] || command+=(--user "$user:$password") + # Force tls v1.2 and no redirect to http if url scheme is https + [ "${url:0:5}" != "https" ] || command+=(--proto '=https' --tlsv1.2) + # Stuff in any additional arguments as headers + for header in "$@"; do + command+=(-H "$header") + done + # Debug + log::debug "${command[*]} $url" + # Exec + "${command[@]}" "$url" || { + log::error "Failed to connect to $url with $retry retries every $delay seconds" + return 1 + } +} + +http::get(){ + local output="$1" + local url="$2" + shift + shift + + _http::get "$url" "$output" "2" "1" "" "" "$@" +} + +http::healthcheck(){ + local url="$1" + local retry="${2:-5}" + local delay="${3:-1}" + local user="${4:-}" + local password="${5:-}" + shift + shift + shift + shift + shift + + _http::get "$url" /dev/null "$retry" "$delay" "$user" "$password" "$@" +} + +http::checksum(){ + local urls=("$@") + local url + + local temp + temp="$(fs::mktemp "http-checksum")" + + for url in "${urls[@]}"; do + http::get "$temp/${url##*/}" "$url" + done + + cd "$temp" + shasum -a 256 ./* + cd - >/dev/null || true +} + +# Github API helpers +# Set GITHUB_TOKEN to use authenticated requests to workaround limitations + +github::settoken(){ + local token="$1" + # If passed token is a github action pattern replace, and we are NOT on github, ignore it + # shellcheck disable=SC2016 + [ "${token:0:3}" == '${{' ] || GITHUB_TOKEN="$token" +} + +github::request(){ + local endpoint="$1" + local args=( + "Accept: application/vnd.github+json" + "X-GitHub-Api-Version: 2022-11-28" + ) + + [ "${GITHUB_TOKEN:-}" == "" ] || args+=("Authorization: Bearer $GITHUB_TOKEN") + + http::get /dev/stdout https://api.github.com/"$endpoint" "${args[@]}" +} + +github::tags::latest(){ + local repo="$1" + github::request "repos/$repo/tags" | jq -rc .[0].name +} + +github::releases(){ + local repo="$1" + github::request "repos/$repo/releases" | + jq -rc .[] +} + +github::releases::latest(){ + local repo="$1" + github::request "repos/$repo/releases/latest" | jq -rc . +} + +log::init +host::require jq +host::require tar +host::require curl +host::require shasum diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go index f42b4861e8e..4797c9460a8 100644 --- a/pkg/testutil/testutil.go +++ b/pkg/testutil/testutil.go @@ -51,6 +51,8 @@ type Base struct { DaemonIsKillable bool EnableIPv6 bool IPv6Compatible bool + EnableKube bool + KubeCompatible bool Binary string Args []string Env []string @@ -539,12 +541,14 @@ var ( flagTestTarget Target flagTestKillDaemon bool flagTestIPv6 bool + flagTestKube bool ) func M(m *testing.M) { flag.StringVar(&flagTestTarget, "test.target", Nerdctl, "target to test") flag.BoolVar(&flagTestKillDaemon, "test.kill-daemon", false, "enable tests that kill the daemon") flag.BoolVar(&flagTestIPv6, "test.ipv6", false, "enable tests on IPv6") + flag.BoolVar(&flagTestKube, "test.kube", false, "enable tests on Kube") flag.Parse() fmt.Fprintf(os.Stderr, "test target: %q\n", flagTestTarget) os.Exit(m.Run()) @@ -561,6 +565,10 @@ func GetEnableIPv6() bool { return flagTestIPv6 } +func GetEnableKube() bool { + return flagTestKube +} + func GetDaemonIsKillable() bool { return flagTestKillDaemon } @@ -680,24 +688,30 @@ func NewBaseWithNamespace(t *testing.T, ns string) *Base { if ns == "" || ns == "default" || ns == Namespace { t.Fatalf(`the other base namespace cannot be "%s"`, ns) } - return newBase(t, ns, false) + return newBase(t, ns, false, false) } func NewBaseWithIPv6Compatible(t *testing.T) *Base { - return newBase(t, Namespace, true) + return newBase(t, Namespace, true, false) +} + +func NewBaseForKube(t *testing.T) *Base { + return newBase(t, Namespace, false, true) } func NewBase(t *testing.T) *Base { - return newBase(t, Namespace, false) + return newBase(t, Namespace, false, false) } -func newBase(t *testing.T, ns string, ipv6Compatible bool) *Base { +func newBase(t *testing.T, ns string, ipv6Compatible bool, kubeCompatible bool) *Base { base := &Base{ T: t, Target: GetTarget(), DaemonIsKillable: GetDaemonIsKillable(), EnableIPv6: GetEnableIPv6(), IPv6Compatible: ipv6Compatible, + EnableKube: GetEnableKube(), + KubeCompatible: kubeCompatible, Env: os.Environ(), } if base.EnableIPv6 && !base.IPv6Compatible { @@ -705,6 +719,11 @@ func newBase(t *testing.T, ns string, ipv6Compatible bool) *Base { } else if !base.EnableIPv6 && base.IPv6Compatible { t.Skip("runner skips IPv6 compatible tests in the non-IPv6 environment") } + if base.EnableKube && !base.KubeCompatible { + t.Skip("runner skips non-kube compatible tests in the kube environment") + } else if !base.EnableKube && base.KubeCompatible { + t.Skip("runner skips kube compatible tests in the non-kube environment") + } var err error switch base.Target { case Nerdctl: