Kube on the CI

Signed-off-by: apostasie <[email protected]>
containerd · Aug 12, 2024 · c8bf25a · c8bf25a
1 parent 728421a
commit c8bf25a
Show file tree

Hide file tree

Showing 7 changed files with 417 additions and 148 deletions.
diff --git a/.github/workflows/test-kube.yml b/.github/workflows/test-kube.yml
@@ -0,0 +1,29 @@
+# This pipeline purpose is solely meant to run a subset of our test suites against a kube cluster
+name: kube
+
+on:
+  push:
+    branches:
+      - main
+      - 'release/**'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+
+env:
+  ROOTFUL: true
+
+jobs:
+  linux:
+    runs-on: "ubuntu-24.04"
+    timeout-minutes: 40
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+      - name: "Prepare integration test environment"
+        run: |
+          ./hack/build-integration-kube.sh
+      - name: "Run Kube integration tests"
+        run: |
+          sudo ./_output/nerdctl exec -ti nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test ./cmd/nerdctl/ -test.kube'
diff --git a/cmd/nerdctl/container_commit_test.go b/cmd/nerdctl/container_commit_test.go
@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"testing"
 
+	"gotest.tools/v3/assert"
+
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 )
 
@@ -53,3 +55,12 @@ func TestCommit(t *testing.T) {
 		base.Cmd("rmi", testImage).Run()
 	}
 }
+
+func TestKubeCommit(t *testing.T) {
+	t.Parallel()
+	_ = testutil.NewBaseForKube(t)
+
+	t.Run("not an actual test", func(t *testing.T) {
+		assert.Assert(t, true, "this is a non-test")
+	})
+}
diff --git a/hack/build-integration-canary.sh b/hack/build-integration-canary.sh
@@ -16,6 +16,10 @@
 
 # shellcheck disable=SC2034,SC2015
 set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
+readonly root
+# shellcheck source=/dev/null
+. "$root/scripts/lib.sh"
 
 ######################
 # Definitions
@@ -67,141 +71,6 @@ STARGZ_SNAPSHOTTER_CHECKSUM=linux
 # We specifically want the static ones
 TINI_CHECKSUM=static
 
-
-######################
-# Lib
-######################
-
-# Simple logger
-readonly LOG_LEVEL_DEBUG=0
-readonly LOG_LEVEL_INFO=1
-readonly LOG_LEVEL_WARNING=2
-readonly LOG_LEVEL_ERROR=3
-
-readonly LOG_COLOR_BLACK=0
-readonly LOG_COLOR_RED=1
-readonly LOG_COLOR_GREEN=2
-readonly LOG_COLOR_YELLOW=3
-readonly LOG_COLOR_BLUE=4
-readonly LOG_COLOR_MAGENTA=5
-readonly LOG_COLOR_CYAN=6
-readonly LOG_COLOR_WHITE=7
-readonly LOG_COLOR_DEFAULT=9
-
-readonly LOG_STYLE_DEBUG=( setaf "$LOG_COLOR_WHITE" )
-readonly LOG_STYLE_INFO=( setaf "$LOG_COLOR_GREEN" )
-readonly LOG_STYLE_WARNING=( setaf "$LOG_COLOR_YELLOW" )
-readonly LOG_STYLE_ERROR=( setaf "$LOG_COLOR_RED" )
-
-_log::log(){
-  local level
-  local style
-  local numeric_level
-  local message="$2"
-
-  level="$(printf "%s" "$1" | tr '[:lower:]' '[:upper:]')"
-  numeric_level="$(printf "LOG_LEVEL_%s" "$level")"
-  style="LOG_STYLE_${level}[@]"
-
-  [ "${!numeric_level}" -ge "$LOG_LEVEL" ] || return 0
-
-  [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput "${!style}" 2>/dev/null || true
-  >&2 printf "[%s] %s: %s\n" "$(date 2>/dev/null || true)" "$(printf "%s" "$level" | tr '[:lower:]' '[:upper:]')" "$message"
-  [ ! "$TERM" ] || [ ! -t 2 ] || >&2 tput op 2>/dev/null || true
-}
-
-log::init(){
-  local _ll
-  # Default log to warning if unspecified
-  _ll="$(printf "LOG_LEVEL_%s" "${NERDCTL_CI_LOG_LEVEL:-warning}" | tr '[:lower:]' '[:upper:]')"
-  # Default to 3 (warning) if unrecognized
-  LOG_LEVEL="${!_ll:-3}"
-}
-
-log::debug(){
-  _log::log debug "$@"
-}
-
-log::info(){
-  _log::log info "$@"
-}
-
-log::warning(){
-  _log::log warning "$@"
-}
-
-log::error(){
-  _log::log error "$@"
-}
-
-# Helpers
-host::require(){
-  local binary="$1"
-  command -v "$binary" >/dev/null || {
-    log::error "You need $binary for this script to work, and it cannot be found in your path"
-    exit 1
-  }
-}
-
-fs::mktemp(){
-  mktemp -dq "${TMPDIR:-/tmp}/$prefix.XXXXXX" 2>/dev/null || mktemp -dq || {
-    log::error "Failed to create temporary directory"
-    exit 1
-  }
-}
-
-http::get(){
-  local args=(curl --proto '=https' --tlsv1.2 -fsSL)
-  args+=("$@")
-
-  log::debug "${args[*]}"
-  "${args[@]}"
-}
-
-http::checksum(){
-  local urls=("$@")
-  local url
-
-  local prefix="nerdctl-checksum"
-
-  local temp
-  temp="$(fs::mktemp)"
-
-  for url in "${urls[@]}"; do
-    http::get -o "$temp/${url##*/}" "$url"
-  done
-
-  cd "$temp"
-  shasum -a 256 ./*
-  cd - >/dev/null || true
-}
-
-# Github API helpers
-# Set GITHUB_TOKEN to use authenticated requests to workaround limitations
-github::request(){
-  local endpoint="$1"
-  local args=(
-    -H "Accept: application/vnd.github+json"
-    -H "X-GitHub-Api-Version: 2022-11-28"
-  )
-
-  [ "${GITHUB_TOKEN:-}" == "" ] || args+=(-H "Authorization: Bearer $GITHUB_TOKEN")
-
-  http::get "${args[@]}" https://api.github.com/"$endpoint"
-}
-
-github::tags::getlatest(){
-  local repo="$1"
-  github::request "repos/$repo/tags" |
-    jq -rc .[0].name
-}
-
-github::releases::latest(){
-  local repo="$1"
-  github::request "repos/$repo/releases" |
-    jq -rc .[]
-}
-
 version::compare(){
   local raw_version_fd="$1"
   local parsed
@@ -304,7 +173,7 @@ latest::release(){
       higher_data="$line"
       higher_readable="$(echo "$line" | jq -rc .name | sed -E 's/(.*[ ])?(v?[0-9][0-9.a-z-]+).*/\2/')"
     fi
-  done < <(github::releases::latest "$repo")
+  done < <(github::releases "$repo")
 
   log::info " >>> latest release detected: $higher_readable"
 }
@@ -314,7 +183,7 @@ latest::tag(){
   local repo="$1"
 
   log::info "Analyzing tags for $repo"
-  github::tags::getlatest "$repo"
+  github::tags::latest "$repo"
 }
 
 # Once a latest release has been retrieved for a given project, you can get the url to the asset matching OS and ARCH
@@ -342,13 +211,6 @@ assets::get(){
   }
 }
 
-log::init
-host::require jq
-host::require curl
-host::require shasum
-host::require docker
-host::require tput
-
 ######################
 # Script
 ######################

diff --git a/hack/build-integration-kube.sh b/hack/build-integration-kube.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# shellcheck disable=SC2034,SC2015
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
+readonly root
+# shellcheck source=/dev/null
+. "$root/scripts/lib.sh"
+
+GO_VERSION=1.22
+KIND_VERSION=v0.23.0
+
+[ "$(uname -m)" == "aarch64" ] && GOARCH=arm64 || GOARCH=amd64
+
+_rootful=
+
+configure::rootful(){
+  log::debug "Configuring rootful to: ${1:+true}"
+  _rootful="${1:+true}"
+}
+
+install::kind(){
+  local version="$1"
+  local temp
+  temp="$(fs::mktemp "install")"
+
+  http::get "$temp"/kind "https://kind.sigs.k8s.io/dl/$version/kind-linux-${GOARCH:-amd64}"
+  host::install "$temp"/kind
+}
+
+# shellcheck disable=SC2120
+install::kubectl(){
+  local version="${1:-}"
+  [ "$version" ] || version="$(http::get /dev/stdout https://dl.k8s.io/release/stable.txt)"
+  local temp
+  temp="$(fs::mktemp "install")"
+
+  http::get "$temp"/kubectl "https://storage.googleapis.com/kubernetes-release/release/$version/bin/linux/${GOARCH:-amd64}/kubectl"
+  host::install "$temp"/kubectl
+}
+
+exec::kind(){
+  local args=()
+  [ ! "$_rootful" ] || args=(sudo -E)
+  args+=(kind)
+
+  log::debug "${args[*]} $*"
+  "${args[@]}" "$@"
+}
+
+exec::nerdctl(){
+  local args=()
+  [ ! "$_rootful" ] || args=(sudo -E)
+  args+=("$(pwd)"/_output/nerdctl)
+
+  log::debug "${args[*]} $*"
+  "${args[@]}" "$@"
+}
+
+# Install dependencies
+main(){
+  log::info "Configuring rootful"
+  configure::rootful "${ROOTFUL:-}"
+
+  log::info "Installing host dependencies if necessary"
+  host::require kind 2>/dev/null || install::kind "$KIND_VERSION"
+  host::require kubectl 2>/dev/null || install::kubectl
+
+  # Build nerdctl to use for kind
+  make binaries
+  PATH=$(pwd)/_output:"$PATH"
+  export PATH
+
+  # Hack to get go into kind control plane
+  exec::nerdctl rm -f go-kind 2>/dev/null || true
+  exec::nerdctl run -d --name go-kind golang:"$GO_VERSION" sleep Inf
+  exec::nerdctl cp go-kind:/usr/local/go /tmp/go
+
+  # Create fresh cluster
+  log::info "Creating new cluster"
+  export KIND_EXPERIMENTAL_PROVIDER=nerdctl
+  exec::kind delete cluster --name nerdctl-test 2>/dev/null || true
+  exec::kind create cluster --name nerdctl-test --config=./hack/kind.yaml
+}
+
+main "$@"
diff --git a/hack/kind.yaml b/hack/kind.yaml
@@ -0,0 +1,12 @@
+# https://pkg.go.dev/sigs.k8s.io/kind/pkg/apis/config/v1alpha4#Cluster
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: _output/nerdctl
+        containerPath: /usr/local/bin/nerdctl
+      - hostPath: /tmp/go
+        containerPath: /usr/local/go
+      - hostPath: .
+        containerPath: /nerdctl-source