From 42fc67cc8052d722f2cdec8cd9d5232f45a858db Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Thu, 2 Sep 2021 12:41:12 -0700 Subject: [PATCH] Update versions, run nomad in non-dev mode and fix tests. Signed-off-by: Shishir Mahajan --- Vagrantfile | 32 +++++++++++++++--------------- example/agent.hcl | 4 ++++ example/entrypoint.nomad | 6 +++--- tests/001-test-redis.sh | 6 +++--- tests/002-test-signal-handler.sh | 6 +++--- tests/003-test-capabilities.sh | 6 +++--- tests/004-test-privileged.sh | 6 +++--- tests/005-test-volume-mount.sh | 6 +++--- tests/006-test-dns.sh | 6 +++--- tests/007-test-extra-hosts.sh | 6 +++--- tests/008-test-entrypoint.sh | 4 ++-- tests/009-test-auth.sh | 4 ++-- tests/010-test-allow-privileged.sh | 6 +++--- tests/run_tests.sh | 26 ++++++++++++------------ vagrant/setup.sh | 6 +++--- 15 files changed, 67 insertions(+), 63 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index b978c89..7425869 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,34 +28,34 @@ Vagrant.configure("2") do |config| # without keeping HOME env, 'sudo make test' will try to find files under /root/go/ echo "Defaults env_keep += HOME" | sudo tee /etc/sudoers.d/keep_home - # Install golang-1.14.3 + # Install golang-1.17 if [ ! -f "/usr/local/go/bin/go" ]; then - curl -s -L -o go1.14.3.linux-amd64.tar.gz https://dl.google.com/go/go1.14.3.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.14.3.linux-amd64.tar.gz + curl -s -L -o go1.17.linux-amd64.tar.gz https://dl.google.com/go/go1.17.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz sudo chmod +x /usr/local/go - rm -f go1.14.3.linux-amd64.tar.gz + rm -f go1.17.linux-amd64.tar.gz fi - # Install nomad-1.1.0 + # Install nomad-1.1.4 if [ ! -f "/usr/bin/nomad" ]; then - wget --quiet https://releases.hashicorp.com/nomad/1.1.0/nomad_1.1.0_linux_amd64.zip - unzip nomad_1.1.0_linux_amd64.zip -d /usr/bin + wget --quiet https://releases.hashicorp.com/nomad/1.1.4/nomad_1.1.4_linux_amd64.zip + unzip nomad_1.1.4_linux_amd64.zip -d /usr/bin chmod +x /usr/bin/nomad - rm -f nomad_1.1.0_linux_amd64.zip + rm -f nomad_1.1.4_linux_amd64.zip fi - # Install containerd-1.3.4 + # Install containerd-1.5.5 if [ ! -f "/usr/local/bin/containerd" ]; then - curl -L --silent -o containerd-1.3.4.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v1.3.4/containerd-1.3.4.linux-amd64.tar.gz - tar -C /usr/local -xzf containerd-1.3.4.linux-amd64.tar.gz - rm -f containerd-1.3.4.linux-amd64.tar.gz + curl -L --silent -o containerd-1.5.5-linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v1.5.5/containerd-1.5.5-linux-amd64.tar.gz + tar -C /usr/local -xzf containerd-1.5.5-linux-amd64.tar.gz + rm -f containerd-1.5.5-linux-amd64.tar.gz fi - # Install nerdctl 0.10.0 + # Install nerdctl 0.11.1 if [ ! -f "/usr/local/bin/nerdctl" ]; then - curl -L --silent -o nerdctl-0.10.0-linux-amd64.tar.gz https://github.com/containerd/nerdctl/releases/download/v0.10.0/nerdctl-0.10.0-linux-amd64.tar.gz - tar -C /usr/local/bin -xzf nerdctl-0.10.0-linux-amd64.tar.gz - rm -f nerdctl-0.10.0-linux-amd64.tar.gz + curl -L --silent -o nerdctl-0.11.1-linux-amd64.tar.gz https://github.com/containerd/nerdctl/releases/download/v0.11.1/nerdctl-0.11.1-linux-amd64.tar.gz + tar -C /usr/local/bin -xzf nerdctl-0.11.1-linux-amd64.tar.gz + rm -f nerdctl-0.11.1-linux-amd64.tar.gz fi # Create source directory for privileged.nomad example job. diff --git a/example/agent.hcl b/example/agent.hcl index eefd6a4..4b579b2 100644 --- a/example/agent.hcl +++ b/example/agent.hcl @@ -1,4 +1,5 @@ log_level = "INFO" +data_dir = "/tmp/nomad" plugin "containerd-driver" { config { @@ -9,6 +10,8 @@ plugin "containerd-driver" { } server { + enabled = true + bootstrap_expect = 1 default_scheduler_config { scheduler_algorithm = "spread" memory_oversubscription_enabled = true @@ -22,6 +25,7 @@ server { } client { + enabled = true host_volume "s1" { path = "/tmp/host_volume/s1" read_only = false diff --git a/example/entrypoint.nomad b/example/entrypoint.nomad index 4721969..448a540 100644 --- a/example/entrypoint.nomad +++ b/example/entrypoint.nomad @@ -6,9 +6,9 @@ job "entrypoint" { driver = "containerd-driver" config { - image = "ubuntu:16.04" - entrypoint = ["/bin/echo"] - args = ["container1", "container2"] + image = "ubuntu:16.04" + entrypoint = ["/bin/bash"] + args = ["-c", "for i in {1..100}; do echo container1 container2; sleep 1s; done"] } resources { diff --git a/tests/001-test-redis.sh b/tests/001-test-redis.sh index 2bf8829..f39d088 100755 --- a/tests/001-test-redis.sh +++ b/tests/001-test-redis.sh @@ -6,7 +6,7 @@ test_redis_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad redis job using nomad-driver-containerd." - nomad job run redis.nomad + nomad job run -detach redis.nomad redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $redis_status != "running" ];then @@ -63,7 +63,7 @@ test_redis_nomad_job() { fi echo "INFO: Stopping nomad redis job." - nomad job stop redis + nomad job stop -detach redis redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $redis_status != "dead(stopped)" ];then echo "ERROR: Error in stopping redis job." @@ -71,7 +71,7 @@ test_redis_nomad_job() { fi echo "INFO: purge nomad redis job." - nomad job stop -purge redis + nomad job stop -detach -purge redis popd } diff --git a/tests/002-test-signal-handler.sh b/tests/002-test-signal-handler.sh index eeccf7d..41eca72 100755 --- a/tests/002-test-signal-handler.sh +++ b/tests/002-test-signal-handler.sh @@ -6,7 +6,7 @@ test_signal_handler_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad signal handler job using nomad-driver-containerd." - nomad job run signal.nomad + nomad job run -detach signal.nomad echo "INFO: Checking status of signal handler job." signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') @@ -40,7 +40,7 @@ test_signal_handler_nomad_job() { cleanup "$outfile" echo "INFO: Stopping nomad signal handler job." - nomad job stop signal + nomad job stop -detach signal signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $signal_status != "dead(stopped)" ];then echo "ERROR: Error in stopping signal handler job." @@ -48,7 +48,7 @@ test_signal_handler_nomad_job() { fi echo "INFO: purge nomad signal handler job." - nomad job stop -purge signal + nomad job stop -detach -purge signal popd } diff --git a/tests/003-test-capabilities.sh b/tests/003-test-capabilities.sh index 3f853d3..e03fbcb 100755 --- a/tests/003-test-capabilities.sh +++ b/tests/003-test-capabilities.sh @@ -7,7 +7,7 @@ test_capabilities_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad capabilities job using nomad-driver-containerd." - nomad job run capabilities.nomad + nomad job run -detach capabilities.nomad echo "INFO: Checking status of capabilities job." cap_status=$(nomad job status -short capabilities|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') @@ -59,7 +59,7 @@ test_capabilities_nomad_job() { cleanup "$outfile" echo "INFO: Stopping nomad capabilities job." - nomad job stop capabilities + nomad job stop -detach capabilities cap_status=$(nomad job status -short capabilities|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $cap_status != "dead(stopped)" ];then echo "ERROR: Error in stopping capabilities job." @@ -67,7 +67,7 @@ test_capabilities_nomad_job() { fi echo "INFO: purge nomad capabilities job." - nomad job stop -purge capabilities + nomad job stop -detach -purge capabilities popd } diff --git a/tests/004-test-privileged.sh b/tests/004-test-privileged.sh index 03df39f..d79f13e 100755 --- a/tests/004-test-privileged.sh +++ b/tests/004-test-privileged.sh @@ -9,7 +9,7 @@ test_privileged_nomad_job() { setup_bind_source echo "INFO: Starting nomad privileged job using nomad-driver-containerd." - nomad job run privileged.nomad + nomad job run -detach privileged.nomad echo "INFO: Checking status of privileged job." job_status=$(nomad job status -short privileged|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') @@ -68,7 +68,7 @@ test_privileged_nomad_job() { fi echo "INFO: Stopping nomad privileged job." - nomad job stop privileged + nomad job stop -detach privileged job_status=$(nomad job status -short privileged|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $job_status != "dead(stopped)" ];then echo "ERROR: Error in stopping privileged job." @@ -76,7 +76,7 @@ test_privileged_nomad_job() { fi echo "INFO: purge nomad privileged job." - nomad job stop -purge privileged + nomad job stop -detach -purge privileged popd } diff --git a/tests/005-test-volume-mount.sh b/tests/005-test-volume-mount.sh index 2521c3f..46ca699 100755 --- a/tests/005-test-volume-mount.sh +++ b/tests/005-test-volume-mount.sh @@ -12,7 +12,7 @@ test_volume_mount_nomad_job() { setup_bind_source echo "INFO: Starting nomad $job_name job using nomad-driver-containerd." - nomad job run $job_name.nomad + nomad job run -detach $job_name.nomad # Even though $(nomad job status) reports job status as "running" # The actual container process might not be running yet. @@ -54,7 +54,7 @@ test_volume_mount_nomad_job() { fi echo "INFO: Stopping nomad ${job_name} job." - nomad job stop ${job_name} + nomad job stop -detach ${job_name} job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $job_status != "dead(stopped)" ];then echo "ERROR: Error in stopping ${job_name} job." @@ -62,7 +62,7 @@ test_volume_mount_nomad_job() { fi echo "INFO: purge nomad ${job_name} job." - nomad job stop -purge ${job_name} + nomad job stop -detach -purge ${job_name} popd } diff --git a/tests/006-test-dns.sh b/tests/006-test-dns.sh index adcd235..959b1cd 100755 --- a/tests/006-test-dns.sh +++ b/tests/006-test-dns.sh @@ -8,7 +8,7 @@ test_dns_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad $job_name job using nomad-driver-containerd." - nomad job run $job_name.nomad + nomad job run -detach $job_name.nomad # Even though $(nomad job status) reports job status as "running" # The actual container process might not be running yet. @@ -62,7 +62,7 @@ test_dns_nomad_job() { fi echo "INFO: Stopping nomad ${job_name} job." - nomad job stop ${job_name} + nomad job stop -detach ${job_name} job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $job_status != "dead(stopped)" ];then echo "ERROR: Error in stopping ${job_name} job." @@ -70,7 +70,7 @@ test_dns_nomad_job() { fi echo "INFO: purge nomad ${job_name} job." - nomad job stop -purge ${job_name} + nomad job stop -detach -purge ${job_name} popd } diff --git a/tests/007-test-extra-hosts.sh b/tests/007-test-extra-hosts.sh index fa9420e..5ced3e2 100755 --- a/tests/007-test-extra-hosts.sh +++ b/tests/007-test-extra-hosts.sh @@ -8,7 +8,7 @@ test_extra_hosts_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad $job_name job using nomad-driver-containerd." - nomad job run $job_name.nomad + nomad job run -detach $job_name.nomad # Even though $(nomad job status) reports job status as "running" # The actual container process might not be running yet. @@ -34,7 +34,7 @@ test_extra_hosts_nomad_job() { done echo "INFO: Stopping nomad ${job_name} job." - nomad job stop ${job_name} + nomad job stop -detach ${job_name} job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') if [ $job_status != "dead(stopped)" ];then echo "ERROR: Error in stopping ${job_name} job." @@ -42,7 +42,7 @@ test_extra_hosts_nomad_job() { fi echo "INFO: purge nomad ${job_name} job." - nomad job stop -purge ${job_name} + nomad job stop -detach -purge ${job_name} popd } diff --git a/tests/008-test-entrypoint.sh b/tests/008-test-entrypoint.sh index 82e0f17..b2126bf 100755 --- a/tests/008-test-entrypoint.sh +++ b/tests/008-test-entrypoint.sh @@ -8,7 +8,7 @@ test_entrypoint_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad $job_name job using nomad-driver-containerd." - nomad job run $job_name.nomad + nomad job run -detach $job_name.nomad # Even though $(nomad job status) reports job status as "running" # The actual container process might not be running yet. @@ -33,7 +33,7 @@ test_entrypoint_nomad_job() { done echo "INFO: purge nomad ${job_name} job." - nomad job stop -purge ${job_name} + nomad job stop -detach -purge ${job_name} popd } diff --git a/tests/009-test-auth.sh b/tests/009-test-auth.sh index 4696054..309c145 100755 --- a/tests/009-test-auth.sh +++ b/tests/009-test-auth.sh @@ -9,7 +9,7 @@ test_auth_nomad_job() { pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example echo "INFO: Starting nomad $job_name job using nomad-driver-containerd." - nomad job run $job_name.nomad + nomad job run -detach $job_name.nomad wait_nomad_job_status $job_name failed @@ -23,7 +23,7 @@ test_auth_nomad_job() { fi echo "INFO: purge nomad ${job_name} job." - nomad job stop -purge ${job_name} + nomad job stop -detach -purge ${job_name} popd } diff --git a/tests/010-test-allow-privileged.sh b/tests/010-test-allow-privileged.sh index 87769c1..71badee 100755 --- a/tests/010-test-allow-privileged.sh +++ b/tests/010-test-allow-privileged.sh @@ -9,12 +9,12 @@ test_allow_privileged() { cp agent.hcl agent.hcl.bkp - sed -i '8 i \ allow_privileged = false' agent.hcl + sed -i '9 i \ allow_privileged = false' agent.hcl sudo systemctl restart nomad is_systemd_service_active "nomad.service" true echo "INFO: Starting nomad ${job_name} job using nomad-driver-containerd." - nomad job run privileged_not_allowed.nomad + nomad job run -detach privileged_not_allowed.nomad # Sleep for 5 seconds, to allow ${alloc_id} to get populated. sleep 5s @@ -28,7 +28,7 @@ test_allow_privileged() { fi echo "INFO: purge nomad ${job_name} job." - nomad job stop -purge ${job_name} + nomad job stop -detach -purge ${job_name} mv agent.hcl.bkp agent.hcl popd diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 475d3e0..0544a74 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -2,8 +2,8 @@ set -eo pipefail -export NOMAD_VERSION=1.1.0 -export CONTAINERD_VERSION=1.3.4 +export NOMAD_VERSION=1.1.4 +export CONTAINERD_VERSION=1.5.5 export PATH=$PATH:/usr/local/go/bin export PATH=$PATH:/usr/local/bin if [ -e /home/runner ]; then @@ -11,7 +11,7 @@ if [ -e /home/runner ]; then else export GOPATH=$HOME/go fi -export GO_VERSION=1.14.3 +export GO_VERSION=1.17 export SRCDIR=`dirname $0` source $SRCDIR/utils.sh @@ -83,10 +83,10 @@ setup() { # Change $(pwd) to /tmp pushd /tmp - # Install containerd 1.3.4 - curl -L -o containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz - rm -f containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz + # Install containerd 1.5.5 + curl -L -o containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz + sudo tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz + rm -f containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz # Drop containerd systemd unit file into /lib/systemd/system. cat << EOF > containerd.service @@ -120,17 +120,17 @@ EOF sudo systemctl start containerd is_systemd_service_active "containerd.service" false - # Remove default golang (1.7.3) and install a custom version (1.14.3) of golang. + # Remove default golang (1.7.3) and install a custom version (1.17) of golang. # This is required for supporting go mod, and to be able to compile nomad-driver-containerd. sudo rm -rf /usr/local/go - # Install golang 1.14.3 + # Install golang 1.17 curl -L -o go${GO_VERSION}.linux-amd64.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz sudo tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz sudo chmod +x /usr/local/go rm -f go${GO_VERSION}.linux-amd64.tar.gz - # Install nomad 1.1.0 + # Install nomad 1.1.4 curl -L -o nomad_${NOMAD_VERSION}_linux_amd64.zip https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip sudo unzip -d /usr/local/bin nomad_${NOMAD_VERSION}_linux_amd64.zip sudo chmod +x /usr/local/bin/nomad @@ -143,16 +143,16 @@ EOF mkdir -p /tmp/nomad-driver-containerd mv containerd-driver /tmp/nomad-driver-containerd - # Drop nomad server (dev) + nomad-driver-containerd systemd unit file into /lib/systemd/system. + # Drop nomad server + nomad-driver-containerd systemd unit file into /lib/systemd/system. cat << EOF > nomad.service # /lib/systemd/system/nomad.service [Unit] -Description=nomad server (dev) + nomad-driver-containerd +Description=nomad server + nomad-driver-containerd Documentation=https://nomadproject.io After=network.target [Service] -ExecStart=/usr/local/bin/nomad agent -dev -config=$GOPATH/src/github.com/Roblox/nomad-driver-containerd/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd +ExecStart=/usr/local/bin/nomad agent -config=$GOPATH/src/github.com/Roblox/nomad-driver-containerd/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd KillMode=process Delegate=yes LimitNOFILE=1048576 diff --git a/vagrant/setup.sh b/vagrant/setup.sh index 4a1a43b..fcc44b9 100755 --- a/vagrant/setup.sh +++ b/vagrant/setup.sh @@ -45,16 +45,16 @@ main() { drop_nomad_unit_file() { local nomad=$(which nomad) - # Drop nomad server (dev) + nomad-driver-containerd systemd unit file into /lib/systemd/system. + # Drop nomad server + nomad-driver-containerd systemd unit file into /lib/systemd/system. cat << EOF > nomad.service # /lib/systemd/system/nomad.service [Unit] -Description=nomad server (dev) + nomad-driver-containerd +Description=nomad server + nomad-driver-containerd Documentation=https://nomadproject.io After=network.target containerd.service [Service] -ExecStart=$nomad agent -dev -bind=0.0.0.0 -config=$1/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd +ExecStart=$nomad agent -bind=0.0.0.0 -config=$1/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd KillMode=process Delegate=yes LimitNOFILE=1048576