From f4a27f615c8daf02f5b002b8e5d43c4af6bf80f8 Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Thu, 8 Apr 2021 11:26:37 -0700 Subject: [PATCH 1/3] Add support for sysctl. --- containerd/containerd.go | 23 ++++----------- containerd/driver.go | 37 +++++++++++++----------- containerd/utils.go | 62 ++++++++++++++++++++++++++++++++++++++++ go.sum | 4 +++ 4 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 containerd/utils.go diff --git a/containerd/containerd.go b/containerd/containerd.go index bd3c481..4350688 100644 --- a/containerd/containerd.go +++ b/containerd/containerd.go @@ -20,8 +20,6 @@ package containerd import ( "context" "fmt" - "os" - "syscall" "time" etchosts "github.com/Roblox/nomad-driver-containerd/etchosts" @@ -118,6 +116,11 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC opts = append(opts, oci.WithPidsLimit(config.PidsLimit)) } + // Set sysctls + if len(config.Sysctl) > 0 { + opts = append(opts, WithSysctls(config.Sysctl)) + } + if !config.Seccomp && config.SeccompProfile != "" { return nil, fmt.Errorf("seccomp must be set to true, if using a custom seccomp_profile.") } @@ -265,16 +268,6 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC ) } -// buildMountpoint builds the mount point for the container. -func buildMountpoint(mountType, mountTarget, mountSource string, mountOptions []string) specs.Mount { - m := specs.Mount{} - m.Type = mountType - m.Destination = mountTarget - m.Source = mountSource - m.Options = mountOptions - return m -} - func (d *Driver) loadContainer(id string) (containerd.Container, error) { ctxWithTimeout, cancel := context.WithTimeout(d.ctxContainerd, 30*time.Second) defer cancel() @@ -299,12 +292,6 @@ func (d *Driver) createTask(container containerd.Container, stdoutPath, stderrPa return container.NewTask(ctxWithTimeout, cio.NewCreator(cio.WithStreams(nil, stdout, stderr))) } -// FIFO's are named pipes in linux. -// openFIFO() opens the nomad task stdout/stderr pipes and returns the fd. -func openFIFO(path string) (*os.File, error) { - return os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0600) -} - func (d *Driver) getTask(container containerd.Container) (containerd.Task, error) { ctxWithTimeout, cancel := context.WithTimeout(d.ctxContainerd, 30*time.Second) defer cancel() diff --git a/containerd/driver.go b/containerd/driver.go index 3e8db37..6258c52 100644 --- a/containerd/driver.go +++ b/containerd/driver.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/drivers/shared/eventer" "github.com/hashicorp/nomad/drivers/shared/resolvconf" + "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/shared/hclspec" @@ -107,6 +108,7 @@ var ( "entrypoint": hclspec.NewAttr("entrypoint", "list(string)", false), "seccomp": hclspec.NewAttr("seccomp", "bool", false), "seccomp_profile": hclspec.NewAttr("seccomp_profile", "string", false), + "sysctl": hclspec.NewAttr("sysctl", "list(map(string))", false), "readonly_rootfs": hclspec.NewAttr("readonly_rootfs", "bool", false), "host_network": hclspec.NewAttr("host_network", "bool", false), "mounts": hclspec.NewBlockList("mounts", hclspec.NewObject(map[string]*hclspec.Spec{ @@ -151,23 +153,24 @@ type Mount struct { // TaskConfig contains configuration information for a task that runs with // this plugin type TaskConfig struct { - Image string `codec:"image"` - Command string `codec:"command"` - Args []string `codec:"args"` - CapAdd []string `codec:"cap_add"` - CapDrop []string `codec:"cap_drop"` - Cwd string `codec:"cwd"` - Devices []string `codec:"devices"` - Seccomp bool `codec:"seccomp"` - SeccompProfile string `codec:"seccomp_profile"` - Privileged bool `codec:"privileged"` - PidsLimit int64 `codec:"pids_limit"` - HostDNS bool `codec:"host_dns"` - ExtraHosts []string `codec:"extra_hosts"` - Entrypoint []string `codec:"entrypoint"` - ReadOnlyRootfs bool `codec:"readonly_rootfs"` - HostNetwork bool `codec:"host_network"` - Mounts []Mount `codec:"mounts"` + Image string `codec:"image"` + Command string `codec:"command"` + Args []string `codec:"args"` + CapAdd []string `codec:"cap_add"` + CapDrop []string `codec:"cap_drop"` + Cwd string `codec:"cwd"` + Devices []string `codec:"devices"` + Seccomp bool `codec:"seccomp"` + SeccompProfile string `codec:"seccomp_profile"` + Sysctl hclutils.MapStrStr `codec:"sysctl"` + Privileged bool `codec:"privileged"` + PidsLimit int64 `codec:"pids_limit"` + HostDNS bool `codec:"host_dns"` + ExtraHosts []string `codec:"extra_hosts"` + Entrypoint []string `codec:"entrypoint"` + ReadOnlyRootfs bool `codec:"readonly_rootfs"` + HostNetwork bool `codec:"host_network"` + Mounts []Mount `codec:"mounts"` } // TaskState is the runtime state which is encoded in the handle returned to diff --git a/containerd/utils.go b/containerd/utils.go new file mode 100644 index 0000000..ef9c4e9 --- /dev/null +++ b/containerd/utils.go @@ -0,0 +1,62 @@ +/* +Copyright 2020 Roblox Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package containerd + +import ( + "context" + "os" + "syscall" + + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/oci" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// buildMountpoint builds the mount point for the container. +func buildMountpoint(mountType, mountTarget, mountSource string, mountOptions []string) specs.Mount { + m := specs.Mount{} + m.Type = mountType + m.Destination = mountTarget + m.Source = mountSource + m.Options = mountOptions + return m +} + +// FIFO's are named pipes in linux. +// openFIFO() opens the nomad task stdout/stderr pipes and returns the fd. +func openFIFO(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0600) +} + +// WithSysctls sets the provided sysctls onto the spec +// Original code referenced from: +// https://github.com/containerd/containerd/blob/master/pkg/cri/opts/spec_linux.go#L546-L560 +func WithSysctls(sysctls map[string]string) oci.SpecOpts { + return func(ctx context.Context, client oci.Client, c *containers.Container, s *specs.Spec) error { + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + if s.Linux.Sysctl == nil { + s.Linux.Sysctl = make(map[string]string) + } + for k, v := range sysctls { + s.Linux.Sysctl[k] = v + } + return nil + } +} diff --git a/go.sum b/go.sum index c33d2dd..67861ab 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,7 @@ github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3 h1:ZSTrOEhi github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= +github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0= github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/appc/spec v0.8.11 h1:BFwMCTHSDwanDlAA3ONbsLllTw4pCW85kVm290dNrV4= github.com/appc/spec v0.8.11/go.mod h1:2F+EK25qCkHIzwA7HQjWIK7r2LOL1gQlou8mm2Fdif0= @@ -518,6 +519,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee h1:8B4HqvMUtYSjsGkYjiQGStc9pXffY2J+Z2SPQAj+wMY= github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee/go.mod h1:gwlu9+/P9MmKtYrMsHeFRZPXj2CTPm11TDnMeaRHS7g= +github.com/hashicorp/hcl/v2 v2.7.1-0.20201020204811-68a97f93bb48 h1:iaau0VStfX9CgOlpbceawI94uVEM3sliqnjpHSVQqUo= github.com/hashicorp/hcl/v2 v2.7.1-0.20201020204811-68a97f93bb48/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY= github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80 h1:PFfGModn55JA0oBsvFghhj0v93me+Ctr3uHC/UmFAls= github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80/go.mod h1:Cxv+IJLuBiEhQ7pBYGEuORa0nr4U994pE8mYLuFd7v0= @@ -874,6 +876,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1194,6 +1197,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= From 83579cb186ba0ee08c04cbd06aa7cbae502280c2 Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Thu, 8 Apr 2021 11:55:23 -0700 Subject: [PATCH 2/3] Update README.md. --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 486660b..bb2f794 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ More detailed instructions are in the [`example README.md`](https://github.com/R | **host_dns** | bool | no | Default (`true`). By default, a container launched using `containerd-driver` will use host `/etc/resolv.conf`. This is similar to [`docker behavior`](https://docs.docker.com/config/containers/container-networking/#dns-services). However, if you don't want to use host DNS, you can turn off this flag by setting `host_dns=false`. | | **seccomp** | bool | no | Enable default seccomp profile. List of [`allowed syscalls`](https://github.com/containerd/containerd/blob/master/contrib/seccomp/seccomp_default.go#L51-L395). | | **seccomp_profile** | string | no | Path to custom seccomp profile. `seccomp` must be set to `true` in order to use `seccomp_profile`. The default `docker` seccomp profile found [`here`](https://github.com/moby/moby/blob/master/profiles/seccomp/default.json) can be used as a reference, and modified to create a custom seccomp profile. | +| **sysctl** | map[string]string | no | A key-value map of sysctl configurations to set to the containers on start. | | **readonly_rootfs** | bool | no | Container root filesystem will be read-only. | | **host_network** | bool | no | Enable host network. This is equivalent to `--net=host` in docker. | | **extra_hosts** | []string | no | A list of hosts, given as host:IP, to be added to /etc/hosts. | @@ -148,6 +149,17 @@ config { } ``` +**Sysctl example** + +``` +config { + sysctl = { + "net.core.somaxconn" = "16384" + "net.ipv4.ip_forward" = "1" + } +} +``` + ## Networking `nomad-driver-containerd` supports **host** and **bridge** networks.
From 9c8c3cf9eba0d076b520582d54c1a9d59197b81c Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Thu, 8 Apr 2021 15:13:24 -0700 Subject: [PATCH 3/3] Add integration test. --- example/dns.nomad | 9 ++++----- tests/006-test-dns.sh | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/example/dns.nomad b/example/dns.nomad index e531726..8e73a34 100644 --- a/example/dns.nomad +++ b/example/dns.nomad @@ -1,9 +1,6 @@ - job "dns" { datacenters = ["dc1"] - group "dns-group" { - network { dns { servers = ["127.0.0.1", "127.0.0.2"] @@ -11,15 +8,17 @@ job "dns" { options = ["ndots:2"] } } - task "dns-task" { driver = "containerd-driver" config { image = "ubuntu:16.04" command = "sleep" args = ["600s"] + sysctl = { + "net.core.somaxconn" = "16384" + "net.ipv4.ip_forward" = "1" + } } - resources { cpu = 500 memory = 256 diff --git a/tests/006-test-dns.sh b/tests/006-test-dns.sh index 2169f80..adcd235 100755 --- a/tests/006-test-dns.sh +++ b/tests/006-test-dns.sh @@ -47,6 +47,20 @@ test_dns_nomad_job() { return 1 fi + echo "INFO: Checking sysctl net.core.somaxconn=16384" + output=$(nomad alloc exec -job ${job_name} cat /proc/sys/net/core/somaxconn) + if [ "$output" != "16384" ];then + echo "ERROR: Job ${job_name}: sysctl net.core.somaxconn=16384 not found." + return 1 + fi + + echo "INFO: Checking sysctl net.ipv4.ip_forward=1" + output=$(nomad alloc exec -job ${job_name} cat /proc/sys/net/ipv4/ip_forward) + if [ "$output" != "1" ];then + echo "ERROR: Job ${job_name}: sysctl net.ipv4.ip_forward=1 not found." + return 1 + fi + echo "INFO: Stopping nomad ${job_name} job." nomad job stop ${job_name} job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')