Merge pull request #84 from Roblox/sysctl

Add support for sysctl.
Roblox · Apr 8, 2021 · 1431042 · 1431042
2 parents d8b9127 + 9c8c3cf
commit 1431042
Show file tree

Hide file tree

Showing 7 changed files with 121 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -102,6 +102,7 @@ More detailed instructions are in the [`example README.md`](https://github.com/R
 | **host_dns** | bool | no | Default (`true`). By default, a container launched using `containerd-driver` will use host `/etc/resolv.conf`. This is similar to [`docker behavior`](https://docs.docker.com/config/containers/container-networking/#dns-services). However, if you don't want to use host DNS, you can turn off this flag by setting `host_dns=false`. |
 | **seccomp** | bool | no | Enable default seccomp profile. List of [`allowed syscalls`](https://github.com/containerd/containerd/blob/master/contrib/seccomp/seccomp_default.go#L51-L395). |
 | **seccomp_profile** | string | no | Path to custom seccomp profile. `seccomp` must be set to `true` in order to use `seccomp_profile`. The default `docker` seccomp profile found [`here`](https://github.com/moby/moby/blob/master/profiles/seccomp/default.json) can be used as a reference, and modified to create a custom seccomp profile. |
+| **sysctl** | map[string]string | no | A key-value map of sysctl configurations to set to the containers on start. |
 | **readonly_rootfs** | bool | no | Container root filesystem will be read-only. |
 | **host_network** | bool | no | Enable host network. This is equivalent to `--net=host` in docker. |
 | **extra_hosts** | []string | no | A list of hosts, given as host:IP, to be added to /etc/hosts. |
@@ -148,6 +149,17 @@ config {
 }
 ```
 
+**Sysctl example**
+
+```
+config {
+  sysctl = {
+    "net.core.somaxconn"  = "16384"
+    "net.ipv4.ip_forward" = "1"
+  }
+}
+```
+
 ## Networking
 
 `nomad-driver-containerd` supports **host** and **bridge** networks.<br/>

diff --git a/containerd/containerd.go b/containerd/containerd.go
@@ -20,8 +20,6 @@ package containerd
 import (
 	"context"
 	"fmt"
-	"os"
-	"syscall"
 	"time"
 
 	etchosts "github.com/Roblox/nomad-driver-containerd/etchosts"
@@ -118,6 +116,11 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 		opts = append(opts, oci.WithPidsLimit(config.PidsLimit))
 	}
 
+	// Set sysctls
+	if len(config.Sysctl) > 0 {
+		opts = append(opts, WithSysctls(config.Sysctl))
+	}
+
 	if !config.Seccomp && config.SeccompProfile != "" {
 		return nil, fmt.Errorf("seccomp must be set to true, if using a custom seccomp_profile.")
 	}
@@ -265,16 +268,6 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 	)
 }
 
-// buildMountpoint builds the mount point for the container.
-func buildMountpoint(mountType, mountTarget, mountSource string, mountOptions []string) specs.Mount {
-	m := specs.Mount{}
-	m.Type = mountType
-	m.Destination = mountTarget
-	m.Source = mountSource
-	m.Options = mountOptions
-	return m
-}
-
 func (d *Driver) loadContainer(id string) (containerd.Container, error) {
 	ctxWithTimeout, cancel := context.WithTimeout(d.ctxContainerd, 30*time.Second)
 	defer cancel()
@@ -299,12 +292,6 @@ func (d *Driver) createTask(container containerd.Container, stdoutPath, stderrPa
 	return container.NewTask(ctxWithTimeout, cio.NewCreator(cio.WithStreams(nil, stdout, stderr)))
 }
 
-// FIFO's are named pipes in linux.
-// openFIFO() opens the nomad task stdout/stderr pipes and returns the fd.
-func openFIFO(path string) (*os.File, error) {
-	return os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0600)
-}
-
 func (d *Driver) getTask(container containerd.Container) (containerd.Task, error) {
 	ctxWithTimeout, cancel := context.WithTimeout(d.ctxContainerd, 30*time.Second)
 	defer cancel()

diff --git a/containerd/driver.go b/containerd/driver.go
@@ -32,6 +32,7 @@ import (
 	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/drivers/shared/eventer"
 	"github.com/hashicorp/nomad/drivers/shared/resolvconf"
+	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/drivers"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
@@ -107,6 +108,7 @@ var (
 		"entrypoint":      hclspec.NewAttr("entrypoint", "list(string)", false),
 		"seccomp":         hclspec.NewAttr("seccomp", "bool", false),
 		"seccomp_profile": hclspec.NewAttr("seccomp_profile", "string", false),
+		"sysctl":          hclspec.NewAttr("sysctl", "list(map(string))", false),
 		"readonly_rootfs": hclspec.NewAttr("readonly_rootfs", "bool", false),
 		"host_network":    hclspec.NewAttr("host_network", "bool", false),
 		"mounts": hclspec.NewBlockList("mounts", hclspec.NewObject(map[string]*hclspec.Spec{
@@ -151,23 +153,24 @@ type Mount struct {
 // TaskConfig contains configuration information for a task that runs with
 // this plugin
 type TaskConfig struct {
-	Image          string   `codec:"image"`
-	Command        string   `codec:"command"`
-	Args           []string `codec:"args"`
-	CapAdd         []string `codec:"cap_add"`
-	CapDrop        []string `codec:"cap_drop"`
-	Cwd            string   `codec:"cwd"`
-	Devices        []string `codec:"devices"`
-	Seccomp        bool     `codec:"seccomp"`
-	SeccompProfile string   `codec:"seccomp_profile"`
-	Privileged     bool     `codec:"privileged"`
-	PidsLimit      int64    `codec:"pids_limit"`
-	HostDNS        bool     `codec:"host_dns"`
-	ExtraHosts     []string `codec:"extra_hosts"`
-	Entrypoint     []string `codec:"entrypoint"`
-	ReadOnlyRootfs bool     `codec:"readonly_rootfs"`
-	HostNetwork    bool     `codec:"host_network"`
-	Mounts         []Mount  `codec:"mounts"`
+	Image          string             `codec:"image"`
+	Command        string             `codec:"command"`
+	Args           []string           `codec:"args"`
+	CapAdd         []string           `codec:"cap_add"`
+	CapDrop        []string           `codec:"cap_drop"`
+	Cwd            string             `codec:"cwd"`
+	Devices        []string           `codec:"devices"`
+	Seccomp        bool               `codec:"seccomp"`
+	SeccompProfile string             `codec:"seccomp_profile"`
+	Sysctl         hclutils.MapStrStr `codec:"sysctl"`
+	Privileged     bool               `codec:"privileged"`
+	PidsLimit      int64              `codec:"pids_limit"`
+	HostDNS        bool               `codec:"host_dns"`
+	ExtraHosts     []string           `codec:"extra_hosts"`
+	Entrypoint     []string           `codec:"entrypoint"`
+	ReadOnlyRootfs bool               `codec:"readonly_rootfs"`
+	HostNetwork    bool               `codec:"host_network"`
+	Mounts         []Mount            `codec:"mounts"`
 }
 
 // TaskState is the runtime state which is encoded in the handle returned to

diff --git a/containerd/utils.go b/containerd/utils.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2020 Roblox Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package containerd
+
+import (
+	"context"
+	"os"
+	"syscall"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/oci"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// buildMountpoint builds the mount point for the container.
+func buildMountpoint(mountType, mountTarget, mountSource string, mountOptions []string) specs.Mount {
+	m := specs.Mount{}
+	m.Type = mountType
+	m.Destination = mountTarget
+	m.Source = mountSource
+	m.Options = mountOptions
+	return m
+}
+
+// FIFO's are named pipes in linux.
+// openFIFO() opens the nomad task stdout/stderr pipes and returns the fd.
+func openFIFO(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0600)
+}
+
+// WithSysctls sets the provided sysctls onto the spec
+// Original code referenced from:
+// https://github.com/containerd/containerd/blob/master/pkg/cri/opts/spec_linux.go#L546-L560
+func WithSysctls(sysctls map[string]string) oci.SpecOpts {
+	return func(ctx context.Context, client oci.Client, c *containers.Container, s *specs.Spec) error {
+		if s.Linux == nil {
+			s.Linux = &specs.Linux{}
+		}
+		if s.Linux.Sysctl == nil {
+			s.Linux.Sysctl = make(map[string]string)
+		}
+		for k, v := range sysctls {
+			s.Linux.Sysctl[k] = v
+		}
+		return nil
+	}
+}
diff --git a/example/dns.nomad b/example/dns.nomad
@@ -1,25 +1,24 @@
-
 job "dns" {
   datacenters = ["dc1"]
-
   group "dns-group" {
-
     network {
         dns {
             servers = ["127.0.0.1", "127.0.0.2"]
             searches = ["internal.corp"]
             options = ["ndots:2"]
         }
     }
-
     task "dns-task" {
       driver = "containerd-driver"
       config {
         image           = "ubuntu:16.04"
         command         = "sleep"
         args            = ["600s"]
+        sysctl = {
+           "net.core.somaxconn"  = "16384"
+           "net.ipv4.ip_forward" = "1"
+        }
       }
-
       resources {
         cpu    = 500
         memory = 256

diff --git a/go.sum b/go.sum
@@ -85,6 +85,7 @@ github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3 h1:ZSTrOEhi
 github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
 github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0=
 github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec=
 github.com/appc/spec v0.8.11 h1:BFwMCTHSDwanDlAA3ONbsLllTw4pCW85kVm290dNrV4=
 github.com/appc/spec v0.8.11/go.mod h1:2F+EK25qCkHIzwA7HQjWIK7r2LOL1gQlou8mm2Fdif0=
@@ -518,6 +519,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee h1:8B4HqvMUtYSjsGkYjiQGStc9pXffY2J+Z2SPQAj+wMY=
 github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee/go.mod h1:gwlu9+/P9MmKtYrMsHeFRZPXj2CTPm11TDnMeaRHS7g=
+github.com/hashicorp/hcl/v2 v2.7.1-0.20201020204811-68a97f93bb48 h1:iaau0VStfX9CgOlpbceawI94uVEM3sliqnjpHSVQqUo=
 github.com/hashicorp/hcl/v2 v2.7.1-0.20201020204811-68a97f93bb48/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
 github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80 h1:PFfGModn55JA0oBsvFghhj0v93me+Ctr3uHC/UmFAls=
 github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80/go.mod h1:Cxv+IJLuBiEhQ7pBYGEuORa0nr4U994pE8mYLuFd7v0=
@@ -874,6 +876,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -1194,6 +1197,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

diff --git a/tests/006-test-dns.sh b/tests/006-test-dns.sh
@@ -47,6 +47,20 @@ test_dns_nomad_job() {
        return 1
     fi
 
+    echo "INFO: Checking sysctl net.core.somaxconn=16384"
+    output=$(nomad alloc exec -job ${job_name} cat /proc/sys/net/core/somaxconn)
+    if [ "$output" != "16384" ];then
+        echo "ERROR: Job ${job_name}: sysctl net.core.somaxconn=16384 not found."
+        return 1
+    fi
+
+    echo "INFO: Checking sysctl net.ipv4.ip_forward=1"
+    output=$(nomad alloc exec -job ${job_name} cat /proc/sys/net/ipv4/ip_forward)
+    if [ "$output" != "1" ];then
+        echo "ERROR: Job ${job_name}: sysctl net.ipv4.ip_forward=1 not found."
+        return 1
+    fi
+
     echo "INFO: Stopping nomad ${job_name} job."
     nomad job stop ${job_name}
     job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')