Handle config changes on docker updates

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.10-alpine AS builder
+FROM golang:1.15.6-alpine AS builder
 
 RUN apk --no-cache add gcc musl-dev
 
@@ -7,11 +7,7 @@ COPY . ${GOPATH}/src/github.com/mcuadros/ofelia
 
 RUN go build -o /go/bin/ofelia .
 
-FROM alpine:3.11
-
-# this label is required to identify container with ofelia running
-LABEL ofelia.service=true
-LABEL ofelia.enabled=true
+FROM alpine:3.12
 
 RUN apk --no-cache add ca-certificates tzdata
 

README.md

@@ -65,8 +65,6 @@ In order to use this type of configurations, ofelia need access to docker socket
 ```sh
 docker run -it --rm \
     -v /var/run/docker.sock:/var/run/docker.sock:ro \
-    --label ofelia.job-local.my-test-job.schedule="@every 5s" \
-    --label ofelia.job-local.my-test-job.command="date" \
         mcuadros/ofelia:latest daemon --docker
 ```
 
@@ -86,9 +84,8 @@ docker run -it --rm \
         nginx
 ```
 
-Now if we start `ofelia` container with the command provided above, it will pickup 2 jobs:
+Now if we start `ofelia` container with the command provided above, it will execute the task:
 
-- Local - `date`
 - Exec  - `uname -a`
 
 Or with docker-compose:
@@ -103,9 +100,6 @@ services:
     command: daemon --docker
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
-    labels:
-      ofelia.job-local.my-test-job.schedule: "@every 5s"
-      ofelia.job-local.my-test-job.command: "date"
 
   nginx:
     image: nginx
@@ -115,6 +109,46 @@ services:
       ofelia.job-exec.datecron.command: "uname -a"
 ```
 
+#### Dynamic docker configuration
+
+You can start ofelia in its own container or on the host itself, and it will magically pick up any container that starts, stops or is modified on the fly.
+In order to achieve this, you simply have to use docker containers with the labels described above and let ofelia take care of the rest. 
+
+#### Hybrid configuration (INI files + Docker)
+
+You can specify part of the configuration on the INI files, such as globals for the middlewares or even declare tasks in there but also merge them with docker.
+The docker labels will be parsed, added and removed on the fly but also, the file config can be used to execute tasks that are not possible using just docker labels 
+such as:
+
+- job-local
+- job-run
+
+**Use the INI file to:**
+
+- Configure the slack or other middleware integration
+- Configure any global setting
+- Create a job-run so it executes on a new container each time
+
+```ini
+[global]
+slack-webhook = https://myhook.com/auth
+
+[job-run "job-executed-on-new-container"]
+schedule = @hourly
+image = ubuntu:latest
+command = touch /tmp/example
+```
+
+**Use docker to:**
+
+```sh
+docker run -it --rm \
+    --label ofelia.enabled=true \
+    --label ofelia.job-exec.test-exec-job.schedule="@every 5s" \
+    --label ofelia.job-exec.test-exec-job.command="uname -a" \
+        nginx
+```
+
 ### Logging
 **Ofelia** comes with three different logging drivers that can be configured in the `[global]` section:
 - `mail` to send mails

cli/config.go

@@ -1,145 +1,107 @@
 package cli
 
 import (
-	"os"
-
-	docker "github.com/fsouza/go-dockerclient"
 	"github.com/mcuadros/ofelia/core"
 	"github.com/mcuadros/ofelia/middlewares"
-	logging "github.com/op/go-logging"
 
 	defaults "github.com/mcuadros/go-defaults"
 	gcfg "gopkg.in/gcfg.v1"
 )
 
 const (
-	logFormat     = "%{color}%{shortfile} ▶ %{level}%{color:reset} %{message}"
 	jobExec       = "job-exec"
 	jobRun        = "job-run"
 	jobServiceRun = "job-service-run"
 	jobLocal      = "job-local"
 )
 
-var IsDockerEnv bool
-
 // Config contains the configuration
 type Config struct {
 	Global struct {
 		middlewares.SlackConfig `mapstructure:",squash"`
 		middlewares.SaveConfig  `mapstructure:",squash"`
 		middlewares.MailConfig  `mapstructure:",squash"`
 	}
-	ExecJobs    map[string]*ExecJobConfig    `gcfg:"job-exec" mapstructure:"job-exec,squash"`
-	RunJobs     map[string]*RunJobConfig     `gcfg:"job-run" mapstructure:"job-run,squash"`
-	ServiceJobs map[string]*RunServiceConfig `gcfg:"job-service-run" mapstructure:"job-service-run,squash"`
-	LocalJobs   map[string]*LocalJobConfig   `gcfg:"job-local" mapstructure:"job-local,squash"`
+	ExecJobs      map[string]*ExecJobConfig    `gcfg:"job-exec" mapstructure:"job-exec,squash"`
+	RunJobs       map[string]*RunJobConfig     `gcfg:"job-run" mapstructure:"job-run,squash"`
+	ServiceJobs   map[string]*RunServiceConfig `gcfg:"job-service-run" mapstructure:"job-service-run,squash"`
+	LocalJobs     map[string]*LocalJobConfig   `gcfg:"job-local" mapstructure:"job-local,squash"`
+	sh            *core.Scheduler
+	dockerHandler *DockerHandler
+	logger        core.Logger
 }
 
-// BuildFromDockerLabels builds a scheduler using the config from a docker labels
-func BuildFromDockerLabels() (*core.Scheduler, error) {
+func NewConfig(logger core.Logger) *Config {
+	// Initialize
 	c := &Config{}
-
-	d, err := c.buildDockerClient()
-	if err != nil {
-		return nil, err
-	}
-
-	labels, err := getLabels(d)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := c.buildFromDockerLabels(labels); err != nil {
-		return nil, err
-	}
-
-	return c.build()
+	c.ExecJobs = make(map[string]*ExecJobConfig)
+	c.RunJobs = make(map[string]*RunJobConfig)
+	c.ServiceJobs = make(map[string]*RunServiceConfig)
+	c.LocalJobs = make(map[string]*LocalJobConfig)
+	c.logger = logger
+	defaults.SetDefaults(c)
+	return c
 }
 
 // BuildFromFile builds a scheduler using the config from a file
-func BuildFromFile(filename string) (*core.Scheduler, error) {
-	c := &Config{}
-	if err := gcfg.ReadFileInto(c, filename); err != nil {
-		return nil, err
-	}
-
-	return c.build()
+func BuildFromFile(filename string, logger core.Logger) (*Config, error) {
+	c := NewConfig(logger)
+	err := gcfg.ReadFileInto(c, filename)
+	return c, err
 }
 
 // BuildFromString builds a scheduler using the config from a string
-func BuildFromString(config string) (*core.Scheduler, error) {
-	c := &Config{}
+func BuildFromString(config string, logger core.Logger) (*Config, error) {
+	c := NewConfig(logger)
 	if err := gcfg.ReadStringInto(c, config); err != nil {
 		return nil, err
 	}
-
-	return c.build()
+	return c, nil
 }
 
-func (c *Config) build() (*core.Scheduler, error) {
-	defaults.SetDefaults(c)
+// Call this only once at app init
+func (c *Config) InitializeApp() error {
+	c.sh = core.NewScheduler(c.logger)
+	c.buildSchedulerMiddlewares(c.sh)
 
-	d, err := c.buildDockerClient()
+	var err error
+	c.dockerHandler, err = NewDockerHandler(c, c.logger)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	sh := core.NewScheduler(c.buildLogger())
-	c.buildSchedulerMiddlewares(sh)
-
 	for name, j := range c.ExecJobs {
 		defaults.SetDefaults(j)
-
-		j.Client = d
+		j.Client = c.dockerHandler.GetInternalDockerClient()
 		j.Name = name
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
 	for name, j := range c.RunJobs {
 		defaults.SetDefaults(j)
-
-		j.Client = d
+		j.Client = c.dockerHandler.GetInternalDockerClient()
 		j.Name = name
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
 	for name, j := range c.LocalJobs {
 		defaults.SetDefaults(j)
-
 		j.Name = name
 		j.buildMiddlewares()
-		sh.AddJob(j)
+		c.sh.AddJob(j)
 	}
 
 	for name, j := range c.ServiceJobs {
 		defaults.SetDefaults(j)
 		j.Name = name
-		j.Client = d
+		j.Client = c.dockerHandler.GetInternalDockerClient()
 		j.buildMiddlewares()
-		sh.AddJob(j)
-	}
-
-	return sh, nil
-}
-
-func (c *Config) buildDockerClient() (*docker.Client, error) {
-	d, err := docker.NewClientFromEnv()
-	if err != nil {
-		return nil, err
+		c.sh.AddJob(j)
 	}
 
-	return d, nil
-}
-
-func (c *Config) buildLogger() core.Logger {
-	stdout := logging.NewLogBackend(os.Stdout, "", 0)
-	// Set the backends to be used.
-	logging.SetBackend(stdout)
-	logging.SetFormatter(logging.MustStringFormatter(logFormat))
-
-	return logging.MustGetLogger("ofelia")
+	return nil
 }
 
 func (c *Config) buildSchedulerMiddlewares(sh *core.Scheduler) {
@@ -148,6 +110,64 @@ func (c *Config) buildSchedulerMiddlewares(sh *core.Scheduler) {
 	sh.Use(middlewares.NewMail(&c.Global.MailConfig))
 }
 
+func (c *Config) dockerLabelsUpdate(labels map[string]map[string]string) {
+	// Get the current labels
+	var parsedLabelConfig Config
+	parsedLabelConfig.buildFromDockerLabels(labels)
+
+	// Calculate the delta
+	for name, j := range c.ExecJobs {
+		found := false
+		for newJobsName, newJob := range parsedLabelConfig.ExecJobs {
+			// Check if the schedule has changed
+			if name == newJobsName {
+				found = true
+				// There is a slight race condition were a job can be canceled / restarted with different params
+				// so, lets take care of it by simply restarting
+				// For the hash to work properly, we must fill the fields before calling it
+				defaults.SetDefaults(newJob)
+				newJob.Client = c.dockerHandler.GetInternalDockerClient()
+				newJob.Name = newJobsName
+				if newJob.Hash() != j.Hash() {
+					// Remove from the scheduler
+					c.sh.RemoveJob(j)
+					// Add the job back to the scheduler
+					newJob.buildMiddlewares()
+					c.sh.AddJob(newJob)
+					// Update the job config
+					c.ExecJobs[name] = newJob
+				}
+				break
+			}
+		}
+		if !found {
+			// Remove the job
+			c.sh.RemoveJob(j)
+			delete(c.ExecJobs, name)
+		}
+	}
+
+	// Check for aditions
+	for newJobsName, newJob := range parsedLabelConfig.ExecJobs {
+		found := false
+		for name := range c.ExecJobs {
+			if name == newJobsName {
+				found = true
+				break
+			}
+		}
+		if !found {
+			defaults.SetDefaults(newJob)
+			newJob.Client = c.dockerHandler.GetInternalDockerClient()
+			newJob.Name = newJobsName
+			newJob.buildMiddlewares()
+			c.sh.AddJob(newJob)
+			c.ExecJobs[newJobsName] = newJob
+		}
+	}
+
+}
+
 // ExecJobConfig contains all configuration params needed to build a ExecJob
 type ExecJobConfig struct {
 	core.ExecJob              `mapstructure:",squash"`

cli/config_test.go

@@ -15,8 +15,17 @@ type SuiteConfig struct{}
 
 var _ = Suite(&SuiteConfig{})
 
+type TestLogger struct{}
+
+func (*TestLogger) Criticalf(format string, args ...interface{}) {}
+func (*TestLogger) Debugf(format string, args ...interface{})    {}
+func (*TestLogger) Errorf(format string, args ...interface{})    {}
+func (*TestLogger) Noticef(format string, args ...interface{})   {}
+func (*TestLogger) Warningf(format string, args ...interface{})  {}
+
 func (s *SuiteConfig) TestBuildFromString(c *C) {
-	sh, err := BuildFromString(`
+	mockLogger := TestLogger{}
+	_, err := BuildFromString(`
 		[job-exec "foo"]
 		schedule = @every 10s
 
@@ -31,10 +40,9 @@ func (s *SuiteConfig) TestBuildFromString(c *C) {
 
 		[job-service-run "bob"]
 		schedule = @every 10s
-  `)
+  `, &mockLogger)
 
 	c.Assert(err, IsNil)
-	c.Assert(sh.Jobs, HasLen, 5)
 }
 
 func (s *SuiteConfig) TestJobDefaultsSet(c *C) {