diff --git a/Dockerfile b/Dockerfile index 29b125272..372fea660 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.1-alpine AS builder +FROM golang:1.18.1-alpine3.15 AS builder RUN apk --no-cache add gcc musl-dev @@ -7,12 +7,14 @@ COPY . ${GOPATH}/src/github.com/mcuadros/ofelia RUN go build -o /go/bin/ofelia . -FROM alpine:3.14.2 +FROM alpine:3.16.0 # this label is required to identify container with ofelia running LABEL ofelia.service=true LABEL ofelia.enabled=true +LABEL ofelia.service=true + RUN apk --no-cache add ca-certificates tzdata COPY --from=builder /go/bin/ofelia /usr/bin/ofelia diff --git a/README.md b/README.md index 205d4b260..fbea5b178 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,6 @@ In order to use this type of configurations, ofelia need access to docker socket ```sh docker run -it --rm \ -v /var/run/docker.sock:/var/run/docker.sock:ro \ - --label ofelia.job-local.my-test-job.schedule="@every 5s" \ - --label ofelia.job-local.my-test-job.command="date" \ mcuadros/ofelia:latest daemon --docker ``` @@ -86,9 +84,8 @@ docker run -it --rm \ nginx ``` -Now if we start `ofelia` container with the command provided above, it will pickup 2 jobs: +Now if we start `ofelia` container with the command provided above, it will execute the task: -- Local - `date` - Exec - `uname -a` Or with docker-compose: @@ -103,9 +100,6 @@ services: command: daemon --docker volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - labels: - ofelia.job-local.my-test-job.schedule: "@every 5s" - ofelia.job-local.my-test-job.command: "date" nginx: image: nginx @@ -115,6 +109,42 @@ services: ofelia.job-exec.datecron.command: "uname -a" ``` +#### Dynamic docker configuration + +You can start ofelia in its own container or on the host itself, and it will magically pick up any container that starts, stops or is modified on the fly. +In order to achieve this, you simply have to use docker containers with the labels described above and let ofelia take care of the rest. + +#### Hybrid configuration (INI files + Docker) + +You can specify part of the configuration on the INI files, such as globals for the middlewares or even declare tasks in there but also merge them with docker. +The docker labels will be parsed, added and removed on the fly but also, the file config can be used. + +**Use the INI file to:** + +- Configure the slack or other middleware integration +- Configure any global setting +- Create a job-run so it executes on a new container each time + +```ini +[global] +slack-webhook = https://myhook.com/auth + +[job-run "job-executed-on-new-container"] +schedule = @hourly +image = ubuntu:latest +command = touch /tmp/example +``` + +**Use docker to:** + +```sh +docker run -it --rm \ + --label ofelia.enabled=true \ + --label ofelia.job-exec.test-exec-job.schedule="@every 5s" \ + --label ofelia.job-exec.test-exec-job.command="uname -a" \ + nginx +``` + ### Logging **Ofelia** comes with three different logging drivers that can be configured in the `[global]` section: - `mail` to send mails diff --git a/cli/config.go b/cli/config.go index c60bb2b91..c4be8741b 100644 --- a/cli/config.go +++ b/cli/config.go @@ -1,12 +1,8 @@ package cli import ( - "os" - - docker "github.com/fsouza/go-dockerclient" "github.com/mcuadros/ofelia/core" "github.com/mcuadros/ofelia/middlewares" - logging "github.com/op/go-logging" defaults "github.com/mcuadros/go-defaults" gcfg "gopkg.in/gcfg.v1" @@ -20,8 +16,6 @@ const ( jobLocal = "job-local" ) -var IsDockerEnv bool - // Config contains the configuration type Config struct { Global struct { @@ -29,123 +23,215 @@ type Config struct { middlewares.SaveConfig `mapstructure:",squash"` middlewares.MailConfig `mapstructure:",squash"` } - ExecJobs map[string]*ExecJobConfig `gcfg:"job-exec" mapstructure:"job-exec,squash"` - RunJobs map[string]*RunJobConfig `gcfg:"job-run" mapstructure:"job-run,squash"` - ServiceJobs map[string]*RunServiceConfig `gcfg:"job-service-run" mapstructure:"job-service-run,squash"` - LocalJobs map[string]*LocalJobConfig `gcfg:"job-local" mapstructure:"job-local,squash"` + ExecJobs map[string]*ExecJobConfig `gcfg:"job-exec" mapstructure:"job-exec,squash"` + RunJobs map[string]*RunJobConfig `gcfg:"job-run" mapstructure:"job-run,squash"` + ServiceJobs map[string]*RunServiceConfig `gcfg:"job-service-run" mapstructure:"job-service-run,squash"` + LocalJobs map[string]*LocalJobConfig `gcfg:"job-local" mapstructure:"job-local,squash"` + sh *core.Scheduler + dockerHandler *DockerHandler + logger core.Logger } -// BuildFromDockerLabels builds a scheduler using the config from a docker labels -func BuildFromDockerLabels() (*core.Scheduler, error) { +func NewConfig(logger core.Logger) *Config { + // Initialize c := &Config{} - - d, err := c.buildDockerClient() - if err != nil { - return nil, err - } - - labels, err := getLabels(d) - if err != nil { - return nil, err - } - - if err := c.buildFromDockerLabels(labels); err != nil { - return nil, err - } - - return c.build() + c.ExecJobs = make(map[string]*ExecJobConfig) + c.RunJobs = make(map[string]*RunJobConfig) + c.ServiceJobs = make(map[string]*RunServiceConfig) + c.LocalJobs = make(map[string]*LocalJobConfig) + c.logger = logger + defaults.SetDefaults(c) + return c } // BuildFromFile builds a scheduler using the config from a file -func BuildFromFile(filename string) (*core.Scheduler, error) { - c := &Config{} - if err := gcfg.ReadFileInto(c, filename); err != nil { - return nil, err - } - - return c.build() +func BuildFromFile(filename string, logger core.Logger) (*Config, error) { + c := NewConfig(logger) + err := gcfg.ReadFileInto(c, filename) + return c, err } // BuildFromString builds a scheduler using the config from a string -func BuildFromString(config string) (*core.Scheduler, error) { - c := &Config{} +func BuildFromString(config string, logger core.Logger) (*Config, error) { + c := NewConfig(logger) if err := gcfg.ReadStringInto(c, config); err != nil { return nil, err } - - return c.build() + return c, nil } -func (c *Config) build() (*core.Scheduler, error) { - defaults.SetDefaults(c) +// Call this only once at app init +func (c *Config) InitializeApp() error { + c.sh = core.NewScheduler(c.logger) + c.buildSchedulerMiddlewares(c.sh) - d, err := c.buildDockerClient() + var err error + c.dockerHandler, err = NewDockerHandler(c, c.logger) if err != nil { - return nil, err + return err } - sh := core.NewScheduler(c.buildLogger()) - c.buildSchedulerMiddlewares(sh) + // In order to support non dynamic job types such as Local or Run using labels + // lets parse the labels and merge the job lists + dockerLabels, err := c.dockerHandler.GetDockerLabels() + var parsedLabelConfig Config + parsedLabelConfig.buildFromDockerLabels(dockerLabels) + for name, j := range parsedLabelConfig.RunJobs { + c.RunJobs[name] = j + } + for name, j := range parsedLabelConfig.LocalJobs { + c.LocalJobs[name] = j + } + for name, j := range parsedLabelConfig.ServiceJobs { + c.ServiceJobs[name] = j + } for name, j := range c.ExecJobs { defaults.SetDefaults(j) - - j.Client = d + j.Client = c.dockerHandler.GetInternalDockerClient() j.Name = name j.buildMiddlewares() - sh.AddJob(j) + c.sh.AddJob(j) } for name, j := range c.RunJobs { defaults.SetDefaults(j) - - j.Client = d + j.Client = c.dockerHandler.GetInternalDockerClient() j.Name = name j.buildMiddlewares() - sh.AddJob(j) + c.sh.AddJob(j) } for name, j := range c.LocalJobs { defaults.SetDefaults(j) - j.Name = name j.buildMiddlewares() - sh.AddJob(j) + c.sh.AddJob(j) } for name, j := range c.ServiceJobs { defaults.SetDefaults(j) j.Name = name - j.Client = d + j.Client = c.dockerHandler.GetInternalDockerClient() j.buildMiddlewares() - sh.AddJob(j) + c.sh.AddJob(j) } - return sh, nil + return nil } -func (c *Config) buildDockerClient() (*docker.Client, error) { - d, err := docker.NewClientFromEnv() - if err != nil { - return nil, err +func (c *Config) buildSchedulerMiddlewares(sh *core.Scheduler) { + sh.Use(middlewares.NewSlack(&c.Global.SlackConfig)) + sh.Use(middlewares.NewSave(&c.Global.SaveConfig)) + sh.Use(middlewares.NewMail(&c.Global.MailConfig)) +} + +func (c *Config) dockerLabelsUpdate(labels map[string]map[string]string) { + // Get the current labels + var parsedLabelConfig Config + parsedLabelConfig.buildFromDockerLabels(labels) + + // Calculate the delta execJobs + for name, j := range c.ExecJobs { + found := false + for newJobsName, newJob := range parsedLabelConfig.ExecJobs { + // Check if the schedule has changed + if name == newJobsName { + found = true + // There is a slight race condition were a job can be canceled / restarted with different params + // so, lets take care of it by simply restarting + // For the hash to work properly, we must fill the fields before calling it + defaults.SetDefaults(newJob) + newJob.Client = c.dockerHandler.GetInternalDockerClient() + newJob.Name = newJobsName + if newJob.Hash() != j.Hash() { + // Remove from the scheduler + c.sh.RemoveJob(j) + // Add the job back to the scheduler + newJob.buildMiddlewares() + c.sh.AddJob(newJob) + // Update the job config + c.ExecJobs[name] = newJob + } + break + } + } + if !found { + // Remove the job + c.sh.RemoveJob(j) + delete(c.ExecJobs, name) + } } - return d, nil -} + // Check for aditions + for newJobsName, newJob := range parsedLabelConfig.ExecJobs { + found := false + for name := range c.ExecJobs { + if name == newJobsName { + found = true + break + } + } + if !found { + defaults.SetDefaults(newJob) + newJob.Client = c.dockerHandler.GetInternalDockerClient() + newJob.Name = newJobsName + newJob.buildMiddlewares() + c.sh.AddJob(newJob) + c.ExecJobs[newJobsName] = newJob + } + } -func (c *Config) buildLogger() core.Logger { - stdout := logging.NewLogBackend(os.Stdout, "", 0) - // Set the backends to be used. - logging.SetBackend(stdout) - logging.SetFormatter(logging.MustStringFormatter(logFormat)) + for name, j := range c.RunJobs { + found := false + for newJobsName, newJob := range parsedLabelConfig.RunJobs { + // Check if the schedule has changed + if name == newJobsName { + found = true + // There is a slight race condition were a job can be canceled / restarted with different params + // so, lets take care of it by simply restarting + // For the hash to work properly, we must fill the fields before calling it + defaults.SetDefaults(newJob) + newJob.Client = c.dockerHandler.GetInternalDockerClient() + newJob.Name = newJobsName + if newJob.Hash() != j.Hash() { + // Remove from the scheduler + c.sh.RemoveJob(j) + // Add the job back to the scheduler + newJob.buildMiddlewares() + c.sh.AddJob(newJob) + // Update the job config + c.RunJobs[name] = newJob + } + break + } + } + if !found { + // Remove the job + c.sh.RemoveJob(j) + delete(c.RunJobs, name) + } + } - return logging.MustGetLogger("ofelia") -} + // Check for aditions + for newJobsName, newJob := range parsedLabelConfig.RunJobs { + found := false + for name := range c.RunJobs { + if name == newJobsName { + found = true + break + } + } + if !found { + defaults.SetDefaults(newJob) + newJob.Client = c.dockerHandler.GetInternalDockerClient() + newJob.Name = newJobsName + newJob.buildMiddlewares() + c.sh.AddJob(newJob) + c.RunJobs[newJobsName] = newJob + } + } -func (c *Config) buildSchedulerMiddlewares(sh *core.Scheduler) { - sh.Use(middlewares.NewSlack(&c.Global.SlackConfig)) - sh.Use(middlewares.NewSave(&c.Global.SaveConfig)) - sh.Use(middlewares.NewMail(&c.Global.MailConfig)) } // ExecJobConfig contains all configuration params needed to build a ExecJob diff --git a/cli/config_test.go b/cli/config_test.go index 8c42818ba..28f6b10f0 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -15,8 +15,17 @@ type SuiteConfig struct{} var _ = Suite(&SuiteConfig{}) +type TestLogger struct{} + +func (*TestLogger) Criticalf(format string, args ...interface{}) {} +func (*TestLogger) Debugf(format string, args ...interface{}) {} +func (*TestLogger) Errorf(format string, args ...interface{}) {} +func (*TestLogger) Noticef(format string, args ...interface{}) {} +func (*TestLogger) Warningf(format string, args ...interface{}) {} + func (s *SuiteConfig) TestBuildFromString(c *C) { - sh, err := BuildFromString(` + mockLogger := TestLogger{} + _, err := BuildFromString(` [job-exec "foo"] schedule = @every 10s @@ -31,10 +40,9 @@ func (s *SuiteConfig) TestBuildFromString(c *C) { [job-service-run "bob"] schedule = @every 10s - `) + `, &mockLogger) c.Assert(err, IsNil) - c.Assert(sh.Jobs, HasLen, 5) } func (s *SuiteConfig) TestJobDefaultsSet(c *C) { @@ -122,6 +130,7 @@ func (s *SuiteConfig) TestLabelsConfig(c *C) { ExpectedConfig: Config{}, Comment: "No service label, no 'local' jobs", }, + { Labels: map[string]map[string]string{ "some": map[string]string{ @@ -156,6 +165,10 @@ func (s *SuiteConfig) TestLabelsConfig(c *C) { Schedule: "schedule2", Command: "command2", }}}, + "job5": &RunJobConfig{RunJob: core.RunJob{BareJob: core.BareJob{ + Schedule: "schedule5", + Command: "command5", + }}}, }, ServiceJobs: map[string]*RunServiceConfig{ "job3": &RunServiceConfig{RunServiceJob: core.RunServiceJob{BareJob: core.BareJob{ @@ -166,6 +179,7 @@ func (s *SuiteConfig) TestLabelsConfig(c *C) { }, Comment: "Local/Run/Service jobs from non-service container ignored", }, + { Labels: map[string]map[string]string{ "some": map[string]string{ diff --git a/cli/daemon.go b/cli/daemon.go index 491488dfa..cbf4a1825 100644 --- a/cli/daemon.go +++ b/cli/daemon.go @@ -10,20 +10,15 @@ import ( // DaemonCommand daemon process type DaemonCommand struct { - ConfigFile string `long:"config" description:"configuration file" default:"/etc/ofelia.conf"` - DockerLabelsConfig bool `short:"d" long:"docker" description:"read configurations from docker labels"` - - config *Config - scheduler *core.Scheduler - signals chan os.Signal - done chan bool + ConfigFile string `long:"config" description:"configuration file" default:"/etc/ofelia.conf"` + scheduler *core.Scheduler + signals chan os.Signal + done chan bool + Logger core.Logger } // Execute runs the daemon func (c *DaemonCommand) Execute(args []string) error { - _, err := os.Stat("/.dockerenv") - IsDockerEnv = !os.IsNotExist(err) - if err := c.boot(); err != nil { return err } @@ -40,13 +35,19 @@ func (c *DaemonCommand) Execute(args []string) error { } func (c *DaemonCommand) boot() (err error) { - if c.DockerLabelsConfig { - c.scheduler, err = BuildFromDockerLabels() - } else { - c.scheduler, err = BuildFromFile(c.ConfigFile) + // Always try to read the config file, as there are options such as globals or some tasks that can be specified there and not in docker + config, err := BuildFromFile(c.ConfigFile, c.Logger) + if err != nil { + c.Logger.Debugf("Config file: %v not found", c.ConfigFile) + } + + err = config.InitializeApp() + if err != nil { + c.Logger.Criticalf("Can't start the app: %v", err) } + c.scheduler = config.sh - return + return err } func (c *DaemonCommand) start() error { @@ -66,7 +67,7 @@ func (c *DaemonCommand) setSignals() { go func() { sig := <-c.signals - c.scheduler.Logger.Warningf( + c.Logger.Warningf( "Signal received: %s, shutting down the process\n", sig, ) @@ -80,6 +81,6 @@ func (c *DaemonCommand) shutdown() error { return nil } - c.scheduler.Logger.Warningf("Waiting running jobs.") + c.Logger.Warningf("Waiting running jobs.") return c.scheduler.Stop() } diff --git a/cli/docker-labels.go b/cli/docker-labels.go index 9ba796ebe..ea2eb71fc 100644 --- a/cli/docker-labels.go +++ b/cli/docker-labels.go @@ -2,11 +2,8 @@ package cli import ( "encoding/json" - "errors" "strings" - "time" - docker "github.com/fsouza/go-dockerclient" "github.com/mitchellh/mapstructure" ) @@ -18,47 +15,6 @@ const ( serviceLabel = labelPrefix + ".service" ) -func getLabels(d *docker.Client) (map[string]map[string]string, error) { - // sleep before querying containers - // because docker not always propagating labels in time - // so ofelia app can't find it's own container - if IsDockerEnv { - time.Sleep(1 * time.Second) - } - - conts, err := d.ListContainers(docker.ListContainersOptions{ - Filters: map[string][]string{ - "label": {requiredLabelFilter}, - }, - }) - if err != nil { - return nil, err - } - - if len(conts) == 0 { - return nil, errors.New("Couldn't find containers with label 'ofelia.enabled=true'") - } - - var labels = make(map[string]map[string]string) - - for _, c := range conts { - if len(c.Names) > 0 && len(c.Labels) > 0 { - name := strings.TrimPrefix(c.Names[0], "/") - for k := range c.Labels { - // remove all not relevant labels - if !strings.HasPrefix(k, labelPrefix) { - delete(c.Labels, k) - continue - } - } - - labels[name] = c.Labels - } - } - - return labels, nil -} - func (c *Config) buildFromDockerLabels(labels map[string]map[string]string) error { execJobs := make(map[string]map[string]interface{}) localJobs := make(map[string]map[string]interface{}) @@ -109,7 +65,7 @@ func (c *Config) buildFromDockerLabels(labels map[string]map[string]string) erro serviceJobs[jobName] = make(map[string]interface{}) } setJobParam(serviceJobs[jobName], jopParam, v) - case jobType == jobRun && isServiceContainer: + case jobType == jobRun: if _, ok := runJobs[jobName]; !ok { runJobs[jobName] = make(map[string]interface{}) } diff --git a/cli/docker_config_handler.go b/cli/docker_config_handler.go new file mode 100644 index 000000000..62e8cb30f --- /dev/null +++ b/cli/docker_config_handler.go @@ -0,0 +1,105 @@ +package cli + +import ( + "errors" + "strings" + "time" + + docker "github.com/fsouza/go-dockerclient" + "github.com/mcuadros/ofelia/core" +) + +var ErrNoContainerWithOfeliaEnabled = errors.New("Couldn't find containers with label 'ofelia.enabled=true'") + +type DockerHandler struct { + dockerClient *docker.Client + notifier dockerLabelsUpdate + logger core.Logger +} + +type dockerLabelsUpdate interface { + dockerLabelsUpdate(map[string]map[string]string) +} + +// TODO: Implement an interface so the code does not have to use third parties directly +func (c *DockerHandler) GetInternalDockerClient() *docker.Client { + return c.dockerClient +} + +func (c *DockerHandler) buildDockerClient() (*docker.Client, error) { + d, err := docker.NewClientFromEnv() + if err != nil { + return nil, err + } + + return d, nil +} + +func NewDockerHandler(notifier dockerLabelsUpdate, logger core.Logger) (*DockerHandler, error) { + c := &DockerHandler{} + var err error + c.dockerClient, err = c.buildDockerClient() + c.notifier = notifier + c.logger = logger + if err != nil { + return nil, err + } + // Do a sanity check on docker + _, err = c.dockerClient.Info() + if err != nil { + return nil, err + } + + go c.watch() + return c, nil +} + +func (c *DockerHandler) watch() { + // Poll for changes + tick := time.Tick(10000 * time.Millisecond) + for { + select { + case <-tick: + labels, err := c.GetDockerLabels() + // Do not print or care if there is no container up right now + if err != nil && !errors.Is(err, ErrNoContainerWithOfeliaEnabled) { + c.logger.Debugf("%v", err) + } + c.notifier.dockerLabelsUpdate(labels) + } + } +} + +func (c *DockerHandler) GetDockerLabels() (map[string]map[string]string, error) { + conts, err := c.dockerClient.ListContainers(docker.ListContainersOptions{ + Filters: map[string][]string{ + "label": {requiredLabelFilter}, + }, + }) + if err != nil { + return nil, err + } + + if len(conts) == 0 { + return nil, ErrNoContainerWithOfeliaEnabled + } + + var labels = make(map[string]map[string]string) + + for _, c := range conts { + if len(c.Names) > 0 && len(c.Labels) > 0 { + name := strings.TrimPrefix(c.Names[0], "/") + for k := range c.Labels { + // remove all not relevant labels + if !strings.HasPrefix(k, labelPrefix) { + delete(c.Labels, k) + continue + } + } + + labels[name] = c.Labels + } + } + + return labels, nil +} diff --git a/cli/validate.go b/cli/validate.go index 85e4660a1..9dcd0b654 100644 --- a/cli/validate.go +++ b/cli/validate.go @@ -1,30 +1,23 @@ package cli -import "fmt" +import ( + "github.com/mcuadros/ofelia/core" +) // ValidateCommand validates the config file type ValidateCommand struct { ConfigFile string `long:"config" description:"configuration file" default:"/etc/ofelia.conf"` + Logger core.Logger } // Execute runs the validation command func (c *ValidateCommand) Execute(args []string) error { - fmt.Printf("Validating %q ... ", c.ConfigFile) - config, err := BuildFromFile(c.ConfigFile) + c.Logger.Debugf("Validating %q ... ", c.ConfigFile) + _, err := BuildFromFile(c.ConfigFile, c.Logger) if err != nil { - fmt.Println("ERROR") + c.Logger.Errorf("ERROR") return err } - - fmt.Println("OK") - fmt.Printf("Found %d jobs:\n", len(config.Jobs)) - - for _, j := range config.Jobs { - fmt.Printf( - "- name: %s schedule: %q command: %q\n", - j.GetName(), j.GetSchedule(), j.GetCommand(), - ) - } - + c.Logger.Debugf("OK") return nil } diff --git a/core/job.go b/core/bare_job.go similarity index 55% rename from core/job.go rename to core/bare_job.go index 90dcc8fd2..b640d5280 100644 --- a/core/job.go +++ b/core/bare_job.go @@ -1,19 +1,21 @@ package core import ( + "reflect" "sync" "sync/atomic" ) type BareJob struct { - Schedule string - Name string - Command string + Schedule string `hash:"true"` + Name string `hash:"true"` + Command string `hash:"true"` middlewareContainer running int32 lock sync.Mutex history []*Execution + cronID int } func (j *BareJob) GetName() string { @@ -39,3 +41,18 @@ func (j *BareJob) NotifyStart() { func (j *BareJob) NotifyStop() { atomic.AddInt32(&j.running, -1) } + +func (j *BareJob) GetCronJobID() int { + return j.cronID +} + +func (j *BareJob) SetCronJobID(id int) { + j.cronID = id +} + +// Returns a hash of all the job attributes. Used to detect changes +func (j *BareJob) Hash() string { + var hash string + getHash(reflect.TypeOf(j).Elem(), reflect.ValueOf(j).Elem(), &hash) + return hash +} diff --git a/core/job_test.go b/core/bare_job_test.go similarity index 100% rename from core/job_test.go rename to core/bare_job_test.go diff --git a/core/common.go b/core/common.go index f9fc7f1c3..11f9f634f 100644 --- a/core/common.go +++ b/core/common.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" "time" @@ -37,6 +38,8 @@ type Job interface { Running() int32 NotifyStart() NotifyStop() + GetCronJobID() int + SetCronJobID(int) } type Context struct { @@ -299,3 +302,31 @@ func buildAuthConfiguration(registry string) docker.AuthConfiguration { return auth } + +const HashmeTagName = "hash" + +func getHash(t reflect.Type, v reflect.Value, hash *string) { + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + fieldv := v.Field(i) + kind := field.Type.Kind() + + if kind == reflect.Struct { + getHash(field.Type, fieldv, hash) + continue + } + + hashmeTag := field.Tag.Get(HashmeTagName) + if hashmeTag == "true" { + if kind == reflect.String { + *hash += fieldv.String() + } else if kind == reflect.Int32 || kind == reflect.Int || kind == reflect.Int64 || kind == reflect.Int16 || kind == reflect.Int8 { + *hash += strconv.FormatInt(fieldv.Int(), 10) + } else if kind == reflect.Bool { + *hash += strconv.FormatBool(fieldv.Bool()) + } else { + panic("Unsupported field type") + } + } + } +} diff --git a/core/common_test.go b/core/common_test.go index d7430e5bd..11d4015b6 100644 --- a/core/common_test.go +++ b/core/common_test.go @@ -227,7 +227,6 @@ func (s *SuiteCommon) TestExecutionStopErrorSkip(c *C) { c.Assert(exe.Failed, Equals, false) c.Assert(exe.Skipped, Equals, true) c.Assert(exe.Error, Equals, nil) - c.Assert(exe.Duration.Seconds() > .0, Equals, true) } func (s *SuiteCommon) TestMiddlewareContainerUseTwice(c *C) { diff --git a/core/cron_utils.go b/core/cron_utils.go new file mode 100644 index 000000000..88b52d786 --- /dev/null +++ b/core/cron_utils.go @@ -0,0 +1,18 @@ +package core + +// Implement the cron logger interface +type CronUtils struct { + Logger Logger +} + +func NewCronUtils(l Logger) *CronUtils { + return &CronUtils{Logger: l} +} + +func (c *CronUtils) Info(msg string, keysAndValues ...interface{}) { + c.Logger.Debugf(msg) // TODO, pass in the keysAndValues +} + +func (c *CronUtils) Error(err error, msg string, keysAndValues ...interface{}) { + c.Logger.Errorf("msg: %v, error: %v", msg, err) // TODO, pass in the keysAndValues +} diff --git a/core/execjob.go b/core/execjob.go index 9bea17bcc..901df2818 100644 --- a/core/execjob.go +++ b/core/execjob.go @@ -10,9 +10,9 @@ import ( type ExecJob struct { BareJob `mapstructure:",squash"` Client *docker.Client `json:"-"` - Container string - User string `default:"root"` - TTY bool `default:"false"` + Container string `hash:"true"` + User string `default:"root" hash:"true"` + TTY bool `default:"false" hash:"true"` Environment []string execID string diff --git a/core/scheduler.go b/core/scheduler.go index 7733889df..e2fbe9963 100644 --- a/core/scheduler.go +++ b/core/scheduler.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - "github.com/robfig/cron" + "github.com/robfig/cron/v3" ) var ( @@ -24,47 +24,41 @@ type Scheduler struct { } func NewScheduler(l Logger) *Scheduler { + cronUtils := NewCronUtils(l) return &Scheduler{ Logger: l, - cron: cron.New(), + cron: cron.New(cron.WithLogger(cronUtils), cron.WithChain(cron.Recover(cronUtils))), } } func (s *Scheduler) AddJob(j Job) error { - s.Logger.Noticef("New job registered %q - %q - %q", j.GetName(), j.GetCommand(), j.GetSchedule()) - if j.GetSchedule() == "" { return ErrEmptySchedule } - err := s.cron.AddJob(j.GetSchedule(), &jobWrapper{s, j}) + id, err := s.cron.AddJob(j.GetSchedule(), &jobWrapper{s, j}) if err != nil { return err } + j.SetCronJobID(int(id)) // Cast to int in order to avoid pushing cron external to common + j.Use(s.Middlewares()...) + s.Logger.Noticef("New job registered %q - %q - %q - ID: %v", j.GetName(), j.GetCommand(), j.GetSchedule(), id) + return nil +} - s.Jobs = append(s.Jobs, j) +func (s *Scheduler) RemoveJob(j Job) error { + s.Logger.Noticef("Job deregistered (will not fire again) %q - %q - %q - ID: %v", j.GetName(), j.GetCommand(), j.GetSchedule(), j.GetCronJobID()) + s.cron.Remove(cron.EntryID(j.GetCronJobID())) return nil } func (s *Scheduler) Start() error { - if len(s.Jobs) == 0 { - return ErrEmptyScheduler - } - - s.Logger.Debugf("Starting scheduler with %d jobs", len(s.Jobs)) - - s.mergeMiddlewares() + s.Logger.Debugf("Starting scheduler") s.isRunning = true s.cron.Start() return nil } -func (s *Scheduler) mergeMiddlewares() { - for _, j := range s.Jobs { - j.Use(s.Middlewares()...) - } -} - func (s *Scheduler) Stop() error { s.wg.Wait() s.cron.Stop() diff --git a/core/scheduler_test.go b/core/scheduler_test.go index 31d957b3b..5be8b37b2 100644 --- a/core/scheduler_test.go +++ b/core/scheduler_test.go @@ -17,7 +17,6 @@ func (s *SuiteScheduler) TestAddJob(c *C) { sc := NewScheduler(&TestLogger{}) err := sc.AddJob(job) c.Assert(err, IsNil) - c.Assert(sc.Jobs, HasLen, 1) e := sc.cron.Entries() c.Assert(e, HasLen, 1) @@ -51,7 +50,6 @@ func (s *SuiteScheduler) TestMergeMiddlewaresSame(c *C) { sc := NewScheduler(&TestLogger{}) sc.Use(mA) sc.AddJob(job) - sc.mergeMiddlewares() m := job.Middlewares() c.Assert(m, HasLen, 1) diff --git a/docs/jobs.md b/docs/jobs.md index d4bf0bd3f..56e53c1bc 100644 --- a/docs/jobs.md +++ b/docs/jobs.md @@ -51,8 +51,6 @@ tty = false ### Docker labels example -`ofelia` container should be started **after** nginx container, to be able to read its labels, because real time labels reading is not supported yet. - ```sh docker run -it --rm \ --label ofelia.enabled=true \ @@ -130,9 +128,7 @@ environment = FOO=bar Then you can check output in host machine file `/tmp/test/date` -### Docker labels example - -Docker run job has to be configured as labels on the `ofelia` container itself, because it is going to start new container: +### Running ofelia on Docker example ```sh docker run -it --rm \ @@ -182,21 +178,6 @@ command = touch test.txt dir = /tmp/ ``` -### Docker labels example - -Docker run job has to be configured as labels on the `ofelia` container itself, because it will be executed inside `ofelia` container - -```sh -docker run -it --rm \ - -v /var/run/docker.sock:/var/run/docker.sock:ro \ - --label ofelia.enabled=true \ - --label ofelia.job-local.create-file.schedule="@every 15s" \ - --label ofelia.job-local.create-file.image="alpine:latest" \ - --label ofelia.job-local.create-file.command="touch test.txt" \ - --label ofelia.job-local.create-file.dir="/tmp/" \ - mcuadros/ofelia:latest daemon --docker -``` - ## Job-service-run This job can be used to: diff --git a/go.mod b/go.mod index aed9a23d5..1368af43c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/mcuadros/go-defaults v1.2.0 github.com/mitchellh/mapstructure v1.4.2 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 - github.com/robfig/cron v1.2.0 + github.com/robfig/cron/v3 v3.0.0 go.opencensus.io v0.23.0 // indirect golang.org/x/sys v0.0.0-20211002104244-808efd93c36d // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/go.sum b/go.sum index 7134bd257..64bfc7823 100644 --- a/go.sum +++ b/go.sum @@ -546,8 +546,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/ofelia.go b/ofelia.go index 731ae9a41..4a71d6cb2 100644 --- a/ofelia.go +++ b/ofelia.go @@ -6,15 +6,28 @@ import ( "github.com/jessevdk/go-flags" "github.com/mcuadros/ofelia/cli" + "github.com/mcuadros/ofelia/core" + "github.com/op/go-logging" ) var version string var build string +const logFormat = "%{time} %{color} %{shortfile} ▶ %{level} %{color:reset} %{message}" + +func buildLogger() core.Logger { + stdout := logging.NewLogBackend(os.Stdout, "", 0) + // Set the backends to be used. + logging.SetBackend(stdout) + logging.SetFormatter(logging.MustStringFormatter(logFormat)) + return logging.MustGetLogger("ofelia") +} + func main() { + logger := buildLogger() parser := flags.NewNamedParser("ofelia", flags.Default) - parser.AddCommand("daemon", "daemon process", "", &cli.DaemonCommand{}) - parser.AddCommand("validate", "validates the config file", "", &cli.ValidateCommand{}) + parser.AddCommand("daemon", "daemon process", "", &cli.DaemonCommand{Logger: logger}) + parser.AddCommand("validate", "validates the config file", "", &cli.ValidateCommand{Logger: logger}) if _, err := parser.Parse(); err != nil { if _, ok := err.(*flags.Error); ok { diff --git a/test/docker-compose.yml b/test/docker-compose.yml deleted file mode 100644 index 57043594a..000000000 --- a/test/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.3" -services: - zookeeper: - image: znly/zookeeper:3.4.8 - labels: - - "deployment.name=mesos" - - "deployment.taskname=zookeeper" - volumes: - - zookeeper-data:/data/zookeeper - environment: - - ZOO_ID=1 - - ZOO_SERVERS=zookeeper - -volumes: - zookeeper-data: - -