From a3dd865f2575381afed73f74b9b7cb1bff00f043 Mon Sep 17 00:00:00 2001 From: Christoph Petrausch Date: Tue, 26 Mar 2024 10:59:29 +0100 Subject: [PATCH] Implement hot reloading of configuration file for changed list of targets --- README.md | 4 +++ go.mod | 1 + go.sum | 2 ++ main.go | 83 ++++++++++++++++++++++++++++++++++++++++++++-------- tailscale.go | 2 +- target.go | 35 ++++++++++++++++++++++ 6 files changed, 114 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 221c0b3..99a4e47 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ $ # use Cloudflare's public DNS server $ ./ping_exporter --dns.nameserver=1.1.1.1:53 [other options] ``` +The configuration file is watched via inotify. If the configuration is changed, +ping_exporter will update the targets. To change any global options like the ping +interval or history size, you must restart the exporter. + ### Exported metrics - `ping_rtt_best_seconds`: Best round trip time in seconds diff --git a/go.mod b/go.mod index 23412ac..33d87ad 100644 --- a/go.mod +++ b/go.mod @@ -46,4 +46,5 @@ require ( golang.org/x/tools v0.4.1-0.20221208213631-3f74d914ae6d // indirect golang.zx2c4.com/wireguard/windows v0.5.3 // indirect google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect ) diff --git a/go.sum b/go.sum index aac0ea1..6b9d930 100644 --- a/go.sum +++ b/go.sum @@ -562,6 +562,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 4bb98d6..39049cc 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" "gopkg.in/alecthomas/kingpin.v2" + inotify "gopkg.in/fsnotify.v1" ) const version string = "1.1.0" @@ -45,7 +46,7 @@ var ( disableIPv6 = kingpin.Flag("options.disable-ipv6", "Disable DNS from resolving IPv6 AAAA records").Default().Bool() disableIPv4 = kingpin.Flag("options.disable-ipv4", "Disable DNS from resolving IPv4 A records").Default().Bool() logLevel = kingpin.Flag("log.level", "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]").Default("info").String() - targets = kingpin.Arg("targets", "A list of targets to ping").Strings() + targetFlag = kingpin.Arg("targets", "A list of targets to ping").Strings() tailnet = kingpin.Flag("ts.tailnet", "tailnet name").String() ) @@ -56,9 +57,11 @@ var ( rttMetricsScale = rttInMills // might change in future rttMode = kingpin.Flag("metrics.rttunit", "Export ping results as either seconds (default), or milliseconds (deprecated), or both (for migrations). Valid choices: [s, ms, both]").Default("s").String() + desiredTargets *targets ) func main() { + desiredTargets = &targets{} kingpin.Parse() if len(*tailnet) > 0 { @@ -156,7 +159,19 @@ func startMonitor(cfg *config.Config) (*mon.Monitor, error) { cfg.Ping.Timeout.Duration()) monitor.HistorySize = cfg.Ping.History - targets := make([]*target, len(cfg.Targets)) + err = upsertTargets(desiredTargets, resolver, cfg, monitor) + if err != nil { + log.Fatalln(err) + } + + go startDNSAutoRefresh(cfg.DNS.Refresh.Duration(), desiredTargets, monitor, cfg) + go watchConfig(desiredTargets, resolver, monitor) + return monitor, nil +} + +func upsertTargets(globalTargets *targets, resolver *net.Resolver, cfg *config.Config, monitor *mon.Monitor) error { + oldTargets := globalTargets.Targets() + newTargets := make([]*target, len(cfg.Targets)) for i, t := range cfg.Targets { t := &target{ host: t.Addr, @@ -164,35 +179,79 @@ func startMonitor(cfg *config.Config) (*mon.Monitor, error) { delay: time.Duration(10*i) * time.Millisecond, resolver: resolver, } - targets[i] = t + newTargets[i] = t err := t.addOrUpdateMonitor(monitor, targetOpts{ disableIPv4: cfg.Options.DisableIPv4, disableIPv6: cfg.Options.DisableIPv6, }) if err != nil { - log.Errorln(err) + return fmt.Errorf("failed to setup target: %w", err) } } - go startDNSAutoRefresh(cfg.DNS.Refresh.Duration(), targets, monitor, cfg) + globalTargets.SetTargets(newTargets) - return monitor, nil + removed := removedTargets(oldTargets, globalTargets) + for _, removedTarget := range removed { + log.Infof("remove target: %s\n", removedTarget.host) + removedTarget.removeFromMonitor(monitor) + } + return nil +} + +func watchConfig(globalTargets *targets, resolver *net.Resolver, monitor *mon.Monitor) { + watcher, err := inotify.NewWatcher() + if err != nil { + log.Fatalf("unable to create file watcher: %v", err) + } + + err = watcher.Add(*configFile) + if err != nil { + log.Fatalf("unable to watch file: %v", err) + } + for { + select { + case <-watcher.Events: + cfg, err := loadConfig() + if err != nil { + log.Errorf("unable to load config: %v", err) + continue + } + log.Infof("reloading config file %s", *configFile) + if err := upsertTargets(globalTargets, resolver, cfg, monitor); err != nil { + log.Errorf("failed to reload config: %v", err) + continue + } + case err := <-watcher.Errors: + log.Errorf("watching file failed: %v", err) + } + } +} + +func removedTargets(old []*target, new *targets) []*target { + var ret []*target + for _, oldTarget := range old { + if !new.Contains(oldTarget) { + ret = append(ret, oldTarget) + } + } + return ret } -func startDNSAutoRefresh(interval time.Duration, targets []*target, monitor *mon.Monitor, cfg *config.Config) { +func startDNSAutoRefresh(interval time.Duration, tar *targets, monitor *mon.Monitor, cfg *config.Config) { if interval <= 0 { return } for range time.NewTicker(interval).C { - refreshDNS(targets, monitor, cfg) + refreshDNS(tar, monitor, cfg) } } -func refreshDNS(targets []*target, monitor *mon.Monitor, cfg *config.Config) { +func refreshDNS(tar *targets, monitor *mon.Monitor, cfg *config.Config) { log.Infoln("refreshing DNS") - for _, t := range targets { + for _, t := range tar.Targets() { go func(ta *target) { err := ta.addOrUpdateMonitor(monitor, targetOpts{ disableIPv4: cfg.Options.DisableIPv4, @@ -324,8 +383,8 @@ func setupResolver(cfg *config.Config) *net.Resolver { // config has non-zero values. func addFlagToConfig(cfg *config.Config) { if len(cfg.Targets) == 0 { - cfg.Targets = make([]config.TargetConfig, len(*targets)) - for i, t := range *targets { + cfg.Targets = make([]config.TargetConfig, len(*targetFlag)) + for i, t := range *targetFlag { cfg.Targets[i] = config.TargetConfig{ Addr: t, } diff --git a/tailscale.go b/tailscale.go index 3c31a68..da49a7a 100644 --- a/tailscale.go +++ b/tailscale.go @@ -19,6 +19,6 @@ func tsDiscover() { } for _, dev := range devices { - *targets = append(*targets, dev.Hostname) + *targetFlag = append(*targetFlag, dev.Hostname) } } diff --git a/target.go b/target.go index d6ffbcc..838a335 100644 --- a/target.go +++ b/target.go @@ -25,6 +25,35 @@ type target struct { mutex sync.Mutex } +type targets struct { + t []*target + mutex sync.RWMutex +} + +func (t *targets) SetTargets(tar []*target) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.t = tar +} + +func (t *targets) Contains(tar *target) bool { + for _, ta := range t.t { + if ta.host == tar.host { + return true + } + } + return false +} + +func (t *targets) Targets() []*target { + t.mutex.RLock() + defer t.mutex.RUnlock() + + ret := make([]*target, len(t.t)) + copy(ret, t.t) + return ret +} + type targetOpts struct { disableIPv4 bool disableIPv6 bool @@ -35,6 +64,12 @@ const ( ipv6 ipVersion = 6 ) +func (t *target) removeFromMonitor(monitor *mon.Monitor) { + for _, addr := range t.addresses { + monitor.RemoveTarget(t.nameForIP(addr)) + } +} + func (t *target) addOrUpdateMonitor(monitor *mon.Monitor, opts targetOpts) error { t.mutex.Lock() defer t.mutex.Unlock()