Implement hot reloading of configuration file for changed list of targets

README.md

@@ -49,6 +49,10 @@ $ # use Cloudflare's public DNS server
 $ ./ping_exporter --dns.nameserver=1.1.1.1:53 [other options]
 ```
 
+The configuration file is watched via inotify. If the configuration is changed,
+ping_exporter will update the targets. To change any global options like the ping
+interval or history size, you must restart the exporter.
+
 ### Exported metrics
 
 - `ping_rtt_best_seconds`:          Best round trip time in seconds

go.mod

@@ -46,4 +46,5 @@ require (
 	golang.org/x/tools v0.4.1-0.20221208213631-3f74d914ae6d // indirect
 	golang.zx2c4.com/wireguard/windows v0.5.3 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 )

go.sum

@@ -562,6 +562,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
+	inotify "gopkg.in/fsnotify.v1"
 )
 
 const version string = "1.1.0"
@@ -45,7 +46,7 @@ var (
 	disableIPv6             = kingpin.Flag("options.disable-ipv6", "Disable DNS from resolving IPv6 AAAA records").Default().Bool()
 	disableIPv4             = kingpin.Flag("options.disable-ipv4", "Disable DNS from resolving IPv4 A records").Default().Bool()
 	logLevel                = kingpin.Flag("log.level", "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]").Default("info").String()
-	targets                 = kingpin.Arg("targets", "A list of targets to ping").Strings()
+	targetFlag              = kingpin.Arg("targets", "A list of targets to ping").Strings()
 
 	tailnet = kingpin.Flag("ts.tailnet", "tailnet name").String()
 )
@@ -56,9 +57,11 @@ var (
 
 	rttMetricsScale = rttInMills // might change in future
 	rttMode         = kingpin.Flag("metrics.rttunit", "Export ping results as either seconds (default), or milliseconds (deprecated), or both (for migrations). Valid choices: [s, ms, both]").Default("s").String()
+	desiredTargets  *targets
 )
 
 func main() {
+	desiredTargets = &targets{}
 	kingpin.Parse()
 
 	if len(*tailnet) > 0 {
@@ -156,43 +159,99 @@ func startMonitor(cfg *config.Config) (*mon.Monitor, error) {
 		cfg.Ping.Timeout.Duration())
 	monitor.HistorySize = cfg.Ping.History
 
-	targets := make([]*target, len(cfg.Targets))
+	err = upsertTargets(desiredTargets, resolver, cfg, monitor)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	go startDNSAutoRefresh(cfg.DNS.Refresh.Duration(), desiredTargets, monitor, cfg)
+	go watchConfig(desiredTargets, resolver, monitor)
+	return monitor, nil
+}
+
+func upsertTargets(globalTargets *targets, resolver *net.Resolver, cfg *config.Config, monitor *mon.Monitor) error {
+	oldTargets := globalTargets.Targets()
+	newTargets := make([]*target, len(cfg.Targets))
 	for i, t := range cfg.Targets {
 		t := &target{
 			host:      t.Addr,
 			addresses: make([]net.IPAddr, 0),
 			delay:     time.Duration(10*i) * time.Millisecond,
 			resolver:  resolver,
 		}
-		targets[i] = t
+		newTargets[i] = t
 
 		err := t.addOrUpdateMonitor(monitor, targetOpts{
 			disableIPv4: cfg.Options.DisableIPv4,
 			disableIPv6: cfg.Options.DisableIPv6,
 		})
 		if err != nil {
-			log.Errorln(err)
+			return fmt.Errorf("failed to setup target: %w", err)
 		}
 	}
 
-	go startDNSAutoRefresh(cfg.DNS.Refresh.Duration(), targets, monitor, cfg)
+	globalTargets.SetTargets(newTargets)
 
-	return monitor, nil
+	removed := removedTargets(oldTargets, globalTargets)
+	for _, removedTarget := range removed {
+		log.Infof("remove target: %s\n", removedTarget.host)
+		removedTarget.removeFromMonitor(monitor)
+	}
+	return nil
+}
+
+func watchConfig(globalTargets *targets, resolver *net.Resolver, monitor *mon.Monitor) {
+	watcher, err := inotify.NewWatcher()
+	if err != nil {
+		log.Fatalf("unable to create file watcher: %v", err)
+	}
+
+	err = watcher.Add(*configFile)
+	if err != nil {
+		log.Fatalf("unable to watch file: %v", err)
+	}
+	for {
+		select {
+		case <-watcher.Events:
+			cfg, err := loadConfig()
+			if err != nil {
+				log.Errorf("unable to load config: %v", err)
+				continue
+			}
+			log.Infof("reloading config file %s", *configFile)
+			if err := upsertTargets(globalTargets, resolver, cfg, monitor); err != nil {
+				log.Errorf("failed to reload config: %v", err)
+				continue
+			}
+		case err := <-watcher.Errors:
+			log.Errorf("watching file failed: %v", err)
+		}
+	}
+}
+
+func removedTargets(old []*target, new *targets) []*target {
+	var ret []*target
+	for _, oldTarget := range old {
+		if !new.Contains(oldTarget) {
+			ret = append(ret, oldTarget)
+		}
+	}
+	return ret
 }
 
-func startDNSAutoRefresh(interval time.Duration, targets []*target, monitor *mon.Monitor, cfg *config.Config) {
+func startDNSAutoRefresh(interval time.Duration, tar *targets, monitor *mon.Monitor, cfg *config.Config) {
 	if interval <= 0 {
 		return
 	}
 
 	for range time.NewTicker(interval).C {
-		refreshDNS(targets, monitor, cfg)
+		refreshDNS(tar, monitor, cfg)
 	}
 }
 
-func refreshDNS(targets []*target, monitor *mon.Monitor, cfg *config.Config) {
+func refreshDNS(tar *targets, monitor *mon.Monitor, cfg *config.Config) {
 	log.Infoln("refreshing DNS")
-	for _, t := range targets {
+	for _, t := range tar.Targets() {
 		go func(ta *target) {
 			err := ta.addOrUpdateMonitor(monitor, targetOpts{
 				disableIPv4: cfg.Options.DisableIPv4,
@@ -324,8 +383,8 @@ func setupResolver(cfg *config.Config) *net.Resolver {
 // config has non-zero values.
 func addFlagToConfig(cfg *config.Config) {
 	if len(cfg.Targets) == 0 {
-		cfg.Targets = make([]config.TargetConfig, len(*targets))
-		for i, t := range *targets {
+		cfg.Targets = make([]config.TargetConfig, len(*targetFlag))
+		for i, t := range *targetFlag {
 			cfg.Targets[i] = config.TargetConfig{
 				Addr: t,
 			}

tailscale.go

@@ -19,6 +19,6 @@ func tsDiscover() {
 	}
 
 	for _, dev := range devices {
-		*targets = append(*targets, dev.Hostname)
+		*targetFlag = append(*targetFlag, dev.Hostname)
 	}
 }

target.go

@@ -25,6 +25,35 @@ type target struct {
 	mutex     sync.Mutex
 }
 
+type targets struct {
+	t     []*target
+	mutex sync.RWMutex
+}
+
+func (t *targets) SetTargets(tar []*target) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.t = tar
+}
+
+func (t *targets) Contains(tar *target) bool {
+	for _, ta := range t.t {
+		if ta.host == tar.host {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *targets) Targets() []*target {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	ret := make([]*target, len(t.t))
+	copy(ret, t.t)
+	return ret
+}
+
 type targetOpts struct {
 	disableIPv4 bool
 	disableIPv6 bool
@@ -35,6 +64,12 @@ const (
 	ipv6 ipVersion = 6
 )
 
+func (t *target) removeFromMonitor(monitor *mon.Monitor) {
+	for _, addr := range t.addresses {
+		monitor.RemoveTarget(t.nameForIP(addr))
+	}
+}
+
 func (t *target) addOrUpdateMonitor(monitor *mon.Monitor, opts targetOpts) error {
 	t.mutex.Lock()
 	defer t.mutex.Unlock()