[master]feat(pod):add regex support for pod name (#102)

* chore(*):resorted import package order

* feat(pod):add regex support for pod name

cmd/log-pilot/main.go

@@ -5,13 +5,13 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/caicloud/log-pilot/pilot/configurer"
 	"github.com/caicloud/log-pilot/pilot/configurer/filebeat"
 	"github.com/caicloud/log-pilot/pilot/discovery"
 	"github.com/caicloud/log-pilot/pilot/log"
-	"strings"
 )
 
 var (
@@ -23,6 +23,8 @@ var (
 	logLevel          = flag.String("logLevel", "info", "Log level: debug, info, warning, error, critical")
 	wListNS           = flag.String("namespace.whitelist", "", "whitelist of namespaces to watch")
 	bListNS           = flag.String("namespace.blacklist", "", "blacklist of namespaces to ignore")
+	wListPodRegex     = flag.String("pod.whiteRegex", "", "regex with what to match the name of the pod to watch")
+	bListPodRegex     = flag.String("pod.blackRegex", "", "regex with what to match the name of the pod to ignore")
 	logMaxBytes       = flag.Uint("log.maxSize", 10*1024*1024, "Max size of log file in bytes")
 	logMaxBackups     = flag.Uint("log.maxBackups", 7, "Max backups of log files")
 	logToStderr       = flag.Bool("e", false, "Log to stderr")
@@ -46,7 +48,7 @@ func main() {
 		log.Fatalf("Error create configurer: %v", err)
 	}
 
-	d, err := discovery.New(baseDir, *logPrefix, cfgr, parseList(*bListNS), parseList(*wListNS))
+	d, err := discovery.New(baseDir, *logPrefix, cfgr, parseList(*bListNS), parseList(*wListNS), *bListPodRegex, *wListPodRegex)
 	if err != nil {
 		log.Fatalf("Error create discovery: %v", err)
 	}

pilot/configurer/filebeat/filebeat.go

@@ -7,18 +7,18 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
 	"text/template"
 	"time"
-	"regexp"
 
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/go-ucfg"
 
-	"github.com/caicloud/log-pilot/pilot/container"
 	"github.com/caicloud/log-pilot/pilot/configurer"
+	"github.com/caicloud/log-pilot/pilot/container"
 	"github.com/caicloud/log-pilot/pilot/log"
 )
 

pilot/configurer/filebeat/filebeat_test.go

@@ -2,7 +2,7 @@ package filebeat
 
 import (
 	"testing"
-			)
+)
 
 var (
 	expectRenderResult = `

pilot/discovery/blackwhitelist.go

@@ -0,0 +1,58 @@
+package discovery
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+type BlackWhiteList struct {
+	bListNS  string
+	wListNS  string
+	bListPod string
+	wListPod string
+}
+
+func (b *BlackWhiteList) IsResponsible(namespace string, pod string) (bool, error) {
+	bListPodRegex, err := regexp.Compile(b.bListPod)
+	if err != nil {
+		return false, err
+	}
+	wListSVCRegex, err := regexp.Compile(b.wListPod)
+	if err != nil {
+		return false, err
+	}
+	podFilterKey := fmt.Sprintf("%v/%v", namespace, pod)
+	if len(bListPodRegex.String()) > 0 {
+		if match := bListPodRegex.MatchString(podFilterKey); match {
+			return false, nil
+		}
+	}
+	if len(wListSVCRegex.String()) > 0 {
+		match := wListSVCRegex.MatchString(podFilterKey)
+		return match || b.namespaceFilter(namespace), nil
+	}
+	return b.namespaceFilter(namespace), err
+}
+func (b *BlackWhiteList) namespaceFilter(namespace string) bool {
+	bListNS := listToSet(parseList(b.bListNS))
+	wListNS := listToSet(parseList(b.wListNS))
+	if _, inBList := bListNS[namespace]; inBList {
+		return false
+	}
+	if len(b.wListNS) > 0 {
+		_, inWList := wListNS[namespace]
+		return inWList
+	}
+	return true
+}
+func parseList(raw string) []string {
+	if raw == "" {
+		return nil
+	}
+	splitted := strings.Split(raw, ",")
+	for i := range splitted {
+		splitted[i] = strings.Trim(splitted[i], " \n\t")
+	}
+	return splitted
+}

pilot/discovery/blackwhitelist_test.go

@@ -0,0 +1,173 @@
+package discovery
+
+import (
+	"testing"
+)
+
+func TestBlackWhiteList_IsResponsible(t *testing.T) {
+	type fields struct {
+		bListNS  string
+		wListNS  string
+		bListPod string
+		wListPod string
+	}
+	type args struct {
+		namespace string
+		pod       string
+	}
+
+	const (
+		defaultNS        = "default"
+		kubeNS           = "kube-system"
+		defaultAndKubeNS = "default,kube-system"
+		PodListRegex     = "^kube-system/(lb-.+|apiserver-proxy-nginx-preset-.+)$"
+		lbPodRegex       = "^kube-system/(lb-3160836842-proxy-nginx-eqygq-cfb79fccf-s5zz5.*)$"
+		apiServerPodName = "apiserver-proxy-nginx-preset-7db45cfddb-bsk97"
+		lbPodName        = "lb-3160836842-proxy-nginx-eqygq-cfb79fccf-s5zz5"
+	)
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    bool
+		wantErr bool
+	}{
+		{
+			name: "priority in blacklist and whitelist pod",
+			fields: fields{
+				bListNS:  kubeNS,
+				wListNS:  defaultNS,
+				bListPod: lbPodRegex,
+				wListPod: PodListRegex,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       lbPodName,
+			},
+			want: false,
+		},
+		{
+			name: "no pod name",
+			fields: fields{
+				bListNS:  kubeNS,
+				wListPod: PodListRegex,
+			},
+			args: args{
+				namespace: kubeNS,
+			},
+			want: false,
+		},
+		{
+			name: "priority in whitelist pod and blacklist namespace\"",
+			fields: fields{
+				bListNS:  defaultAndKubeNS,
+				wListPod: PodListRegex,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       apiServerPodName,
+			},
+			want: true,
+		},
+
+		{
+			name: "priority in blacklist pod and whitelist namespace",
+			fields: fields{
+				bListNS:  defaultNS,
+				wListNS:  kubeNS,
+				bListPod: PodListRegex,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       apiServerPodName,
+			},
+			want: false,
+		},
+		{
+			name: "in the whitelist, not in the blacklist namespace",
+			fields: fields{
+				bListNS: defaultNS,
+				wListNS: defaultAndKubeNS,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       lbPodName,
+			},
+			want: true,
+		},
+		{
+			name: "in the whitelist, not in the blacklist namespace",
+			fields: fields{
+				bListNS: defaultNS,
+				wListNS: defaultAndKubeNS,
+			},
+			args: args{
+				namespace: defaultNS,
+				pod:       lbPodName,
+			},
+			want: false,
+		},
+		{
+			name: "in the blacklist, not in the whitelist namespace",
+			fields: fields{
+				bListNS: kubeNS,
+				wListNS: defaultNS,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       lbPodName,
+			},
+			want: false,
+		},
+		{
+			name: "both in the blacklist and the whitelist namespace",
+			fields: fields{
+				bListNS: kubeNS,
+				wListNS: defaultAndKubeNS,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       lbPodName,
+			},
+			want: false,
+		},
+		{
+			name: "empty whitelist namespace",
+			fields: fields{
+				bListNS: defaultNS,
+			},
+			args: args{
+				namespace: kubeNS,
+				pod:       lbPodName,
+			},
+			want: true,
+		},
+		{
+			name: "empty blacklist and whitelist namespace",
+			args: args{
+				namespace: kubeNS,
+				pod:       lbPodName,
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &BlackWhiteList{
+				bListNS:  tt.fields.bListNS,
+				wListNS:  tt.fields.wListNS,
+				bListPod: tt.fields.bListPod,
+				wListPod: tt.fields.wListPod,
+			}
+			got, err := b.IsResponsible(tt.args.namespace, tt.args.pod)
+			if err != nil {
+				t.Errorf("IsResponsible() error = %v", err)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("IsResponsible() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pilot/discovery/discovery.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -52,10 +53,20 @@ type discovery struct {
 	mutex           sync.Mutex
 	bListNS         map[string]struct{} // blacklisted namespaces
 	wListNS         map[string]struct{} // whitelisted namespaces
+	bListPodRegex   *regexp.Regexp      // blacklisted Regexp pods
+	wListPodRegex   *regexp.Regexp      // whitelisted Regexp pods
 }
 
 // New creates a new Discovery
-func New(baseDir, logPrefix string, configurer configurer.Configurer, bListNS, wListNS []string) (Discovery, error) {
+func New(baseDir, logPrefix string, configurer configurer.Configurer, bListNS, wListNS []string, bListPodRegex, wListPodRegex string) (Discovery, error) {
+	bListPodCompiledRegex, err := regexp.Compile(bListPodRegex)
+	if err != nil {
+		return nil, fmt.Errorf("parse blacklisted pod name regex:%s", err.Error())
+	}
+	wListPodCompiledRegex, err := regexp.Compile(wListPodRegex)
+	if err != nil {
+		return nil, fmt.Errorf("parse whitelisted pod name regex:%s", err.Error())
+	}
 	if os.Getenv("DOCKER_API_VERSION") == "" {
 		os.Setenv("DOCKER_API_VERSION", "1.23")
 	}
@@ -95,6 +106,8 @@ func New(baseDir, logPrefix string, configurer configurer.Configurer, bListNS, w
 		existContainers: make(map[string]*containerInfo),
 		bListNS:         listToSet(bListNS),
 		wListNS:         listToSet(wListNS),
+		bListPodRegex:   bListPodCompiledRegex,
+		wListPodRegex:   wListPodCompiledRegex,
 	}, nil
 }
 
@@ -259,7 +272,7 @@ func (d *discovery) newContainer(containerJSON *types.ContainerJSON) error {
 	info := getContainerInfo(d.cache, containerJSON)
 	if len(containerJSON.Config.Labels) > 0 {
 		// Skip POD containers
-		if info.Name == "POD" || !d.isResponsible(info.Namespace) {
+		if info.Name == "POD" || !d.isResponsible(info.Namespace, info.Pod) {
 			return nil
 		}
 	}
@@ -312,7 +325,24 @@ func (d *discovery) Stop() {
 	d.configurer.Stop()
 }
 
-func (d *discovery) isResponsible(namespace string) bool {
+func (d *discovery) isResponsible(namespace string, pod string) bool {
+	log.Infof("[debug] isResponsible got namespace=%s, pod=%s", namespace, pod)
+	podFilterKey := fmt.Sprintf("%v/%v", namespace, pod)
+	if len(d.bListPodRegex.String()) > 0 {
+		if match := d.bListPodRegex.MatchString(podFilterKey); match {
+			log.Infof("[debug] isResponsible returned false due to pod black list=%v", d.bListPodRegex.String())
+			return false
+		}
+	}
+	if len(d.wListPodRegex.String()) > 0 {
+		match := d.wListPodRegex.MatchString(podFilterKey)
+		log.Infof("[debug] isResponsible returned %v due to pod white list,blackRegex=%v,whiteRegex=%v", match, d.bListPodRegex.String(), d.wListPodRegex.String())
+		return match || d.namespaceFilter(namespace)
+	}
+	return d.namespaceFilter(namespace)
+}
+
+func (d *discovery) namespaceFilter(namespace string) bool {
 	if _, inBList := d.bListNS[namespace]; inBList {
 		return false
 	}

release/logging-filebeat.yaml

@@ -60,6 +60,7 @@ _config:
       - "--logLevel=info"
       - "-e"
       - "--namespace.blacklist=default,kube-system"
+      - "--pod.whiteRegex=^kube-system/(lb-.+|apiserver-proxy-nginx-preset-.+)$"
       env: 
       - name: OUTPUT_CONFIG
         from: