From 7b811096c65ef5857a1f9ca18c6f52492c3ab04b Mon Sep 17 00:00:00 2001 From: Shawn <10613846+shawn1m@users.noreply.github.com> Date: Wed, 19 Feb 2020 22:02:00 +0800 Subject: [PATCH] Group by TCPPoolConfig config; Improve read config file lines; Fix typo (#209) * Group by TCPPoolConfig config * Improve read config file lines; Fix typo --- README.md | 13 ++- config.sample.json | 5 + config.test.json | 7 +- core/config/config.go | 96 ++++++++----------- core/hosts/hosts.go | 17 +--- core/init.go | 3 +- .../clients/resolver/base_resolver.go | 9 +- .../clients/resolver/resolver_test.go | 42 ++++---- core/outbound/dispatcher.go | 14 ++- core/outbound/dispatcher_test.go | 5 +- 10 files changed, 100 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 9d31ec9..e71f93e 100755 --- a/README.md +++ b/README.md @@ -113,8 +113,11 @@ Configuration file is "config.json" by default: "OnlyPrimaryDNS": false, "IPv6UseAlternativeDNS": false, "AlternativeDNSConcurrent": false, - "PoolIdleTimeout": 15, - "PoolMaxCapacity": 15, + "TCPPoolConfig": { + "InitialCapacity": 0, + "MaxCapacity": 15, + "IdleTimeout": 30 + }, "WhenPrimaryDNSAnswerNoneUse": "PrimaryDNS", "IPNetworkFile": { "Primary": "./ip_network_primary_sample", @@ -208,8 +211,10 @@ IPv6). Overture will handle both TCP and UDP requests. Literal IPv6 addresses ar + OnlyPrimaryDNS: Disable dispatcher feature, use primary DNS only. + IPv6UseAlternativeDNS: Redirect IPv6 DNS queries to alternative DNS servers. + AlternativeDNSConcurrent: Query the PrimaryDNS and AlternativeDNS at the same time -+ PoolIdleTimeout: Specify idle timeout for connection in pool -+ PoolMaxCapacity: Specify max capacity for connection pool ++ TCPPoolConfig: Connection pool for TCP connections. + + IdleTimeout: Specify idle timeout for connection in pool + + InitialCapacity: Specify init capacity for connection pool + + MaxCapacity: Specify max capacity for connection pool + WhenPrimaryDNSAnswerNoneUse: If the response of PrimaryDNS exists and there is no `ANSWER SECTION` in it, the final DNS should be defined. (There is no `AAAA` record for most domains right now) + File: Absolute path like `/path/to/file` is allowed. For Windows users, please use properly escaped path like `C:\\path\\to\\file.txt` in the configuration. diff --git a/config.sample.json b/config.sample.json index 1037690..4cfe600 100644 --- a/config.sample.json +++ b/config.sample.json @@ -32,6 +32,11 @@ "OnlyPrimaryDNS": false, "IPv6UseAlternativeDNS": false, "AlternativeDNSConcurrent": false, + "TCPPoolConfig": { + "InitialCapacity": 0, + "MaxCapacity": 15, + "IdleTimeout": 30 + }, "PoolIdleTimeout": 15, "PoolMaxCapacity": 15, "WhenPrimaryDNSAnswerNoneUse": "PrimaryDNS", diff --git a/config.test.json b/config.test.json index 1200c53..fad1a03 100644 --- a/config.test.json +++ b/config.test.json @@ -32,8 +32,11 @@ "OnlyPrimaryDNS": false, "IPv6UseAlternativeDNS": false, "AlternativeDNSConcurrent": false, - "PoolIdleTimeout": 15, - "PoolMaxCapacity": 15, + "TCPPoolConfig": { + "InitialCapacity": 0, + "MaxCapacity": 15, + "IdleTimeout": 30 + }, "WhenPrimaryDNSAnswerNoneUse": "PrimaryDNS", "IPNetworkFile": { "Primary": "./ip_network_primary_sample", diff --git a/core/config/config.go b/core/config/config.go index b795544..325e9f1 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -38,9 +38,12 @@ type Config struct { OnlyPrimaryDNS bool IPv6UseAlternativeDNS bool AlternativeDNSConcurrent bool - PoolIdleTimeout int - PoolMaxCapacity int - IPNetworkFile struct { + TCPPoolConfig struct { + InitialCapacity int + MaxCapacity int + IdleTimeout int + } + IPNetworkFile struct { Primary string Alternative string } @@ -141,31 +144,26 @@ func getDomainTTLMap(file string) map[string]uint32 { dtl := map[string]uint32{} - reader := bufio.NewReader(f) + scanner := bufio.NewScanner(f) - for { - // The last line may not contains an '\n' - line, err := reader.ReadString('\n') - if err != nil && err != io.EOF { - log.Errorf("Failed to read domain TTL file %s: %s", file, err) - break + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue } - - if line != "" { - words := strings.Fields(line) - if len(words) > 1 { - tempInt64, err := strconv.ParseUint(words[1], 10, 32) - dtl[words[0]] = uint32(tempInt64) - if err != nil { - log.WithFields(log.Fields{"domain": words[0], "ttl": words[1]}).Warnf("Invalid TTL for domain %s: %s", words[0], words[1]) - failures++ - failedLines = append(failedLines, line) - } - successes++ - } else { - failedLines = append(failedLines, line) + words := strings.Fields(line) + if len(words) > 1 { + tempInt64, err := strconv.ParseUint(words[1], 10, 32) + dtl[words[0]] = uint32(tempInt64) + if err != nil { + log.WithFields(log.Fields{"domain": words[0], "ttl": words[1]}).Warnf("Invalid TTL for domain %s: %s", words[0], words[1]) failures++ + failedLines = append(failedLines, line) } + successes++ + } else { + failedLines = append(failedLines, line) + failures++ } if line == "" && err == io.EOF { log.Debugf("Reading domain TTL file %s reached EOF", file) @@ -221,7 +219,7 @@ func getFinder(name string) (f finder.Finder) { case "full-map": return &finderfull.Map{DataMap: make(map[string][]string, 100)} default: - log.Warnf("Matcher %s does not exist, using full-map matcher as default", name) + log.Warnf("Finder %s does not exist, using full-map finder as default", name) return &finderfull.Map{DataMap: make(map[string][]string, 100)} } } @@ -246,15 +244,12 @@ func initDomainMatcher(file string, name string, defaultName string) (m matcher. defer f.Close() lines := 0 - reader := bufio.NewReader(f) - - for { - line, err := reader.ReadString('\n') - if err != nil && err != io.EOF { - log.Errorf("Failed to read domain file %s: %s", file, err) - break + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue } - line = strings.TrimSpace(line) if line != "" { _ = m.Insert(line) @@ -289,31 +284,22 @@ func getIPNetworkList(file string) []*net.IPNet { failures := 0 var failedLines []string - reader := bufio.NewReader(f) - for { - line, err := reader.ReadString('\n') - if err != nil { - if err != io.EOF { - log.Errorf("Failed to read IP network file %s: %s", file, err) - } else { - log.Debugf("Reading IP network file %s has reached EOF", file) - } - break + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue } - - if line != "" { - _, ipNet, err := net.ParseCIDR(strings.TrimSuffix(line, "\n")) - if err != nil { - log.Errorf("Error parsing IP network CIDR %s: %s", line, err) - failures++ - failedLines = append(failedLines, line) - continue - } - ipNetList = append(ipNetList, ipNet) - successes++ + _, ipNet, err := net.ParseCIDR(strings.TrimSuffix(line, "\n")) + if err != nil { + log.Errorf("Error parsing IP network CIDR %s: %s", line, err) + failures++ + failedLines = append(failedLines, line) + continue } + ipNetList = append(ipNetList, ipNet) + successes++ } - if len(ipNetList) > 0 { log.Infof("IP network file %s has been loaded with %d records", file, successes) if failures > 0 { diff --git a/core/hosts/hosts.go b/core/hosts/hosts.go index 4acfb17..01b6bb0 100644 --- a/core/hosts/hosts.go +++ b/core/hosts/hosts.go @@ -7,7 +7,6 @@ package hosts import ( "bufio" - "io" "net" "os" "strings" @@ -64,19 +63,9 @@ func (h *Hosts) initHosts() error { defer f.Close() defer log.Debugf("%s took %s", "Load hosts", time.Since(time.Now())) - reader := bufio.NewReader(f) - for { - line, err := reader.ReadString('\n') - if err != nil { - if err != io.EOF { - log.Errorf("NormalError reading hosts file: %s", err) - } else { - log.Debug("Reading hosts file reached EOF") - } - break - } - - if err := h.parseLine(line); err != nil { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if err := h.parseLine(scanner.Text()); err != nil { log.Warnf("Bad formatted hosts file line: %s", err) } } diff --git a/core/init.go b/core/init.go index b839c0a..eff0d51 100644 --- a/core/init.go +++ b/core/init.go @@ -28,8 +28,7 @@ func InitServer(configFilePath string) { RedirectIPv6Record: conf.IPv6UseAlternativeDNS, AlternativeDNSConcurrent: conf.AlternativeDNSConcurrent, - PoolIdleTimeout: conf.PoolIdleTimeout, - PoolMaxCapacity: conf.PoolMaxCapacity, + TCPPoolConfig: conf.TCPPoolConfig, MinimumTTL: conf.MinimumTTL, DomainTTLMap: conf.DomainTTLMap, diff --git a/core/outbound/clients/resolver/base_resolver.go b/core/outbound/clients/resolver/base_resolver.go index 49b0e24..90ebd7d 100644 --- a/core/outbound/clients/resolver/base_resolver.go +++ b/core/outbound/clients/resolver/base_resolver.go @@ -85,8 +85,9 @@ func (c *BaseResolver) CreateBaseConn() (conn net.Conn, err error) { return conn, err } -var IdleTimeout time.Duration = 30 * time.Second -var PoolMaxCapacity int = 15 +var InitialCapacity = 0 +var IdleTimeout = 30 * time.Second +var MaxCapacity = 15 func (c *BaseResolver) setTimeout(conn net.Conn) { dnsTimeout := time.Duration(c.dnsUpstream.Timeout) * time.Second / 3 @@ -103,8 +104,8 @@ func (c *BaseResolver) setIdleTimeout(conn net.Conn) { func (c *BaseResolver) createConnectionPool(connCreate func() (interface{}, error), connClose func(interface{}) error) pool.Pool { poolConfig := &pool.Config{ - InitialCap: 0, - MaxCap: PoolMaxCapacity, + InitialCap: InitialCapacity, + MaxCap: MaxCapacity, Factory: connCreate, Close: connClose, //Ping: ping, diff --git a/core/outbound/clients/resolver/resolver_test.go b/core/outbound/clients/resolver/resolver_test.go index 70a2aac..517b18e 100644 --- a/core/outbound/clients/resolver/resolver_test.go +++ b/core/outbound/clients/resolver/resolver_test.go @@ -10,11 +10,11 @@ import ( var questionDomain = "www.yahoo.com." var udpUpstream = &common.DNSUpstream{ - Name: "Test-UDP", - Address: "8.8.8.8", - Protocol: "udp", - SOCKS5Address: "", - Timeout: 6, + Name: "Test-UDP", + Address: "114.114.114.114", + Protocol: "udp", + SOCKS5Address: "", + Timeout: 6, EDNSClientSubnet: &common.EDNSClientSubnetType{ Policy: "disable", ExternalIP: "", @@ -23,11 +23,11 @@ var udpUpstream = &common.DNSUpstream{ } var tcpUpstream = &common.DNSUpstream{ - Name: "Test-TCP", - Address: "8.8.8.8", - Protocol: "tcp", - SOCKS5Address: "", - Timeout: 6, + Name: "Test-TCP", + Address: "114.114.114.114", + Protocol: "tcp", + SOCKS5Address: "", + Timeout: 6, EDNSClientSubnet: &common.EDNSClientSubnetType{ Policy: "disable", ExternalIP: "", @@ -36,11 +36,11 @@ var tcpUpstream = &common.DNSUpstream{ } var tcptlsUpstream = &common.DNSUpstream{ - Name: "Test-TCPTLS", - Address: "dns.google:853@8.8.8.8", - Protocol: "tcp-tls", - SOCKS5Address: "", - Timeout: 8, + Name: "Test-TCPTLS", + Address: "dns.google:853@8.8.8.8", + Protocol: "tcp-tls", + SOCKS5Address: "", + Timeout: 8, EDNSClientSubnet: &common.EDNSClientSubnetType{ Policy: "disable", ExternalIP: "", @@ -49,11 +49,11 @@ var tcptlsUpstream = &common.DNSUpstream{ } var httpsUpstream = &common.DNSUpstream{ - Name: "Test-HTTPS", - Address: "https://dns.google/dns-query", - Protocol: "https", - SOCKS5Address: "", - Timeout: 8, + Name: "Test-HTTPS", + Address: "https://dns.google/dns-query", + Protocol: "https", + SOCKS5Address: "", + Timeout: 8, EDNSClientSubnet: &common.EDNSClientSubnetType{ Policy: "disable", ExternalIP: "", @@ -94,7 +94,6 @@ func testTCP(t *testing.T) { } } - func testTCPTLS(t *testing.T) { q := getQueryMsg(questionDomain, dns.TypeA) resolver := NewResolver(udpUpstream) @@ -104,7 +103,6 @@ func testTCPTLS(t *testing.T) { } } - func testHTTPS(t *testing.T) { q := getQueryMsg(questionDomain, dns.TypeA) resolver := NewResolver(udpUpstream) diff --git a/core/outbound/dispatcher.go b/core/outbound/dispatcher.go index ad5c778..3c1fbad 100644 --- a/core/outbound/dispatcher.go +++ b/core/outbound/dispatcher.go @@ -27,8 +27,11 @@ type Dispatcher struct { DomainAlternativeList matcher.Matcher RedirectIPv6Record bool AlternativeDNSConcurrent bool - PoolIdleTimeout int - PoolMaxCapacity int + TCPPoolConfig struct { + InitialCapacity int + MaxCapacity int + IdleTimeout int + } MinimumTTL int DomainTTLMap map[string]uint32 @@ -49,9 +52,10 @@ func createResolver(ul []*common.DNSUpstream) (resolvers []resolver.Resolver) { } func (d *Dispatcher) Init() { - resolver.IdleTimeout = time.Duration(d.PoolIdleTimeout) * time.Second - resolver.PoolMaxCapacity = d.PoolMaxCapacity - log.Debugf("Set pool's IdleTimeout to %d, MaxCapacity to %d", d.PoolIdleTimeout, d.PoolMaxCapacity) + resolver.IdleTimeout = time.Duration(d.TCPPoolConfig.IdleTimeout) * time.Second + resolver.MaxCapacity = d.TCPPoolConfig.MaxCapacity + resolver.InitialCapacity = d.TCPPoolConfig.InitialCapacity + log.Debugf("Set pool's IdleTimeout to %d, InitialCapacity to %d, MaxCapacity to %d", d.TCPPoolConfig.IdleTimeout, d.TCPPoolConfig.InitialCapacity, d.TCPPoolConfig.MaxCapacity) d.primaryResolvers = createResolver(d.PrimaryDNS) d.alternativeResolvers = createResolver(d.AlternativeDNS) } diff --git a/core/outbound/dispatcher_test.go b/core/outbound/dispatcher_test.go index 292945a..343b99c 100644 --- a/core/outbound/dispatcher_test.go +++ b/core/outbound/dispatcher_test.go @@ -30,8 +30,7 @@ func init() { RedirectIPv6Record: conf.IPv6UseAlternativeDNS, AlternativeDNSConcurrent: conf.AlternativeDNSConcurrent, - PoolIdleTimeout: conf.PoolIdleTimeout, - PoolMaxCapacity: conf.PoolMaxCapacity, + TCPPoolConfig: conf.TCPPoolConfig, MinimumTTL: conf.MinimumTTL, DomainTTLMap: conf.DomainTTLMap, @@ -92,7 +91,7 @@ func testCache(t *testing.T) { exchange(questionDomain, dns.TypeA) now := time.Now() exchange(questionDomain, dns.TypeA) - if time.Since(now) > 10 * time.Millisecond { + if time.Since(now) > 10*time.Millisecond { t.Error(time.Since(now).String() + " " + "Cache response slower than 10ms") } }