Group by TCPPoolConfig config; Improve read config file lines; Fix ty…

…po (#209)

* Group by TCPPoolConfig config

* Improve read config file lines; Fix typo

README.md

@@ -113,8 +113,11 @@ Configuration file is "config.json" by default:
   "OnlyPrimaryDNS": false,
   "IPv6UseAlternativeDNS": false,
   "AlternativeDNSConcurrent": false,
-  "PoolIdleTimeout": 15,
-  "PoolMaxCapacity": 15,
+  "TCPPoolConfig": {
+    "InitialCapacity": 0,
+    "MaxCapacity": 15,
+    "IdleTimeout": 30
+  },
   "WhenPrimaryDNSAnswerNoneUse": "PrimaryDNS",
   "IPNetworkFile": {
     "Primary": "./ip_network_primary_sample",
@@ -208,8 +211,10 @@ IPv6). Overture will handle both TCP and UDP requests. Literal IPv6 addresses ar
 + OnlyPrimaryDNS: Disable dispatcher feature, use primary DNS only.
 + IPv6UseAlternativeDNS: Redirect IPv6 DNS queries to alternative DNS servers.
 + AlternativeDNSConcurrent: Query the PrimaryDNS and AlternativeDNS at the same time
-+ PoolIdleTimeout: Specify idle timeout for connection in pool
-+ PoolMaxCapacity: Specify max capacity for connection pool
++ TCPPoolConfig: Connection pool for TCP connections.
+    + IdleTimeout: Specify idle timeout for connection in pool
+    + InitialCapacity: Specify init capacity for connection pool
+    + MaxCapacity: Specify max capacity for connection pool
 + WhenPrimaryDNSAnswerNoneUse: If the response of PrimaryDNS exists and there is no `ANSWER SECTION` in it, the final DNS should be defined. (There is no `AAAA` record for most domains right now) 
 + File: Absolute path like `/path/to/file` is allowed. For Windows users, please use properly escaped path like
   `C:\\path\\to\\file.txt` in the configuration.

config.sample.json

@@ -32,6 +32,11 @@
   "OnlyPrimaryDNS": false,
   "IPv6UseAlternativeDNS": false,
   "AlternativeDNSConcurrent": false,
+  "TCPPoolConfig": {
+    "InitialCapacity": 0,
+    "MaxCapacity": 15,
+    "IdleTimeout": 30
+  },
   "PoolIdleTimeout": 15,
   "PoolMaxCapacity": 15,
   "WhenPrimaryDNSAnswerNoneUse": "PrimaryDNS",

config.test.json

@@ -32,8 +32,11 @@
   "OnlyPrimaryDNS": false,
   "IPv6UseAlternativeDNS": false,
   "AlternativeDNSConcurrent": false,
-  "PoolIdleTimeout": 15,
-  "PoolMaxCapacity": 15,
+  "TCPPoolConfig": {
+    "InitialCapacity": 0,
+    "MaxCapacity": 15,
+    "IdleTimeout": 30
+  },
   "WhenPrimaryDNSAnswerNoneUse": "PrimaryDNS",
   "IPNetworkFile": {
     "Primary": "./ip_network_primary_sample",

core/config/config.go

@@ -38,9 +38,12 @@ type Config struct {
 	OnlyPrimaryDNS           bool
 	IPv6UseAlternativeDNS    bool
 	AlternativeDNSConcurrent bool
-	PoolIdleTimeout          int
-	PoolMaxCapacity          int
-	IPNetworkFile            struct {
+	TCPPoolConfig            struct {
+		InitialCapacity int
+		MaxCapacity     int
+		IdleTimeout     int
+	}
+	IPNetworkFile struct {
 		Primary     string
 		Alternative string
 	}
@@ -141,31 +144,26 @@ func getDomainTTLMap(file string) map[string]uint32 {
 
 	dtl := map[string]uint32{}
 
-	reader := bufio.NewReader(f)
+	scanner := bufio.NewScanner(f)
 
-	for {
-		// The last line may not contains an '\n'
-		line, err := reader.ReadString('\n')
-		if err != nil && err != io.EOF {
-			log.Errorf("Failed to read domain TTL file %s: %s", file, err)
-			break
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
 		}
-
-		if line != "" {
-			words := strings.Fields(line)
-			if len(words) > 1 {
-				tempInt64, err := strconv.ParseUint(words[1], 10, 32)
-				dtl[words[0]] = uint32(tempInt64)
-				if err != nil {
-					log.WithFields(log.Fields{"domain": words[0], "ttl": words[1]}).Warnf("Invalid TTL for domain %s: %s", words[0], words[1])
-					failures++
-					failedLines = append(failedLines, line)
-				}
-				successes++
-			} else {
-				failedLines = append(failedLines, line)
+		words := strings.Fields(line)
+		if len(words) > 1 {
+			tempInt64, err := strconv.ParseUint(words[1], 10, 32)
+			dtl[words[0]] = uint32(tempInt64)
+			if err != nil {
+				log.WithFields(log.Fields{"domain": words[0], "ttl": words[1]}).Warnf("Invalid TTL for domain %s: %s", words[0], words[1])
 				failures++
+				failedLines = append(failedLines, line)
 			}
+			successes++
+		} else {
+			failedLines = append(failedLines, line)
+			failures++
 		}
 		if line == "" && err == io.EOF {
 			log.Debugf("Reading domain TTL file %s reached EOF", file)
@@ -221,7 +219,7 @@ func getFinder(name string) (f finder.Finder) {
 	case "full-map":
 		return &finderfull.Map{DataMap: make(map[string][]string, 100)}
 	default:
-		log.Warnf("Matcher %s does not exist, using full-map matcher as default", name)
+		log.Warnf("Finder %s does not exist, using full-map finder as default", name)
 		return &finderfull.Map{DataMap: make(map[string][]string, 100)}
 	}
 }
@@ -246,15 +244,12 @@ func initDomainMatcher(file string, name string, defaultName string) (m matcher.
 	defer f.Close()
 
 	lines := 0
-	reader := bufio.NewReader(f)
-
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil && err != io.EOF {
-			log.Errorf("Failed to read domain file %s: %s", file, err)
-			break
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
 		}
-
 		line = strings.TrimSpace(line)
 		if line != "" {
 			_ = m.Insert(line)
@@ -289,31 +284,22 @@ func getIPNetworkList(file string) []*net.IPNet {
 	failures := 0
 	var failedLines []string
 
-	reader := bufio.NewReader(f)
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil {
-			if err != io.EOF {
-				log.Errorf("Failed to read IP network file %s: %s", file, err)
-			} else {
-				log.Debugf("Reading IP network file %s has reached EOF", file)
-			}
-			break
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
 		}
-
-		if line != "" {
-			_, ipNet, err := net.ParseCIDR(strings.TrimSuffix(line, "\n"))
-			if err != nil {
-				log.Errorf("Error parsing IP network CIDR %s: %s", line, err)
-				failures++
-				failedLines = append(failedLines, line)
-				continue
-			}
-			ipNetList = append(ipNetList, ipNet)
-			successes++
+		_, ipNet, err := net.ParseCIDR(strings.TrimSuffix(line, "\n"))
+		if err != nil {
+			log.Errorf("Error parsing IP network CIDR %s: %s", line, err)
+			failures++
+			failedLines = append(failedLines, line)
+			continue
 		}
+		ipNetList = append(ipNetList, ipNet)
+		successes++
 	}
-
 	if len(ipNetList) > 0 {
 		log.Infof("IP network file %s has been loaded with %d records", file, successes)
 		if failures > 0 {

core/hosts/hosts.go

@@ -7,7 +7,6 @@ package hosts
 
 import (
 	"bufio"
-	"io"
 	"net"
 	"os"
 	"strings"
@@ -64,19 +63,9 @@ func (h *Hosts) initHosts() error {
 	defer f.Close()
 	defer log.Debugf("%s took %s", "Load hosts", time.Since(time.Now()))
 
-	reader := bufio.NewReader(f)
-	for {
-		line, err := reader.ReadString('\n')
-		if err != nil {
-			if err != io.EOF {
-				log.Errorf("NormalError reading hosts file: %s", err)
-			} else {
-				log.Debug("Reading hosts file reached EOF")
-			}
-			break
-		}
-
-		if err := h.parseLine(line); err != nil {
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		if err := h.parseLine(scanner.Text()); err != nil {
 			log.Warnf("Bad formatted hosts file line: %s", err)
 		}
 	}

core/init.go

@@ -28,8 +28,7 @@ func InitServer(configFilePath string) {
 
 		RedirectIPv6Record:       conf.IPv6UseAlternativeDNS,
 		AlternativeDNSConcurrent: conf.AlternativeDNSConcurrent,
-		PoolIdleTimeout:          conf.PoolIdleTimeout,
-		PoolMaxCapacity:          conf.PoolMaxCapacity,
+		TCPPoolConfig:            conf.TCPPoolConfig,
 		MinimumTTL:               conf.MinimumTTL,
 		DomainTTLMap:             conf.DomainTTLMap,
 

core/outbound/clients/resolver/base_resolver.go

@@ -85,8 +85,9 @@ func (c *BaseResolver) CreateBaseConn() (conn net.Conn, err error) {
 	return conn, err
 }
 
-var IdleTimeout time.Duration = 30 * time.Second
-var PoolMaxCapacity int = 15
+var InitialCapacity = 0
+var IdleTimeout = 30 * time.Second
+var MaxCapacity = 15
 
 func (c *BaseResolver) setTimeout(conn net.Conn) {
 	dnsTimeout := time.Duration(c.dnsUpstream.Timeout) * time.Second / 3
@@ -103,8 +104,8 @@ func (c *BaseResolver) setIdleTimeout(conn net.Conn) {
 
 func (c *BaseResolver) createConnectionPool(connCreate func() (interface{}, error), connClose func(interface{}) error) pool.Pool {
 	poolConfig := &pool.Config{
-		InitialCap: 0,
-		MaxCap:     PoolMaxCapacity,
+		InitialCap: InitialCapacity,
+		MaxCap:     MaxCapacity,
 		Factory:    connCreate,
 		Close:      connClose,
 		//Ping:       ping,

core/outbound/clients/resolver/resolver_test.go

@@ -10,11 +10,11 @@ import (
 
 var questionDomain = "www.yahoo.com."
 var udpUpstream = &common.DNSUpstream{
-	Name:             "Test-UDP",
-	Address:          "8.8.8.8",
-	Protocol:         "udp",
-	SOCKS5Address:    "",
-	Timeout:          6,
+	Name:          "Test-UDP",
+	Address:       "114.114.114.114",
+	Protocol:      "udp",
+	SOCKS5Address: "",
+	Timeout:       6,
 	EDNSClientSubnet: &common.EDNSClientSubnetType{
 		Policy:     "disable",
 		ExternalIP: "",
@@ -23,11 +23,11 @@ var udpUpstream = &common.DNSUpstream{
 }
 
 var tcpUpstream = &common.DNSUpstream{
-	Name:             "Test-TCP",
-	Address:          "8.8.8.8",
-	Protocol:         "tcp",
-	SOCKS5Address:    "",
-	Timeout:          6,
+	Name:          "Test-TCP",
+	Address:       "114.114.114.114",
+	Protocol:      "tcp",
+	SOCKS5Address: "",
+	Timeout:       6,
 	EDNSClientSubnet: &common.EDNSClientSubnetType{
 		Policy:     "disable",
 		ExternalIP: "",
@@ -36,11 +36,11 @@ var tcpUpstream = &common.DNSUpstream{
 }
 
 var tcptlsUpstream = &common.DNSUpstream{
-	Name:             "Test-TCPTLS",
-	Address:          "dns.google:[email protected]",
-	Protocol:         "tcp-tls",
-	SOCKS5Address:    "",
-	Timeout:          8,
+	Name:          "Test-TCPTLS",
+	Address:       "dns.google:[email protected]",
+	Protocol:      "tcp-tls",
+	SOCKS5Address: "",
+	Timeout:       8,
 	EDNSClientSubnet: &common.EDNSClientSubnetType{
 		Policy:     "disable",
 		ExternalIP: "",
@@ -49,11 +49,11 @@ var tcptlsUpstream = &common.DNSUpstream{
 }
 
 var httpsUpstream = &common.DNSUpstream{
-	Name:             "Test-HTTPS",
-	Address:          "https://dns.google/dns-query",
-	Protocol:         "https",
-	SOCKS5Address:    "",
-	Timeout:          8,
+	Name:          "Test-HTTPS",
+	Address:       "https://dns.google/dns-query",
+	Protocol:      "https",
+	SOCKS5Address: "",
+	Timeout:       8,
 	EDNSClientSubnet: &common.EDNSClientSubnetType{
 		Policy:     "disable",
 		ExternalIP: "",
@@ -94,7 +94,6 @@ func testTCP(t *testing.T) {
 	}
 }
 
-
 func testTCPTLS(t *testing.T) {
 	q := getQueryMsg(questionDomain, dns.TypeA)
 	resolver := NewResolver(udpUpstream)
@@ -104,7 +103,6 @@ func testTCPTLS(t *testing.T) {
 	}
 }
 
-
 func testHTTPS(t *testing.T) {
 	q := getQueryMsg(questionDomain, dns.TypeA)
 	resolver := NewResolver(udpUpstream)

core/outbound/dispatcher.go

@@ -27,8 +27,11 @@ type Dispatcher struct {
 	DomainAlternativeList       matcher.Matcher
 	RedirectIPv6Record          bool
 	AlternativeDNSConcurrent    bool
-	PoolIdleTimeout          int
-	PoolMaxCapacity          int
+	TCPPoolConfig               struct {
+		InitialCapacity int
+		MaxCapacity     int
+		IdleTimeout     int
+	}
 
 	MinimumTTL   int
 	DomainTTLMap map[string]uint32
@@ -49,9 +52,10 @@ func createResolver(ul []*common.DNSUpstream) (resolvers []resolver.Resolver) {
 }
 
 func (d *Dispatcher) Init() {
-	resolver.IdleTimeout = time.Duration(d.PoolIdleTimeout) * time.Second
-	resolver.PoolMaxCapacity = d.PoolMaxCapacity
-	log.Debugf("Set pool's IdleTimeout to %d, MaxCapacity to %d", d.PoolIdleTimeout, d.PoolMaxCapacity)
+	resolver.IdleTimeout = time.Duration(d.TCPPoolConfig.IdleTimeout) * time.Second
+	resolver.MaxCapacity = d.TCPPoolConfig.MaxCapacity
+	resolver.InitialCapacity = d.TCPPoolConfig.InitialCapacity
+	log.Debugf("Set pool's IdleTimeout to %d, InitialCapacity to %d, MaxCapacity to %d", d.TCPPoolConfig.IdleTimeout, d.TCPPoolConfig.InitialCapacity, d.TCPPoolConfig.MaxCapacity)
 	d.primaryResolvers = createResolver(d.PrimaryDNS)
 	d.alternativeResolvers = createResolver(d.AlternativeDNS)
 }

core/outbound/dispatcher_test.go

@@ -30,8 +30,7 @@ func init() {
 
 		RedirectIPv6Record:       conf.IPv6UseAlternativeDNS,
 		AlternativeDNSConcurrent: conf.AlternativeDNSConcurrent,
-		PoolIdleTimeout:          conf.PoolIdleTimeout,
-		PoolMaxCapacity:          conf.PoolMaxCapacity,
+		TCPPoolConfig:            conf.TCPPoolConfig,
 		MinimumTTL:               conf.MinimumTTL,
 		DomainTTLMap:             conf.DomainTTLMap,
 
@@ -92,7 +91,7 @@ func testCache(t *testing.T) {
 	exchange(questionDomain, dns.TypeA)
 	now := time.Now()
 	exchange(questionDomain, dns.TypeA)
-	if time.Since(now) > 10 * time.Millisecond {
+	if time.Since(now) > 10*time.Millisecond {
 		t.Error(time.Since(now).String() + " " + "Cache response slower than 10ms")
 	}
 }