hybrid reader some minor feature updates (#18)

cmd/ips/cmd_config.go

@@ -18,6 +18,7 @@ package ips
 
 import (
 	"fmt"
+	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -82,13 +83,13 @@ func Config(cmd *cobra.Command, args []string) {
 			log.Fatal("missing key or value")
 			return
 		}
-		SetConfig(args[1], args[2])
+		SetConfig(args[1], args[2:])
 	case CmdUnset:
 		if len(args) < 2 {
 			log.Fatal("missing key")
 			return
 		}
-		SetConfig(args[1], "")
+		SetConfig(args[1], []string{})
 	case CmdReset:
 		if err := config.ResetConfig(); err != nil {
 			log.Fatal("reset config failed ", err)
@@ -111,23 +112,31 @@ var MagicMap = map[string]string{
 	"dbip-asn":  "dbip-asn-lite.mmdb",
 }
 
-func SetConfig(key string, value string) {
+func SetConfig(key string, value []string) {
 
 	if _key, ok := MagicMap[key]; ok {
 		key = _key
 	}
 
-	if _value, ok := MagicMap[value]; ok {
-		value = _value
+	for i := range value {
+		if _value, ok := MagicMap[value[i]]; ok {
+			value[i] = _value
+		}
+	}
+
+	var val interface{}
+	val = value
+	if len(value) == 1 {
+		val = value[0]
 	}
 
-	if err := config.SetConfig(key, value); err != nil {
+	if err := config.SetConfig(key, val); err != nil {
 		log.Fatal(err)
 	}
 
 	if len(value) == 0 {
 		log.Infof("unset %s success", key)
 	} else {
-		log.Infof("set %s: [%s] success", key, value)
+		log.Infof("set %s: [%s] success", key, strings.Join(value, ","))
 	}
 }

cmd/ips/cmd_dump.go

@@ -32,9 +32,10 @@ func init() {
 	dumpCmd.Flags().StringVarP(&lang, "lang", "", "", UsageLang)
 
 	// input & output
-	dumpCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", UsageDPInputFile)
-	dumpCmd.Flags().StringVarP(&inputFormat, "input-format", "", "", UsageDPInputFormat)
+	dumpCmd.Flags().StringSliceVarP(&inputFile, "input-file", "i", nil, UsageDPInputFile)
+	dumpCmd.Flags().StringSliceVarP(&inputFormat, "input-format", "", nil, UsageDPInputFormat)
 	dumpCmd.Flags().StringVarP(&readerOption, "input-option", "", "", UsageReaderOption)
+	dumpCmd.Flags().StringVarP(&hybridMode, "hybrid-mode", "", "aggregation", UsageHybridMode)
 	dumpCmd.Flags().StringVarP(&outputFile, "output-file", "o", "", UsageDumpOutputFile)
 
 }
@@ -63,7 +64,7 @@ func Dump(cmd *cobra.Command, args []string) {
 	}
 
 	if len(inputFile) == 0 {
-		inputFile = args[0]
+		inputFile = []string{args[0]}
 	}
 
 	if err := manager.Pack(inputFormat, inputFile, plain.DBFormat, outputFile); err != nil {

cmd/ips/cmd_myip.go

@@ -37,13 +37,14 @@ func init() {
 	myipCmd.Flags().StringVarP(&lang, "lang", "", "", UsageLang)
 
 	// database
-	myipCmd.Flags().StringVarP(&rootFile, "file", "i", "", UsageQueryFile)
-	myipCmd.Flags().StringVarP(&rootFormat, "format", "", "", UsageQueryFormat)
-	myipCmd.Flags().StringVarP(&rootIPv4File, "ipv4-file", "", "", UsageQueryIPv4File)
-	myipCmd.Flags().StringVarP(&rootIPv4Format, "ipv4-format", "", "", UsageQueryIPv4Format)
-	myipCmd.Flags().StringVarP(&rootIPv6File, "ipv6-file", "", "", UsageQueryIPv6File)
-	myipCmd.Flags().StringVarP(&rootIPv6Format, "ipv6-format", "", "", UsageQueryIPv6Format)
+	myipCmd.Flags().StringSliceVarP(&rootFile, "file", "i", nil, UsageQueryFile)
+	myipCmd.Flags().StringSliceVarP(&rootFormat, "format", "", nil, UsageQueryFormat)
+	myipCmd.Flags().StringSliceVarP(&rootIPv4File, "ipv4-file", "", nil, UsageQueryIPv4File)
+	myipCmd.Flags().StringSliceVarP(&rootIPv4Format, "ipv4-format", "", nil, UsageQueryIPv4Format)
+	myipCmd.Flags().StringSliceVarP(&rootIPv6File, "ipv6-file", "", nil, UsageQueryIPv6File)
+	myipCmd.Flags().StringSliceVarP(&rootIPv6Format, "ipv6-format", "", nil, UsageQueryIPv6Format)
 	myipCmd.Flags().StringVarP(&readerOption, "database-option", "", "", UsageReaderOption)
+	myipCmd.Flags().StringVarP(&hybridMode, "hybrid-mode", "", "aggregation", UsageHybridMode)
 
 	// output
 	myipCmd.Flags().StringVarP(&rootTextFormat, "text-format", "", "", UsageTextFormat)

cmd/ips/cmd_pack.go

@@ -30,9 +30,10 @@ func init() {
 	packCmd.Flags().StringVarP(&lang, "lang", "", "", UsageLang)
 
 	// input & output
-	packCmd.Flags().StringVarP(&inputFile, "input-file", "i", "", UsageDPInputFile)
-	packCmd.Flags().StringVarP(&inputFormat, "input-format", "", "", UsageDPInputFormat)
+	packCmd.Flags().StringSliceVarP(&inputFile, "input-file", "i", nil, UsageDPInputFile)
+	packCmd.Flags().StringSliceVarP(&inputFormat, "input-format", "", nil, UsageDPInputFormat)
 	packCmd.Flags().StringVarP(&readerOption, "input-option", "", "", UsageReaderOption)
+	packCmd.Flags().StringVarP(&hybridMode, "hybrid-mode", "", "aggregation", UsageHybridMode)
 	packCmd.Flags().StringVarP(&outputFile, "output-file", "o", "", UsagePackOutputFile)
 	packCmd.Flags().StringVarP(&outputFormat, "output-format", "", "", UsagePackOutputFormat)
 	packCmd.Flags().StringVarP(&writerOption, "output-option", "", "", UsageWriterOption)

cmd/ips/cmd_root.go

@@ -18,6 +18,7 @@ package ips
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"os"
 	"path"
@@ -45,13 +46,14 @@ func init() {
 	rootCmd.Flags().StringVarP(&lang, "lang", "", "", UsageLang)
 
 	// database
-	rootCmd.Flags().StringVarP(&rootFile, "file", "i", "", UsageQueryFile)
-	rootCmd.Flags().StringVarP(&rootFormat, "format", "", "", UsageQueryFormat)
-	rootCmd.Flags().StringVarP(&rootIPv4File, "ipv4-file", "", "", UsageQueryIPv4File)
-	rootCmd.Flags().StringVarP(&rootIPv4Format, "ipv4-format", "", "", UsageQueryIPv4Format)
-	rootCmd.Flags().StringVarP(&rootIPv6File, "ipv6-file", "", "", UsageQueryIPv6File)
-	rootCmd.Flags().StringVarP(&rootIPv6Format, "ipv6-format", "", "", UsageQueryIPv6Format)
+	rootCmd.Flags().StringSliceVarP(&rootFile, "file", "i", nil, UsageQueryFile)
+	rootCmd.Flags().StringSliceVarP(&rootFormat, "format", "", nil, UsageQueryFormat)
+	rootCmd.Flags().StringSliceVarP(&rootIPv4File, "ipv4-file", "", nil, UsageQueryIPv4File)
+	rootCmd.Flags().StringSliceVarP(&rootIPv4Format, "ipv4-format", "", nil, UsageQueryIPv4Format)
+	rootCmd.Flags().StringSliceVarP(&rootIPv6File, "ipv6-file", "", nil, UsageQueryIPv6File)
+	rootCmd.Flags().StringSliceVarP(&rootIPv6Format, "ipv6-format", "", nil, UsageQueryIPv6Format)
 	rootCmd.Flags().StringVarP(&readerOption, "database-option", "", "", UsageReaderOption)
+	rootCmd.Flags().StringVarP(&hybridMode, "hybrid-mode", "", "aggregation", UsageHybridMode)
 
 	// output
 	rootCmd.Flags().StringVarP(&rootTextFormat, "text-format", "", "", UsageTextFormat)
@@ -106,6 +108,7 @@ func Root(cmd *cobra.Command, args []string) {
 		}
 
 		scanner := bufio.NewScanner(os.Stdin)
+		scanner.Split(ScanLines)
 		for scanner.Scan() {
 			text := scanner.Text()
 			if len(text) == 0 {
@@ -118,7 +121,7 @@ func Root(cmd *cobra.Command, args []string) {
 			if len(ret) == 0 {
 				continue
 			}
-			fmt.Println(ret)
+			fmt.Print(ret)
 		}
 		return
 	}
@@ -153,3 +156,24 @@ func PreRunInit(cmd *cobra.Command, args []string) {
 	conf := GetFlagConfig()
 	manager = ips.NewManager(conf)
 }
+
+// ScanLines scan lines but keep the suffix \r and \n
+func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	if i := bytes.IndexByte(data, '\n'); i >= 0 {
+		return i + 1, data[:i+1], nil
+	}
+	if i := bytes.IndexByte(data, '\r'); i >= 0 {
+		return i + 1, data[:i+1], nil
+	}
+
+	// If we're at EOF, we have a final, non-terminated line. Return it.
+	if atEOF {
+		return len(data), data, nil
+	}
+	// Request more data.
+	return 0, nil, nil
+}

cmd/ips/cmd_server.go

@@ -32,13 +32,15 @@ func init() {
 	serverCmd.Flags().StringVarP(&lang, "lang", "", "", UsageLang)
 
 	// database
-	serverCmd.Flags().StringVarP(&rootFile, "file", "i", "", UsageQueryFile)
-	serverCmd.Flags().StringVarP(&rootFormat, "format", "", "", UsageQueryFormat)
-	serverCmd.Flags().StringVarP(&rootIPv4File, "ipv4-file", "", "", UsageQueryIPv4File)
-	serverCmd.Flags().StringVarP(&rootIPv4Format, "ipv4-format", "", "", UsageQueryIPv4Format)
-	serverCmd.Flags().StringVarP(&rootIPv6File, "ipv6-file", "", "", UsageQueryIPv6File)
-	serverCmd.Flags().StringVarP(&rootIPv6Format, "ipv6-format", "", "", UsageQueryIPv6Format)
+	serverCmd.Flags().StringSliceVarP(&rootFile, "file", "i", nil, UsageQueryFile)
+	serverCmd.Flags().StringSliceVarP(&rootFormat, "format", "", nil, UsageQueryFormat)
+	serverCmd.Flags().StringSliceVarP(&rootIPv4File, "ipv4-file", "", nil, UsageQueryIPv4File)
+	serverCmd.Flags().StringSliceVarP(&rootIPv4Format, "ipv4-format", "", nil, UsageQueryIPv4Format)
+	serverCmd.Flags().StringSliceVarP(&rootIPv6File, "ipv6-file", "", nil, UsageQueryIPv6File)
+	serverCmd.Flags().StringSliceVarP(&rootIPv6Format, "ipv6-format", "", nil, UsageQueryIPv6Format)
 	serverCmd.Flags().StringVarP(&readerOption, "database-option", "", "", UsageReaderOption)
+	serverCmd.Flags().StringVarP(&hybridMode, "hybrid-mode", "", "aggregation", UsageHybridMode)
+
 }
 
 var serverCmd = &cobra.Command{

cmd/ips/config.go

@@ -55,22 +55,25 @@ var (
 	// root command flags
 	// database
 	// rootFormat defines the format for database.
-	rootFormat string
+	rootFormat []string
 
 	// rootFile specifies the file path for database.
-	rootFile string
+	rootFile []string
 
 	// rootIPv4Format defines the format for IPv4 database.
-	rootIPv4Format string
+	rootIPv4Format []string
 
 	// rootIPv4File specifies the file path for IPv4 database.
-	rootIPv4File string
+	rootIPv4File []string
 
 	// rootIPv6Format defines the format for IPv6 database.
-	rootIPv6Format string
+	rootIPv6Format []string
 
 	// rootIPv6File specifies the file path for IPv6 database.
-	rootIPv6File string
+	rootIPv6File []string
+
+	// hybridMode specifies the operational mode of the HybridReader.
+	hybridMode string
 
 	// output
 	// rootTextFormat defines the format for text output.
@@ -97,10 +100,10 @@ var (
 	dpRewriterFiles string
 
 	// inputFile specifies the input file for dump and pack operations.
-	inputFile string
+	inputFile []string
 
 	// inputFormat specifies the input format for dump and pack operations.
-	inputFormat string
+	inputFormat []string
 
 	// outputFile specifies the output file for dump and pack operations.
 	outputFile string
@@ -180,6 +183,22 @@ func GetFlagConfig() *ips.Config {
 		}
 	}
 
+	if len(conf.IPv4Format) == 0 {
+		conf.IPv4Format = make([]string, len(conf.IPv4File))
+	} else if len(conf.IPv4File) != len(conf.IPv4Format) {
+		log.Fatal("IPv4 file and format mismatch")
+	}
+
+	if len(conf.IPv6Format) == 0 {
+		conf.IPv6Format = make([]string, len(conf.IPv6File))
+	} else if len(conf.IPv6File) != len(conf.IPv6Format) {
+		log.Fatal("IPv6 file and format mismatch")
+	}
+
+	if len(hybridMode) != 0 {
+		conf.HybridMode = hybridMode
+	}
+
 	if len(rootTextFormat) != 0 {
 		conf.TextFormat = rootTextFormat
 	}

cmd/ips/const.go

@@ -45,6 +45,7 @@ const (
 	UsagePackOutputFormat = "The format for the output IP database file."
 	UsageReaderOption     = "Additional options for the database reader, if applicable."
 	UsageWriterOption     = "Additional options for the database writer, if applicable."
+	UsageHybridMode       = "Sets mode for multi-IP source handling; 'comparison' to compare, 'aggregation' to merge data."
 
 	// Output Flags
 

docs/config.md

@@ -17,6 +17,7 @@
     * [ipv4_format](#ipv4format)
     * [ipv6_file](#ipv6file)
     * [ipv6_format](#ipv6format)
+    * [hybrid_mode](#hybridmode)
     * [fields](#fields)
     * [use_db_fields](#usedbfields)
     * [rewrite_files](#rewritefiles)
@@ -168,6 +169,15 @@ IPS 提供了丰富的配置参数以适应不同的使用场景。以下是所
 
 当数据库文件后缀不足以确定文件格式时，用来指定 IPv6 数据库的格式。通常不需要设置。
 
+### hybrid_mode
+
+指定了混合读取器（Hybrid Reader）的操作模式，字符串参数。操作模式决定了如何处理和组合来自多个 IP 数据库的数据。
+
+可选参数为 `comparison` 与 `aggregation`，默认值为 `aggregation`。
+
+- `comparison`：比较模式，适用于需要跨不同 IP 数据库比较数据的场景，输出所有集成数据库的数据，便于识别每个源之间的差异和变化。
+- `aggregation`：聚合模式，适用于需要统一、全面视图的 IP 信息的情况，从多个源聚合数据，用一个数据库中的信息补充另一个数据库中缺失的字段。
+
 ### fields
 
 定义查询结果中显示哪些字段，字符串参数。默认值是 `country,province,city,isp`。该参数支持多种高级用法：

docs/config_en.md

@@ -17,6 +17,7 @@
     * [ipv4_format](#ipv4format)
     * [ipv6_file](#ipv6file)
     * [ipv6_format](#ipv6format)
+    * [hybrid_mode](#hybridmode)
     * [fields](#fields)
     * [use_db_fields](#usedbfields)
     * [rewrite_files](#rewritefiles)
@@ -169,6 +170,15 @@ Specifies the format of the IPv6 database file, a string parameter.
 
 When the database file suffix is insufficient to determine the file format, it is used to specify the format of the IPv6 database. Usually, it is not necessary to set this.
 
+### hybrid_mode
+
+Specifies the operational mode of the Hybrid Reader, a string parameter. The operational mode determines how data from multiple IP databases is processed and combined.
+
+Options are `comparison` and `aggregation`, with the default being `aggregation`.
+
+- `comparison`: Suitable for scenarios requiring data comparison across different IP databases. Outputs data from all integrated databases, facilitating the identification of discrepancies and variations between each source.
+- `aggregation`: Ideal for situations requiring a unified, comprehensive view of IP information. Aggregates data from multiple sources, supplementing missing fields from one database with information from another.
+
 ### fields
 
 Defines which fields to display in the query results, a string parameter. The default value is `country,province,city,isp`. This parameter supports various advanced usages:

docs/dump.md

@@ -30,6 +30,7 @@ ips dump -i inputFile [--input-format] [-o outputFile] [flags]
 - `-i, --input-file string`：指定输入 IP 数据库文件的路径。必填项。
 - `--input-format string`：指定输入 IP 数据库文件的格式。默认为自动检测。
 - `--input-option string`：数据库读取器指定选项。具体信息请查阅数据库文档。
+- `--hybrid-mode string`: 指定混合读取器的操作模式，可选值为 `comparison` 与 `aggregation`，参数详细解释请参考 [IPS 配置说明](./config.md#hybridmode)。
 - `-o, --output-file string`：指定转存文件的路径。不指定转存文件时，输出到标准输出流。
 - `--lang string`：设置输出信息的语言。默认为 `zh-CN` (中文)。
 - `-f, --fields string`：指定从输入文件中获取的字段。默认为所有字段。参数详细解释请参考 [IPS 配置说明](./config.md#fields)。

docs/dump_en.md

@@ -31,6 +31,7 @@ ips dump -i inputFile [--input-format] [-o outputFile] [flags]
 - `-i, --input-file string`：Specifies the path to the input IP database file. Required.
 - `--input-format string`：Specifies the format of the input IP database file. Default is auto-detection.
 - `--input-option string`：Specifies options for the database reader. For more information, refer to the database documentation.
+- `--hybrid-mode string`: Specifies the operational mode for the Hybrid Reader. Options are `comparison` and `aggregation`. For more details, refer to [IPS Configuration Documentation](./config_en.md#hybridmode).
 - `-o, --output-file string`：Specifies the path to the dump file. When not specified, outputs to the standard output stream.
 - `--lang string`：Sets the language for the output information. Default is `zh-CN` (Chinese).
 - `-f, --fields string`：Specifies the fields to be extracted from the input file. Default is all fields. For a detailed explanation of the parameter, refer to  [IPS Configuration Documentation](./config_en.md#fields)。

docs/pack.md

@@ -29,6 +29,7 @@ ips pack -i inputFile [--input-format format] -o outputFile [--output-format for
 - `-i, --input-file string`：指定输入 IP 数据库文件的路径。必填项。
 - `--input-format string`：指定输入 IP 数据库文件的格式。默认为自动检测。
 - `--input-option string`：数据库读取器指定选项。具体信息请查阅相关的数据库格式文档或获取专业支持。
+- `--hybrid-mode string`: 指定混合读取器的操作模式，可选值为 `comparison` 与 `aggregation`，参数详细解释请参考 [IPS 配置说明](./config.md#hybridmode)。
 - `-o, --output-file string`：指定输出 IP 数据库文件的路径。必填项。
 - `--output-format string`：指定输出 IP 数据库文件的格式。未指定时，使用输出文件的扩展名自动检测。
 - `--output-option string`：数据库写入器指定选项。具体信息请查阅相关的数据库格式文档或获取专业支持。

docs/pack_en.md

@@ -29,6 +29,7 @@ ips pack -i inputFile [--input-format format] -o outputFile [--output-format for
 - `-i, --input-file string`：Specifies the path to the input IP database file. required.
 - `--input-format string`：Specifies the format of the input IP database file. The default is auto-detection.
 - `--input-option string`：Specifies options for the database reader. For more information, please consult the relevant database format documentation or obtain professional support.
+- `--hybrid-mode string`: Specifies the operational mode for the Hybrid Reader. Options are `comparison` and `aggregation`. For more details, refer to [IPS Configuration Documentation](./config_en.md#hybridmode).
 - `-o, --output-file string`：Specifies the path to the output IP database file. required.
 - `--output-format string`：Specifies the format of the output IP database file. If not specified, the format is auto-detected based on the output file extension.
 - `--output-option string`：Specifies options for the database writer. For more information, please consult the relevant database format documentation or obtain professional support.

docs/query.md

@@ -42,6 +42,7 @@ echo <ip or text> | ips [flags]
 - `--ipv4-format string`：指定 IPv4 数据库文件的格式，需要与 `--ipv4-file` 配合使用。默认为自动检测。
 - `--ipv6-file string`：指定 IPv6 数据库文件的路径。
 - `--ipv6-format string`：指定 IPv6 数据库文件的格式，需要与 `--ipv6-file` 配合使用。默认为自动检测。
+- `--hybrid-mode string`: 指定混合读取器的操作模式，可选值为 `comparison` 与 `aggregation`，参数详细解释请参考 [IPS 配置说明](./config.md#hybridmode)。
 - `--text-format string`：指定文本输出的格式，支持 %origin 和 %values 参数。
 - `--text-values-sep string`：指定文本输出中值的分隔符，默认为空格。
 - `-j, --json bool`：以 JSON 格式输出结果。

docs/query_en.md

@@ -41,6 +41,7 @@ echo <ip or text> | ips [flags]
 - `--ipv4-format string`：Specifies the format for the IPv4 database file; used in conjunction with `--ipv4-file`. The default is auto-detection.
 - `--ipv6-file string`：Specifies the path to the IPv6 database file.
 - `--ipv6-format string`：Specifies the format for the IPv6 database file; used in conjunction with `--ipv6-file`. The default is auto-detection.
+- `--hybrid-mode string`: Specifies the operational mode for the Hybrid Reader. Options are `comparison` and `aggregation`. For more details, refer to [IPS Configuration Documentation](./config_en.md#hybridmode).
 - `--text-format string`：Specifies the format for text output, supporting `%origin` and `%values` parameters.
 - `--text-values-sep string`：Specifies the separator for values in text output, with the default being a space.
 - `-j, --json bool`：Outputs results in JSON format.