support mmdb and zxinc

README.md

@@ -1,6 +1,12 @@
 ## IPS
 
-ips is a tool for querying, scanning, and packing IP geolocation databases.
+ips is a command-line tool for querying, scanning, and packing IP geolocation database files.
+
+ips is support for multiple database protocols, including popular ones such as MaxMind GeoLite2 and ipip.net, which allows for easy integration with existing systems. In addition, it has flexible query methods that can be chosen based on specific requirements, improving query efficiency.
+
+Using ips, it is easy to query the geolocation information of an IP address, including country, city, and isp. In addition, it can scan the entire database, quickly extract records that meet specific conditions, and package them into a new database file for further processing.
+
+ips is a powerful and easy-to-use tool for IP geolocation, suitable for a wide range of applications that require handling of IP geolocation information.
 
 ### Install
 
@@ -10,16 +16,115 @@ go install github.com/sjzar/ips@latest
 
 ### Feature
 * IP Geolocation Databases Querying, Scanning, Packing
-* Multi Databases Support
+* Multiple Databases Support
 
 ### Databases Support Status
 
-| Database | Query | Scan | Pack | Official                   | Notes     |
-|:---------|:------|:-----|:-----|:---------------------------|:----------|
-| ipdb     | ✅    | ✅   | ✅    | https://ipip.net |           |
-| awdb     | ✅    | ✅   | -    | https://ipplus360.com      |           |
-| qqwry    | ✅    | ✅   | -    | https://cz88.net           | IPv4 only |
+| Database | Query | Scan | Pack | Official              | Comments  |
+|:---------|:------|:-----|:-----|:----------------------|:----------|
+| ipdb     | ✅    | ✅   | ✅    | https://ipip.net      |           |
+| awdb     | ✅    | ✅   | -    | https://ipplus360.com |           |
+| mmdb     | ✅    | ✅   | -    | https://maxmind.com   |           |
+| qqwry    | ✅    | ✅   | -    | https://cz88.net      | IPv4 only |
+| zxinc    | ✅    | ✅   | -    | https://ip.zxinc.org  | IPv6 only |
 
 
 ### Usage
 
+#### Query
+
+```shell
+ips [option] <ip or text>
+
+# Query IP
+ips 61.144.235.160
+61.144.235.160 [广东省深圳市 电信]
+
+# Qeury IP with pipeline
+echo "61.144.235.160" | ips
+61.144.235.160 [广东省深圳市 电信]
+
+# Query IP with database
+ips -d ./city.free.ipdb 61.144.235.160
+61.144.235.160 [中国 广东 深圳]
+
+# Query IP with database and fields
+ips -d ./city.free.ipdb --fields country,province 61.144.235.160
+61.144.235.160 [中国 广东]
+
+# Query IP with database and set format
+ips -d ./city.free.ipdb.rename --format ipdb 61.144.235.160
+61.144.235.160 [中国 广东 深圳]
+ips -d ipdb:./city.free.ipdb.rename 61.144.235.160
+61.144.235.160 [中国 广东 深圳]
+```
+
+#### Scan
+
+```shell
+# Scan database
+ips scan ./qqwry.dat
+# ScanTime: 2006-01-02 15:04:05
+# Fields: country,area
+# IPVersion: 1
+# Meta: {"IPVersion":1,"Fields":["country","area"]}
+0.0.0.0/8       IANA,保留地址
+1.0.0.0/32      美国,亚太互联网络信息中心(CloudFlare节点)
+1.0.0.1/32      美国,APNIC&CloudFlare公共DNS服务器
+1.0.0.2/31      美国,亚太互联网络信息中心(CloudFlare节点)
+<ignore more content>
+
+# Scan database with fields
+ips/ips scan qqwry.dat --fields country
+# ScanTime: 2006-01-02 15:04:05
+# Fields: country
+# IPVersion: 1
+# Meta: {"IPVersion":1,"Fields":["country"]}
+0.0.0.0/8       IANA
+1.0.0.0/24      美国
+1.0.1.0/24      福建省
+1.0.2.0/23      福建省
+1.0.4.0/22      澳大利亚
+<ignore more content>
+
+# Scan database with rewrite
+# rewrite file format: <field>\t<match>\t<replace>\n
+# make rewrite file like:
+# country\t美国\t美利坚合众国
+ips scan -r ./countrydemo.map qqwry.dat
+# ScanTime: 2022-12-10 21:17:01
+# Fields: country,area
+# IPVersion: 1
+# Meta: {"IPVersion":1,"Fields":["country","area"]}
+0.0.0.0/8       IANA,保留地址
+1.0.0.0/32      美利坚合众国,亚太互联网络信息中心(CloudFlare节点)
+1.0.0.1/32      美利坚合众国,APNIC&CloudFlare公共DNS服务器
+1.0.0.2/31      美利坚合众国,亚太互联网络信息中心(CloudFlare节点)
+<ignore more content>
+
+# Scan database and output to file
+ips scan qqwry.dat -o qqwry.ipscan
+ll
+-rw-r--r--  1 sarv  staff    10M 12 10 20:33 qqwry.dat
+-rw-r--r--  1 sarv  staff    48M 12 10 21:19 qqwry.ipscan
+```
+
+#### Pack
+
+```shell
+# Pack ipscan file
+ips pack qqwry.ipscan
+
+# Pack ipscan file and output to another file
+ips pack qqwry.ipscan -o demo1.ipdb
+
+# Pack database
+ips pack qqwry.dat --format qqwry
+
+# Pack database with fields
+ips pack qqwry.dat --format qqwry -f country
+
+# Pack database with rewrite
+ips pack qqwry.dat --format qqwry -r ./countrydemo.map
+```
+

cmd/ips/conf/config.go

@@ -37,8 +37,8 @@ func init() {
 	PrepareDir(ConfigPath)
 
 	// set default config
-	viper.SetDefault("ipv4_file", "city.free.ipdb")
-	viper.SetDefault("ipv6_file", "city.free.ipdb")
+	viper.SetDefault("ipv4_file", "qqwry.dat")
+	viper.SetDefault("ipv6_file", "zxipv6wry.db")
 	//viper.SetDefault("fields", []string{"country", "province", "city", "isp"})
 
 	// read config

cmd/ips/db.go

@@ -2,6 +2,7 @@ package ips
 
 import (
 	"log"
+	"os"
 	"strings"
 	"sync"
 
@@ -24,11 +25,26 @@ var (
 // GetIPv4 returns a ipio.Reader for IPv4
 func GetIPv4() ipio.Reader {
 	ipv4once.Do(func() {
-		database, err := db.NewDatabase(conf.Conf.IPv4Format, conf.ConfigPath+"/"+conf.Conf.IPv4File)
+		format := rootDBFormat
+		if len(format) == 0 {
+			format = conf.Conf.IPv4Format
+		}
+		file := rootDBFile
+		if len(file) == 0 {
+			file = conf.ConfigPath + "/" + conf.Conf.IPv4File
+		}
+		database, err := db.NewDatabase(format, file)
 		if err != nil {
-			log.Fatal(err)
+			log.Println("read database failed", file, err)
+			if file == conf.ConfigPath+"/"+conf.Conf.IPv4File {
+				log.Println("use ips update first")
+			}
+			os.Exit(1)
 		}
 		fields := conf.Conf.Fields
+		if len(rootFields) != 0 {
+			fields = strings.Split(rootFields, ",")
+		}
 		if len(fields) == 0 {
 			fields = database.Meta().Fields
 		}
@@ -42,11 +58,26 @@ func GetIPv4() ipio.Reader {
 // GetIPv6 returns a ipio.Reader for IPv6
 func GetIPv6() ipio.Reader {
 	ipv6once.Do(func() {
-		database, err := db.NewDatabase(conf.Conf.IPv6Format, conf.ConfigPath+"/"+conf.Conf.IPv6File)
+		format := rootDBFormat
+		if len(format) == 0 {
+			format = conf.Conf.IPv4Format
+		}
+		file := rootDBFile
+		if len(file) == 0 {
+			file = conf.ConfigPath + "/" + conf.Conf.IPv6File
+		}
+		database, err := db.NewDatabase(format, file)
 		if err != nil {
-			log.Fatal("read database failed ", conf.Conf.IPv4File, err)
+			log.Println("read database failed", file, err)
+			if file == conf.ConfigPath+"/"+conf.Conf.IPv6File {
+				log.Println("use ips update first")
+			}
+			os.Exit(1)
 		}
 		fields := conf.Conf.Fields
+		if len(rootFields) != 0 {
+			fields = strings.Split(rootFields, ",")
+		}
 		if len(fields) == 0 {
 			fields = database.Meta().Fields
 		}

cmd/ips/root.go

@@ -13,6 +13,18 @@ import (
 	"github.com/sjzar/ips/parser"
 )
 
+var (
+	rootDBFormat string
+	rootDBFile   string
+	rootFields   string
+)
+
+func init() {
+	rootCmd.Flags().StringVarP(&rootDBFormat, "format", "", "", "database format")
+	rootCmd.Flags().StringVarP(&rootDBFile, "database", "d", "", "database file")
+	rootCmd.Flags().StringVarP(&rootFields, "fields", "f", "", "fields")
+}
+
 var rootCmd = &cobra.Command{
 	Use:   "ips",
 	Short: "ips commandline tools",

cmd/ips/scan.go

@@ -72,7 +72,7 @@ func Scan(cmd *cobra.Command, args []string) {
 		split := strings.Split(scanRewriteFiles, ",")
 		for i := range split {
 			if err := rw.DataLoader.LoadFile(split[i]); err != nil {
-				log.Println("load rewrite file failed ", err)
+				log.Fatal("load rewrite file failed ", err)
 			}
 		}
 	}

cmd/ips/update.go

@@ -12,6 +12,16 @@ package ips
 // Updated: daily update
 // https://99wry.cf/qqwry.dat
 
+// MaxMind GeoLite2-City
+// Updated: 2022-12-07
+// https://git.io/GeoLite2-City.mmdb
+// https://github.com/P3TERX/GeoLite.mmdb/releases/latest/download/GeoLite2-City.mmdb
+
+// ZXINC
+// Updated: 2021-05-11
+// https://ip.zxinc.org/ip.7z
+// https://raw.githubusercontent.com/ZX-Inc/zxipdb-python/main/data/ipv6wry.db
+
 import (
 	"io"
 	"log"
@@ -36,6 +46,9 @@ var updateCmd = &cobra.Command{
 func Update(cmd *cobra.Command, args []string) {
 	Download("city.free.ipdb", "https://raw.githubusercontent.com/ipipdotnet/ipdb-go/master/city.free.ipdb")
 	Download("qqwry.dat", "https://99wry.cf/qqwry.dat")
+	Download("zxipv6wry.db", "https://raw.githubusercontent.com/ZX-Inc/zxipdb-python/main/data/ipv6wry.db")
+
+	//Download("GeoLite2-City.mmdb", "https://git.io/GeoLite2-City.mmdb")
 }
 
 func Download(file, url string) {

db/awdb/awdb.go

@@ -27,7 +27,8 @@ func New(file string) (*Database, error) {
 	}
 	if db.Metadata.IPVersion == 4 {
 		meta.IPVersion |= model.IPv4
-	} else if db.Metadata.IPVersion == 6 {
+	}
+	if db.Metadata.IPVersion == 6 {
 		meta.IPVersion |= model.IPv6
 	}
 
@@ -45,8 +46,8 @@ func (d *Database) Meta() model.Meta {
 // Find 查询 IP 对应的网段和结果
 func (d *Database) Find(ip net.IP) (*ipx.Range, map[string]string, error) {
 	var record interface{}
-	ipNet, ok, err := d.db.LookupNetwork(ip, &record)
-	if !ok || err != nil {
+	ipNet, _, err := d.db.LookupNetwork(ip, &record)
+	if err != nil {
 		return nil, nil, err
 	}
 

db/ipdb/writer.go

@@ -203,7 +203,9 @@ func (p *Writer) Nodes(ip net.IP, mask int) (node, index int, ok bool) {
 	// 统一扩展为IPv6的子网掩码进行处理
 	maxMask := mask - 1
 	if ip.To4() != nil {
-		maxMask += 96
+		if maxMask < 32 {
+			maxMask += 96
+		}
 		if len(ip) == net.IPv4len {
 			ip = ip.To16()
 		}