diff --git a/README.md b/README.md index 027b98a..f102f40 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ #### Features: -* Downloads latest archive `GeoLite2-(Country|City)-CSV_%Date%.zip` using MaxMind API https://www.maxmind.com/ by [free license key](https://support.maxmind.com/account-faq/license-keys/how-do-i-generate-a-license-key/) +* Downloads latest archive `GeoLite2-(Country|City)-CSV_%Date%.zip` using MaxMind API https://www.maxmind.com/ by [free license key](https://support.maxmind.com/hc/en-us/articles/4407111582235-Generate-a-License-Key) * Converts files `GeoLite2-(Country|City)-Locations-%LocaleCode%.csv, GeoLite2-(Country|City)-Blocks-(IPv4|IPv6).csv` to SQL DDL/DML * Builds archive `maxmind-geoip2-csv2sql.zip` with scripts splitted per N megabytes, using template engine (Example: If MySQL server `max_allowed_packet` parameter is too small - [64 MB by default](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_allowed_packet)) diff --git a/build.gradle.kts b/build.gradle.kts index 9b977f8..aefe1dd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,7 @@ import org.gradle.api.JavaVersion.VERSION_11 group = "com.github.mbto.maxmind-geoip2-csv2sql-converter" -version = "1.0" +version = "1.1" plugins { java @@ -19,9 +19,6 @@ tasks { exclude("*.ini", "emoji.txt") } test { - if(!project.hasProperty("ManualTestEnabled")) { - exclude("**/ManualTest.class") - } maxParallelForks = Runtime.getRuntime().availableProcessors() } } @@ -37,7 +34,10 @@ dependencies { testCompile("org.projectlombok:lombok:$lombokVer") testAnnotationProcessor("org.projectlombok:lombok:$lombokVer") - testCompile("junit:junit:4.13.2") + val jUnitVer = "5.9.3" + testCompile("org.junit.jupiter:junit-jupiter-engine:$jUnitVer") + testCompile("org.junit.jupiter:junit-jupiter-params:$jUnitVer") + testCompile("org.junit.vintage:junit-vintage-engine:$jUnitVer") } application { diff --git a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/Args.java b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/Args.java index 99d405a..e031c95 100644 --- a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/Args.java +++ b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/Args.java @@ -2,8 +2,6 @@ import com.beust.jcommander.DynamicParameter; import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParametersDelegate; -import com.beust.jcommander.SubParameter; import com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils; import com.github.mbto.maxmind.geoip2.csv2sql.utils.jcommander.IPVersionValidator; import com.github.mbto.maxmind.geoip2.csv2sql.utils.jcommander.LocaleConverter; @@ -13,7 +11,10 @@ import lombok.ToString; import java.nio.file.Path; -import java.util.*; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static com.github.mbto.maxmind.geoip2.csv2sql.utils.Constants.CONFIG_NAME_FORMAT; @@ -38,7 +39,7 @@ public class Args { private String outputArchiveName = DEFAULT_ARCHIVE_NAME; @Parameter(names = "-k", order = 4, - description = "License key for MaxMind API. Free at https://support.maxmind.com/account-faq/license-keys/how-do-i-generate-a-license-key/") + description = "License key for MaxMind API. Free at https://support.maxmind.com/hc/en-us/articles/4407111582235-Generate-a-License-Key") private String licenseKey; @Parameter(names = "-c", order = 5, required = true, @@ -63,10 +64,17 @@ public class Args { @DynamicParameter(names = "-LV", order = 8, description = "Filter values from location files by group name with regex:\n" + " Example - for both GeoLite2-Country-CSV and GeoLite2-City-CSV editions:\n" + - " -LVcontinent_code=EU,NA,OC -LVcountry_iso_code=AU,NZ,GB,IE,US,CA,CY\n" + + " -LVgeoname_id=.*777.* -LVlocale_code=en,ru,de,es,fr\n" + + " -LVcontinent_code=EU,NA,OC -LVcontinent_name=Europe|Africa,Asia\n" + + " -LVcountry_iso_code=AU,NZ,GB,IE,US,CA,CY\n" + " -LVcountry_name=Austr.*,Zealand$,^United,Ireland,Canada|Cyprus\n" + + " -LVis_in_european_union=0|1\n" + " At GeoLite2-City-CSV edition available filter by city_name and other group names:\n" + - " -LVcity_name=Newport,^Clinton$|^Richmond$,\"Mandria, Paphos\",^Salem") + " -LVsubdivision_1_iso_code=WO|JD|NU|GE|A.* -LVsubdivision_1_name=.*O.*\n" + + " -LVsubdivision_2_iso_code=.* -LVsubdivision_2_name=.*A.*\n" + + " -LVcity_name=Newport,^Clinton$|^Richmond$,\"Mandria, Paphos\",^Salem\n" + + " -LVmetro_code=.* -LVtime_zone=.*/.*E.*" + ) private Map allowedLocationValuesRawByGroupName = new LinkedHashMap<>(); @Parameter(names = "-mm", order = 9, diff --git a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/AbstractConverter.java b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/AbstractConverter.java index 959fe48..1b8343a 100644 --- a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/AbstractConverter.java +++ b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/AbstractConverter.java @@ -5,35 +5,37 @@ import java.util.concurrent.Callable; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; import static com.github.mbto.maxmind.geoip2.csv2sql.streaming.Event.TERMINATE; -import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.*; +import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.calcHumanDiff; +import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.threadPrintln; public abstract class AbstractConverter implements Callable { protected final Registry registry; protected final String dataType; protected final LinkedBlockingQueue> messageQueue; + protected final boolean logIgnored; protected Thread writerT; - public AbstractConverter(Registry registry, String dataType, int queueCapacity) { + public AbstractConverter(Registry registry, String dataType, int queueCapacity, boolean logIgnored) { this.registry = registry; this.dataType = dataType; this.messageQueue = new LinkedBlockingQueue<>(queueCapacity); + this.logIgnored = logIgnored; } @Override public Void call() throws Exception { long startEpoch = System.currentTimeMillis(); - threadPrintln(System.out, "Started '" + dataType + " converter'"); + threadPrintln(System.out, "Started " + getConverterName()); try { return work(); } catch (Throwable e) { - throw new Exception("Exception in '" + dataType + " converter'", e); + throw new Exception("Exception in " + getConverterName(), e); } finally { terminateWriter(); - threadPrintln(System.out, "Finished '" + dataType + " converter' in " + calcHumanDiff(startEpoch)); + threadPrintln(System.out, "Finished " + getConverterName() + " in " + calcHumanDiff(startEpoch)); } } @@ -41,12 +43,16 @@ public Void call() throws Exception { protected void terminateWriter() { if (writerT != null && writerT.isAlive()) { - threadPrintln(System.out, "Waiting for terminate writer from '" + dataType + " converter'"); + threadPrintln(System.out, "Waiting for terminate writer from " + getConverterName()); try { messageQueue.put(new Message<>(null, TERMINATE)); - writerT.join(/*TimeUnit.SECONDS.toMillis(3)*/); + writerT.join(/*java.util.concurrent.TimeUnit.SECONDS.toMillis(3)*/); } catch (InterruptedException ignored) {} - threadPrintln(System.out, "Terminated writer from '" + dataType + " converter'"); + threadPrintln(System.out, "Terminated writer from " + getConverterName()); } } + + protected String getConverterName() { + return "'" + dataType + " converter'"; + } } \ No newline at end of file diff --git a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/IPBlockConverter.java b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/IPBlockConverter.java index e2894e0..0cff1a1 100644 --- a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/IPBlockConverter.java +++ b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/IPBlockConverter.java @@ -3,7 +3,6 @@ import com.github.jgonian.ipmath.AbstractIpRange; import com.github.jgonian.ipmath.Ipv4Range; import com.github.jgonian.ipmath.Ipv6Range; -import com.github.jgonian.ipmath.Range; import com.github.mbto.maxmind.geoip2.csv2sql.Registry; import com.github.mbto.maxmind.geoip2.csv2sql.streaming.BRWrapper; import com.github.mbto.maxmind.geoip2.csv2sql.streaming.Location.IPBlock; @@ -21,7 +20,8 @@ import static com.github.mbto.maxmind.geoip2.csv2sql.streaming.Event.WRITE; import static com.github.mbto.maxmind.geoip2.csv2sql.streaming.converters.LocationsConverter.geonameIdsWithEmptyCountryIsoCode; import static com.github.mbto.maxmind.geoip2.csv2sql.streaming.converters.LocationsConverter.ignoredGeonameIds; -import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.*; +import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.extractIPBlockFilename; +import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.threadPrintln; import static com.github.mbto.maxmind.geoip2.csv2sql.utils.placeholder.ParseUtils.StringUtils.split2; /** @@ -29,12 +29,14 @@ */ public class IPBlockConverter extends AbstractConverter { private final String[] priorityGeonameIdGroupNames; + private final boolean logUndefinedAllGeonameIds; public IPBlockConverter(Registry registry, String dataType, int queueCapacity) { - super(registry, dataType, queueCapacity); + super(registry, dataType, queueCapacity, + Boolean.parseBoolean(registry.getFromExportSection("log_ignored_ipblocks", true))); - String priorityGeonameIdGroupNamesRaw = registry.getFromExportSection("ipblocks_priority_geonameId_groupNames", true); - if(!priorityGeonameIdGroupNamesRaw.isEmpty()) { + String priorityGeonameIdGroupNamesRaw = registry.getFromExportSection("ipblocks_priority_geonameId_groupNames", false); + if(priorityGeonameIdGroupNamesRaw != null && !priorityGeonameIdGroupNamesRaw.isEmpty()) { String[] priorityGeonameIdGroupNames = split2(priorityGeonameIdGroupNamesRaw, ',', true, true); if(priorityGeonameIdGroupNames.length != 0) this.priorityGeonameIdGroupNames = priorityGeonameIdGroupNames; @@ -42,6 +44,7 @@ public IPBlockConverter(Registry registry, String dataType, int queueCapacity) { this.priorityGeonameIdGroupNames = null; } else this.priorityGeonameIdGroupNames = null; + this.logUndefinedAllGeonameIds = Boolean.parseBoolean(registry.getFromExportSection("log_undefined_all_geonameIds", true)); } @Override @@ -81,12 +84,25 @@ public Void work() throws Throwable { ++geonameIdsCounter; } if(geonameIdsCounter == 0) { - registry.incStats(sif.getDataType() /*== dataType*/ + "_ignored"); + if(logIgnored) { + threadPrintln(System.out, "Ignored '" + dataType/*== sif.getDataType()*/ + "' in " + getConverterName() + " by filter from line '" + line + "'"); + } + registry.incStats(dataType + " ignored"); continue; } } IPBlock ipBlock = new IPBlock(csvHolder, parseCidrFunc); - ipBlock.setPriorityGeonameId(findPriorityGeonameId(csvHolder)); + Integer priorityGeonameId; + try { + priorityGeonameId = findPriorityGeonameId(csvHolder); + } catch (Throwable e) { + if(logUndefinedAllGeonameIds) { + threadPrintln(System.out, "Ignored '" + dataType + "' in " + getConverterName() + ", due " + e.getMessage() + " from line '" + line + "'"); + } + registry.incStats(dataType + " ignored"); + continue; + } + ipBlock.setPriorityGeonameId(priorityGeonameId); for (Map.Entry entry : ipBlock.getValues().entrySet()) { String value = entry.getValue(); if (value == null) @@ -130,7 +146,6 @@ private Integer findPriorityGeonameId(CsvHolder csvHolder) { if(firstGeonameId != null) return firstGeonameId; throw new IllegalStateException("Failed to determine priority geoname_id by group names " - + Arrays.toString(priorityGeonameIdGroupNames) - + " from " + csvHolder.getValueByGroupName()); + + Arrays.toString(priorityGeonameIdGroupNames)); } } \ No newline at end of file diff --git a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/LocationsConverter.java b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/LocationsConverter.java index 1f6addd..c617a69 100644 --- a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/LocationsConverter.java +++ b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/streaming/converters/LocationsConverter.java @@ -1,7 +1,7 @@ package com.github.mbto.maxmind.geoip2.csv2sql.streaming.converters; -import com.github.mbto.maxmind.geoip2.csv2sql.Registry; import com.github.mbto.maxmind.geoip2.csv2sql.Args.Locale; +import com.github.mbto.maxmind.geoip2.csv2sql.Registry; import com.github.mbto.maxmind.geoip2.csv2sql.streaming.BRWrapper; import com.github.mbto.maxmind.geoip2.csv2sql.streaming.Location; import com.github.mbto.maxmind.geoip2.csv2sql.streaming.Location.LocationData; @@ -16,7 +16,8 @@ import java.util.regex.Pattern; import static com.github.mbto.maxmind.geoip2.csv2sql.streaming.Event.WRITE; -import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.*; +import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.extractLocationsFilenames; +import static com.github.mbto.maxmind.geoip2.csv2sql.utils.ProjectUtils.threadPrintln; /** * 1 thread reads GeoLite2-(Country|City)-Locations-XXXX.csv files and fill messageQueue @@ -24,9 +25,13 @@ public class LocationsConverter extends AbstractConverter { public static final Set geonameIdsWithEmptyCountryIsoCode = new HashSet<>(); public static final Set ignoredGeonameIds = new HashSet<>(); + private final boolean logUndefinedAllSubdivisionsAndCityName; public LocationsConverter(Registry registry, int queueCapacity) { - super(registry, Location.class.getSimpleName().toLowerCase(), queueCapacity); + super(registry, Location.class.getSimpleName().toLowerCase(), queueCapacity, + Boolean.parseBoolean(registry.getFromExportSection("log_ignored_locations", true))); + this.logUndefinedAllSubdivisionsAndCityName = Boolean.parseBoolean(registry.getFromExportSection("log_undefined_all_subdivisions_and_city_name", true)); + // multiple gradle tests not clean static variables geonameIdsWithEmptyCountryIsoCode.clear(); ignoredGeonameIds.clear(); @@ -56,6 +61,7 @@ public Void work() throws Throwable { if (csvHolder == null) throw new IllegalStateException("Unable to define csvHolder, due empty locations_filenames template"); Map> allowedLocationValuesByGroupName = registry.getAllowedLocationValuesByGroupName(); boolean isCityEdition = csvHolder.getHeaders().contains("city_name"); + String dataTypeLabel = !isCityEdition ? "country" : "city"; Writer writer = new Writer(registry, dataType, messageQueue); writerT = new Thread(new FutureTask<>(writer)); writerT.start(); @@ -84,7 +90,13 @@ public Void work() throws Throwable { })) { int geoname_id = Integer.parseInt(location.getValues().get("geoname_id")); ignoredGeonameIds.add(geoname_id); - registry.incStats((!isCityEdition ? "country" : "city") + "_ignored"); + if(logIgnored) { + threadPrintln(System.out, "Ignored '" + dataTypeLabel + "' in " + getConverterName() + + " by filter '" + filteredGroupName + "'" + + " only " + allowedLocationValuePatterns.toString() + + " from " + csvHolder.getValueByGroupName()); + } + registry.incStats(dataTypeLabel + " ignored"); continue outer; } } @@ -117,6 +129,8 @@ public Void work() throws Throwable { if (syntheticKey == null) { // geoname_id with 6255147 6255148 with empty country_iso_code syntheticKey = locationValues.get("geoname_id"); geonameIdsWithEmptyCountryIsoCode.add(Integer.parseInt(syntheticKey)); + threadPrintln(System.out, "Informing: '" + sif.getDataType() + "' in " + getConverterName() + + " without country_iso_code from " + csvHolder.getValueByGroupName()); registry.incStats(sif.getDataType() + " includes which unknown"); } @@ -185,6 +199,10 @@ public Void work() throws Throwable { } return false; }).count() == 5)) { + if(logUndefinedAllSubdivisionsAndCityName) { + threadPrintln(System.out, "Informing: '" + sif.getDataType() + "' in " + getConverterName() + + " without subdivision_* and city_name from " + csvHolder.getValueByGroupName()); + } registry.incStats(sif.getDataType() + " includes which unknown"); } } diff --git a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/Constants.java b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/Constants.java index 5c7eb56..a86f3ab 100644 --- a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/Constants.java +++ b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/Constants.java @@ -8,7 +8,7 @@ public interface Constants { String SOFTWARE_NAME = "maxmind-geoip2-csv2sql-converter"; - String SOFTWARE_INFO = "MaxMind GeoIP2 csv2sql Converter v1.0\nhttps://github.com/mbto/" + SOFTWARE_NAME; + String SOFTWARE_INFO = "MaxMind GeoIP2 csv2sql Converter v1.1\nhttps://github.com/mbto/" + SOFTWARE_NAME; Set supportedLocales = new LinkedHashSet<>( asList("en", "ru", "de", "es", "fr", "ja", "pt-BR", "zh-CN")); diff --git a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/ProjectUtils.java b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/ProjectUtils.java index 07bc01d..821d0c8 100644 --- a/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/ProjectUtils.java +++ b/src/main/java/com/github/mbto/maxmind/geoip2/csv2sql/utils/ProjectUtils.java @@ -5,7 +5,6 @@ import com.github.mbto.maxmind.geoip2.csv2sql.utils.placeholder.Template; import java.io.PrintStream; -import java.net.URL; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -35,16 +34,16 @@ public abstract class ProjectUtils { public static Path resolveConfigDirectory() { String defaultURI = null; try { - defaultURI = URLDecoder.decode(ProjectUtils.class.getResource("").toString(), StandardCharsets.UTF_8); // R:\\test%20test\\ -> R:\\test test\\ + defaultURI = URLDecoder.decode(ProjectUtils.class.getResource("").toString(), StandardCharsets.UTF_8); // C:\\test%20test\\ -> C:\\test test\\ if (defaultURI.startsWith("file:/")) { // file:/C:/idea/maxmind-geoip2-csv2sql-converter/build/classes/java/main/com/github/mbto/maxmind/geoip2/csv2sql/utils/ return Paths.get("").resolve("src").resolve("main").resolve("resources").toAbsolutePath(); } else { -// jar:file:/C:/idea/maxmind-geoip2-csv2sql-converter/build/distributions/maxmind-geoip2-csv2sql-converter-1.0/lib/maxmind-geoip2-csv2sql-converter-1.0.jar!/com/github/mbto/maxmind/geoip2/csv2sql/utils/ +// jar:file:/C:/idea/maxmind-geoip2-csv2sql-converter/build/distributions/maxmind-geoip2-csv2sql-converter-1.1/lib/maxmind-geoip2-csv2sql-converter-1.1.jar!/com/github/mbto/maxmind/geoip2/csv2sql/utils/ String jarPrefix = "jar:file:/"; if (defaultURI.startsWith(jarPrefix)) { -// C:/idea/maxmind-geoip2-csv2sql-converter/build/distributions/maxmind-geoip2-csv2sql-converter-1.0/lib/maxmind-geoip2-csv2sql-converter-1.0.jar +// C:/idea/maxmind-geoip2-csv2sql-converter/build/distributions/maxmind-geoip2-csv2sql-converter-1.1/lib/maxmind-geoip2-csv2sql-converter-1.1.jar String substring = defaultURI.substring(jarPrefix.length(), defaultURI.lastIndexOf('!')); if(!substring.contains(":")) substring = "/" + substring; diff --git a/src/main/resources/GeoLite2-City-CSV.mssql 2019.default.ini b/src/main/resources/GeoLite2-City-CSV.mssql 2019.default.ini index 4e17cd6..aa896fd 100644 --- a/src/main/resources/GeoLite2-City-CSV.mssql 2019.default.ini +++ b/src/main/resources/GeoLite2-City-CSV.mssql 2019.default.ini @@ -41,6 +41,10 @@ ipv6_insert_filename=ipv6_%03d.sql ipv6_indexes_filename=ipv6_indexes.sql // One of valid geoname_id for ipv4_values/ipv6_values templates from Blocks-IPv4/Blocks-IPv6 files (${.priorityGeonameId} placeholder); ipblocks_priority_geonameId_groupNames=geoname_id,registered_country_geoname_id,represented_country_geoname_id +log_ignored_locations=false +log_ignored_ipblocks=false +log_undefined_all_geonameIds=true +log_undefined_all_subdivisions_and_city_name=false schema_name=maxmind_city load_data_filename=load_data.sql diff --git a/src/main/resources/GeoLite2-City-CSV.mysql.default.ini b/src/main/resources/GeoLite2-City-CSV.mysql.default.ini index 9c1b4e8..9f7b469 100644 --- a/src/main/resources/GeoLite2-City-CSV.mysql.default.ini +++ b/src/main/resources/GeoLite2-City-CSV.mysql.default.ini @@ -40,6 +40,10 @@ ipv6_insert_filename=ipv6_%03d.sql ipv6_indexes_filename=ipv6_indexes.sql // One of valid geoname_id for ipv4_values/ipv6_values templates from Blocks-IPv4/Blocks-IPv6 files (${.priorityGeonameId} placeholder); ipblocks_priority_geonameId_groupNames=geoname_id,registered_country_geoname_id,represented_country_geoname_id +log_ignored_locations=false +log_ignored_ipblocks=false +log_undefined_all_geonameIds=true +log_undefined_all_subdivisions_and_city_name=false schema_name=maxmind_city load_data_filename=load_data.sql diff --git a/src/main/resources/GeoLite2-City-CSV.postgresql.default.ini b/src/main/resources/GeoLite2-City-CSV.postgresql.default.ini index aa6e4ea..42a191a 100644 --- a/src/main/resources/GeoLite2-City-CSV.postgresql.default.ini +++ b/src/main/resources/GeoLite2-City-CSV.postgresql.default.ini @@ -40,6 +40,10 @@ ipv6_insert_filename=ipv6_%03d.sql ipv6_indexes_filename=ipv6_indexes.sql // One of valid geoname_id for ipv4_values/ipv6_values templates from Blocks-IPv4/Blocks-IPv6 files (${.priorityGeonameId} placeholder); ipblocks_priority_geonameId_groupNames=geoname_id,registered_country_geoname_id,represented_country_geoname_id +log_ignored_locations=false +log_ignored_ipblocks=false +log_undefined_all_geonameIds=true +log_undefined_all_subdivisions_and_city_name=false schema_name=maxmind_city load_data_filename=load_data.sql diff --git a/src/main/resources/GeoLite2-Country-CSV.mssql 2019.default.ini b/src/main/resources/GeoLite2-Country-CSV.mssql 2019.default.ini index eadb761..7c4d22a 100644 --- a/src/main/resources/GeoLite2-Country-CSV.mssql 2019.default.ini +++ b/src/main/resources/GeoLite2-Country-CSV.mssql 2019.default.ini @@ -29,6 +29,10 @@ ipv6_insert_filename=ipv6_%03d.sql ipv6_indexes_filename=ipv6_indexes.sql // One of valid geoname_id for ipv4_values/ipv6_values templates from Blocks-IPv4/Blocks-IPv6 files (${.priorityGeonameId} placeholder); ipblocks_priority_geonameId_groupNames=geoname_id,registered_country_geoname_id,represented_country_geoname_id +log_ignored_locations=false +log_ignored_ipblocks=false +log_undefined_all_geonameIds=true +log_undefined_all_subdivisions_and_city_name=false schema_name=maxmind_country load_data_filename=load_data.sql diff --git a/src/main/resources/GeoLite2-Country-CSV.mysql.default.ini b/src/main/resources/GeoLite2-Country-CSV.mysql.default.ini index 84e5843..b63b188 100644 --- a/src/main/resources/GeoLite2-Country-CSV.mysql.default.ini +++ b/src/main/resources/GeoLite2-Country-CSV.mysql.default.ini @@ -28,6 +28,10 @@ ipv6_insert_filename=ipv6_%03d.sql ipv6_indexes_filename=ipv6_indexes.sql // One of valid geoname_id for ipv4_values/ipv6_values templates from Blocks-IPv4/Blocks-IPv6 files (${.priorityGeonameId} placeholder); ipblocks_priority_geonameId_groupNames=geoname_id,registered_country_geoname_id,represented_country_geoname_id +log_ignored_locations=false +log_ignored_ipblocks=false +log_undefined_all_geonameIds=true +log_undefined_all_subdivisions_and_city_name=false schema_name=maxmind_country load_data_filename=load_data.sql diff --git a/src/main/resources/GeoLite2-Country-CSV.postgresql.default.ini b/src/main/resources/GeoLite2-Country-CSV.postgresql.default.ini index 58560bd..7e2863d 100644 --- a/src/main/resources/GeoLite2-Country-CSV.postgresql.default.ini +++ b/src/main/resources/GeoLite2-Country-CSV.postgresql.default.ini @@ -28,6 +28,10 @@ ipv6_insert_filename=ipv6_%03d.sql ipv6_indexes_filename=ipv6_indexes.sql // One of valid geoname_id for ipv4_values/ipv6_values templates from Blocks-IPv4/Blocks-IPv6 files (${.priorityGeonameId} placeholder); ipblocks_priority_geonameId_groupNames=geoname_id,registered_country_geoname_id,represented_country_geoname_id +log_ignored_locations=false +log_ignored_ipblocks=false +log_undefined_all_geonameIds=true +log_undefined_all_subdivisions_and_city_name=false schema_name=maxmind_country load_data_filename=load_data.sql diff --git a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CityTest.java b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CityTest.java index fbdfa81..721d4a2 100644 --- a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CityTest.java +++ b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CityTest.java @@ -11,6 +11,7 @@ import java.util.UUID; import static com.github.mbto.maxmind.geoip2.csv2sql.Constants.assertStats; +import static java.util.Map.entry; import static org.junit.runners.Parameterized.Parameters; /** @@ -33,161 +34,177 @@ public static Collection data() { {new String[]{ "-i", "4", "-mm", "32", - }, Map.of( - "city", 121723, - "city includes which unknown", 249, - "country", 252, - "country includes which unknown", 2, - "ipv4", 3566870, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 106689207}, + }, Map.ofEntries( + entry("city", 114790), + entry("city includes which unknown", 249), + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 3757625), + entry("ipv4 ignored", 30), + entry("subdivision1", 3379), + entry("subdivision2", 1082), + entry("timezone", 400)), + 110337707}, {new String[]{ "-i", "4", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv4", 3566870, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 106680973}, + "ipv4", 3757625, + "ipv4 ignored", 30, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 110335462}, {new String[]{ "-i", "4", "-l", "en,ru", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv4", 3566870, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 106080700}, + "ipv4", 3757625, + "ipv4 ignored", 30, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 109746077}, {new String[]{ "-i", "4", "-l", "en", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv4", 3566870, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 105805193}, + "ipv4", 3757625, + "ipv4 ignored", 30, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 109481777}, {new String[]{ "-i", "6", "-mm", "32", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 19317782}, + "ipv6", 1254770, + "ipv6 ignored", 2, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 58942598}, {new String[]{ "-i", "6", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 19315938}, + "ipv6", 1254770, + "ipv6 ignored", 2, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 58937011}, {new String[]{ "-i", "6", "-l", "en,ru", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 18715665}, + "ipv6", 1254770, + "ipv6 ignored", 2, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 58347626}, {new String[]{ "-i", "6", "-l", "en", }, Map.of( - "city", 121723, + "city", 114790, "city includes which unknown", 249, "country", 252, "country includes which unknown", 2, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 18440158}, + "ipv6", 1254770, + "ipv6 ignored", 2, + "subdivision1", 3379, + "subdivision2", 1082, + "timezone", 400), + 58083326}, {new String[]{ "-i", "4,6", "-mm", "32", - }, Map.of( - "city", 121723, - "city includes which unknown", 249, - "country", 252, - "country includes which unknown", 2, - "ipv4", 3566870, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 123781884}, + }, Map.ofEntries( + entry("city", 114790), + entry("city includes which unknown", 249), + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 3757625), + entry("ipv4 ignored", 30), + entry("ipv6", 1254770), + entry("ipv6 ignored", 2), + entry("subdivision1", 3379), + entry("subdivision2", 1082), + entry("timezone", 400)), + 167150785}, {new String[]{ "-i", "4,6", - }, Map.of( - "city", 121723, - "city includes which unknown", 249, - "country", 252, - "country includes which unknown", 2, - "ipv4", 3566870, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 123771806}, + }, Map.ofEntries( + entry("city", 114790), + entry("city includes which unknown", 249), + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 3757625), + entry("ipv4 ignored", 30), + entry("ipv6", 1254770), + entry("ipv6 ignored", 2), + entry("subdivision1", 3379), + entry("subdivision2", 1082), + entry("timezone", 400)), + 167142948}, {new String[]{ "-i", "4,6", "-l", "en,ru", - }, Map.of( - "city", 121723, - "city includes which unknown", 249, - "country", 252, - "country includes which unknown", 2, - "ipv4", 3566870, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 123171533}, + }, Map.ofEntries( + entry("city", 114790), + entry("city includes which unknown", 249), + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 3757625), + entry("ipv4 ignored", 30), + entry("ipv6", 1254770), + entry("ipv6 ignored", 2), + entry("subdivision1", 3379), + entry("subdivision2", 1082), + entry("timezone", 400)), + 166553563}, {new String[]{ "-i", "4,6", "-l", "en", - }, Map.of( - "city", 121723, - "city includes which unknown", 249, - "country", 252, - "country includes which unknown", 2, - "ipv4", 3566870, - "ipv6", 355990, - "subdivision1", 3444, - "subdivision2", 1094, - "timezone", 408), - 122896026}, + }, Map.ofEntries( + entry("city", 114790), + entry("city includes which unknown", 249), + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 3757625), + entry("ipv4 ignored", 30), + entry("ipv6", 1254770), + entry("ipv6 ignored", 2), + entry("subdivision1", 3379), + entry("subdivision2", 1082), + entry("timezone", 400)), + 166289263}, }); } diff --git a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/Constants.java b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/Constants.java index 28a66d5..1cbc3d3 100644 --- a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/Constants.java +++ b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/Constants.java @@ -14,19 +14,22 @@ import static com.github.mbto.maxmind.geoip2.csv2sql.utils.Constants.SOFTWARE_NAME; public interface Constants { - String archiveDate = "20210629"; + String archiveDate = "20230616"; String archiveExtension = ".zip"; String resultArchiveName = "maxmind-geoip2-csv2sql-integration-test-result.zip"; Path integrationTestsDirPath = buildintegrationTestsDirPath(); Path resourceTestsDirPath = Paths.get("build", "resources", "test").toAbsolutePath(); static Path buildintegrationTestsDirPath() { - Path expectedPath = Paths.get("R:\\IntegrationTests").resolve(SOFTWARE_NAME); // RAM disk + Path expectedPath = Paths.get("C:\\") + .resolve("test test") + .resolve("IntegrationTests") + .resolve(SOFTWARE_NAME); Path root = expectedPath; while (true) { Path parent = root.getParent(); if(parent == null) { - return Files.isDirectory(root) ? expectedPath // R:\ exists ? + return Files.isDirectory(root) ? expectedPath // C:\ exists ? : Paths.get(System.getProperty("java.io.tmpdir")).resolve(SOFTWARE_NAME); } root = parent; @@ -102,13 +105,13 @@ static void assertStats(String[] argsRaw, String editionId, String testId, System.out.println(application.getRegistry().getStats() .entrySet() .stream() - .map(entry -> "\t\t\t\"" + entry.getKey() + "\", " + entry.getValue()) + .map(entry -> "\t\t\tentry(\"" + entry.getKey() + "\", " + entry.getValue() + ")") .collect(Collectors.joining(",\n", - "\t\tMap.of(\n", + "\t\tMap.ofEntries(\n", "),\n\t\t\t\t\t\t" + actualArchiveFileSize + "},"))); } if (delete) { - // Deleting testId direcroty from: R:\IntegrationTests\SOFTWARE_NAME\testId\resultArchiveName + // Deleting testId direcroty from: C:\test test\IntegrationTests\SOFTWARE_NAME\testId\resultArchiveName deleteRecursive(fullArchivePath.getParent()); } Assert.assertNull(t); diff --git a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CountryTest.java b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CountryTest.java index da06832..5698182 100644 --- a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CountryTest.java +++ b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/CountryTest.java @@ -11,6 +11,7 @@ import java.util.UUID; import static com.github.mbto.maxmind.geoip2.csv2sql.Constants.assertStats; +import static java.util.Map.entry; import static org.junit.runners.Parameterized.Parameters; /** @@ -33,100 +34,116 @@ public static Collection data() { {new String[]{ "-i", "4", "-mm", "32", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608), - 8043371}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30)), + 9818745}, {new String[]{ "-i", "4", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608), - 8042359}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30)), + 9818571}, {new String[]{ "-i", "4", "-l", "en,ru", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608), - 8032271}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30)), + 9808716}, {new String[]{ "-i", "4", "-l", "en", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608), - 8029644}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30)), + 9806066}, {new String[]{ "-i", "6", "-mm", "32", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv6", 110835), - 5330598}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 11650773}, {new String[]{ "-i", "6", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv6", 110835), - 5330598}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 11649909}, {new String[]{ "-i", "6", "-l", "en,ru", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv6", 110835), - 5320510}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 11640054}, {new String[]{ "-i", "6", "-l", "en", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv6", 110835), - 5317883}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 11637404}, {new String[]{ "-i", "4,6", "-mm", "32", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608, - "ipv6", 110835), - 13352349}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 21448126}, {new String[]{ "-i", "4,6", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608, - "ipv6", 110835), - 13351337}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 21447087}, {new String[]{ "-i", "4,6", "-l", "en,ru", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608, - "ipv6", 110835), - 13341249}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 21437232}, {new String[]{ "-i", "4,6", "-l", "en", - }, Map.of( - "country", 252, - "country includes which unknown", 2, - "ipv4", 338608, - "ipv6", 110835), - 13338622}, + }, Map.ofEntries( + entry("country", 252), + entry("country includes which unknown", 2), + entry("ipv4", 417044), + entry("ipv4 ignored", 30), + entry("ipv6", 270936), + entry("ipv6 ignored", 2)), + 21434582}, }); } diff --git a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/ManualTest.java b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/ManualTest.java index 4b310a2..d7aaf7a 100644 --- a/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/ManualTest.java +++ b/src/test/java/com/github/mbto/maxmind/geoip2/csv2sql/ManualTest.java @@ -9,14 +9,24 @@ import static com.github.mbto.maxmind.geoip2.csv2sql.Constants.*; /** - * For manual testing in IntelliJ: add -PManualTestEnabled in Gradle tab "Run Configuration" -> "Arguments" + * For manual testing in IntelliJ: add environment ManualTestEnabled=1 * https://dev.mysql.com/doc/refman/8.0/en/integer-types.html * https://www.postgresql.org/docs/current/datatype-numeric.html * https://docs.microsoft.com/ru-ru/sql/t-sql/data-types/int-bigint-smallint-and-tinyint-transact-sql?view=sql-server-ver15 */ public class ManualTest { + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean canExecute() { + boolean canExecute = System.getenv("ManualTestEnabled") != null; + if(!canExecute) { + System.out.println("Skipped test, environment ManualTestEnabled=1 required"); + } + return canExecute; + } @Test public void manualBuild1() throws Throwable { + if(!canExecute()) + return; String editionId = "GeoLite2-Country-CSV"; // String editionId = "GeoLite2-City-CSV"; @@ -66,6 +76,8 @@ public void manualBuild1() throws Throwable { @Test public void manualBuild2() throws Throwable { + if(!canExecute()) + return; String editionId = "GeoLite2-Country-CSV"; // String editionId = "GeoLite2-City-CSV"; String[] argsRaw = { @@ -99,13 +111,15 @@ public void manualBuild2() throws Throwable { @Test public void manualBuildWithMaxmindApi() throws Throwable { + if(!canExecute()) + return; // String editionId = "GeoLite2-City-CSV"; String editionId = "GeoLite2-Country-CSV"; String[] argsRaw = { // "-s", integrationTestsDirPath.resolve(buildArchiveName("GeoLite2-Country-CSV")).toString(), - "-s", "R:\\test test\\maxmind-geoip2-csv2sql-converter-1.0\\bin\\converted\\GeoLite2-Country-CSV_20210629.zip", + "-s", "C:\\test test\\maxmind-geoip2-csv2sql-converter-1.1\\bin\\converted\\GeoLite2-Country-CSV_20230616.zip", // "-od", buildIntegrationOutputDirPath(""), - "-od", "R:\\test test\\maxmind-geoip2-csv2sql-converter-1.0\\bin\\converted", + "-od", "C:\\test test\\maxmind-geoip2-csv2sql-converter-1.1\\bin\\converted", "-oa", resultArchiveName, // "-oa", "", "-c", buildMySQLConfigName(editionId), diff --git a/src/test/resources/GeoLite2-City-CSV_20210629.zip b/src/test/resources/GeoLite2-City-CSV_20230616.zip similarity index 71% rename from src/test/resources/GeoLite2-City-CSV_20210629.zip rename to src/test/resources/GeoLite2-City-CSV_20230616.zip index 569eabc..dfcda69 100644 Binary files a/src/test/resources/GeoLite2-City-CSV_20210629.zip and b/src/test/resources/GeoLite2-City-CSV_20230616.zip differ diff --git a/src/test/resources/GeoLite2-Country-CSV_20210629.zip b/src/test/resources/GeoLite2-Country-CSV_20210629.zip deleted file mode 100644 index 949286f..0000000 Binary files a/src/test/resources/GeoLite2-Country-CSV_20210629.zip and /dev/null differ diff --git a/src/test/resources/GeoLite2-Country-CSV_20230616.zip b/src/test/resources/GeoLite2-Country-CSV_20230616.zip new file mode 100644 index 0000000..dd82334 Binary files /dev/null and b/src/test/resources/GeoLite2-Country-CSV_20230616.zip differ diff --git a/src/test/resources/ipv4_ipv6_expanded_from_GeoLite2-Country-CSV_20210629.zip b/src/test/resources/ipv4_ipv6_expanded_from_GeoLite2-Country-CSV_20210629.zip deleted file mode 100644 index e93bcc6..0000000 Binary files a/src/test/resources/ipv4_ipv6_expanded_from_GeoLite2-Country-CSV_20210629.zip and /dev/null differ diff --git a/src/test/resources/ipv4_ipv6_expanded_from_GeoLite2-Country-CSV_20230616.zip b/src/test/resources/ipv4_ipv6_expanded_from_GeoLite2-Country-CSV_20230616.zip new file mode 100644 index 0000000..5969bda Binary files /dev/null and b/src/test/resources/ipv4_ipv6_expanded_from_GeoLite2-Country-CSV_20230616.zip differ