diff --git a/CHANGELOG b/CHANGELOG index 174b785..f2bd548 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +Version 0.7.0 (2020-06-06) +-------------------------- +Add support for anonymous IP database (#132) + Version 0.6.1 (2019-11-25) -------------------------- Default isInEuropeanUnion to false in case of NoSuchMethodError (#123) diff --git a/README.md b/README.md index 2fa1cd2..5af43de 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,12 @@ can also configure an LRU (Least Recently Used) cache of variable size ## Installation -The latest version of scala-maxmind-iplookups is **0.6.1** and is compatible with Scala 2.12. +The latest version of scala-maxmind-iplookups is **0.7.0** and is compatible with Scala 2.12. Add this to your SBT config: ```scala -val maxmindIpLookups = "com.snowplowanalytics" %% "scala-maxmind-iplookups" % "0.6.1" +val maxmindIpLookups = "com.snowplowanalytics" %% "scala-maxmind-iplookups" % "0.7.0" ``` Retrieve the `GeoLite2-City.mmdb` file from the [MaxMind downloads page][maxmind-downloads] @@ -253,7 +253,7 @@ As such we recommend upgrading to version 0.4.0 as soon as possible ## Copyright and license -Copyright 2012-2019 Snowplow Analytics Ltd. +Copyright 2012-2020 Snowplow Analytics Ltd. Licensed under the [Apache License, Version 2.0][license] (the "License"); you may not use this software except in compliance with the License. diff --git a/build.sbt b/build.sbt index e921761..16e0df5 100644 --- a/build.sbt +++ b/build.sbt @@ -17,7 +17,7 @@ lazy val root = project .settings( organization := "com.snowplowanalytics", name := "scala-maxmind-iplookups", - version := "0.6.1", + version := "0.7.0", description := "Scala wrapper for MaxMind GeoIP2 library", scalaVersion := "2.12.8", javacOptions := BuildSettings.javaCompilerOptions, diff --git a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala index 623446e..49e0b91 100644 --- a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala +++ b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2019 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -34,6 +34,7 @@ sealed trait CreateIpLookups[F[_]] { * @param ispFile ISP lookup database file * @param domainFile Domain lookup database file * @param connectionTypeFile Connection type lookup database file + * @param anonymousFile Anonymous lookup database file * @param memCache Whether to use MaxMind's CHMCache * @param lruCacheSize Maximum size of LruMap cache */ @@ -42,6 +43,7 @@ sealed trait CreateIpLookups[F[_]] { ispFile: Option[File] = None, domainFile: Option[File] = None, connectionTypeFile: Option[File] = None, + anonymousFile: Option[File] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 ): F[IpLookups[F]] @@ -52,6 +54,7 @@ sealed trait CreateIpLookups[F[_]] { * @param ispFile ISP lookup database filepath * @param domainFile Domain lookup database filepath * @param connectionTypeFile Connection type lookup database filepath + * @param anonymousFile Anonymous lookup database filepath * @param memCache Whether to use MaxMind's CHMCache * @param lruCacheSize Maximum size of LruMap cache */ @@ -60,6 +63,7 @@ sealed trait CreateIpLookups[F[_]] { ispFile: Option[String] = None, domainFile: Option[String] = None, connectionTypeFile: Option[String] = None, + anonymousFile: Option[String] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 ): F[IpLookups[F]] = createFromFiles( @@ -67,6 +71,7 @@ sealed trait CreateIpLookups[F[_]] { ispFile.map(new File(_)), domainFile.map(new File(_)), connectionTypeFile.map(new File(_)), + anonymousFile.map(new File(_)), memCache, lruCacheSize ) @@ -83,6 +88,7 @@ object CreateIpLookups { ispFile: Option[File] = None, domainFile: Option[File] = None, connectionTypeFile: Option[File] = None, + anonymousFile: Option[File] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 ): F[IpLookups[F]] = @@ -99,6 +105,7 @@ object CreateIpLookups { ispFile, domainFile, connectionTypeFile, + anonymousFile, memCache, lruCache ) @@ -114,6 +121,7 @@ object CreateIpLookups { ispFile: Option[File] = None, domainFile: Option[File] = None, connectionTypeFile: Option[File] = None, + anonymousFile: Option[File] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 ): Eval[IpLookups[Eval]] = @@ -130,6 +138,7 @@ object CreateIpLookups { ispFile, domainFile, connectionTypeFile, + anonymousFile, memCache, lruCache ) @@ -145,6 +154,7 @@ object CreateIpLookups { ispFile: Option[File] = None, domainFile: Option[File] = None, connectionTypeFile: Option[File] = None, + anonymousFile: Option[File] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 ): Id[IpLookups[Id]] = { @@ -159,6 +169,7 @@ object CreateIpLookups { ispFile, domainFile, connectionTypeFile, + anonymousFile, memCache, lruCache ) @@ -181,6 +192,7 @@ class IpLookups[F[_]: Monad] private[iplookups] ( ispFile: Option[File], domainFile: Option[File], connectionTypeFile: Option[File], + anonymousFile: Option[File], memCache: Boolean, lru: Option[LruMap[F, String, IpLookupResult]] )( @@ -194,6 +206,7 @@ class IpLookups[F[_]: Monad] private[iplookups] ( private val domainService = getService(domainFile).map((_, ReaderFunctions.domain)) private val connectionTypeService = getService(connectionTypeFile).map((_, ReaderFunctions.connectionType)) + private val anonymousService = getService(anonymousFile) /** * Get a LookupService from a database file @@ -212,7 +225,7 @@ class IpLookups[F[_]: Monad] private[iplookups] ( /** * Creates an Either from an IPLookup - * @param service ISP, domain or connection type LookupService + * @param service ISP, domain, connection or anonymous type LookupService * @return the result of the lookup */ private def getLookup( @@ -248,6 +261,16 @@ class IpLookups[F[_]: Monad] private[iplookups] ( case _ => Monad[F].pure(None) } + private def getAnonymousIpLookup( + ipAddress: Either[Throwable, InetAddress] + ): F[Option[Either[Throwable, AnonymousIp]]] = (ipAddress, anonymousService) match { + case (Right(ipA), Some(gs)) => + SR.getAnonymousValue(gs, ipA) + .map(loc => loc.map(AnonymousIp(_)).some) + case (Left(f), _) => Monad[F].pure(Some(Left(f))) + case _ => Monad[F].pure(None) + } + /** * This version does not use the LRU cache. * Concurrently looks up information @@ -267,7 +290,8 @@ class IpLookups[F[_]: Monad] private[iplookups] ( org <- getLookup(ipAddress, orgService) domain <- getLookup(ipAddress, domainService) connectionType <- getLookup(ipAddress, connectionTypeService) - } yield IpLookupResult(ipLocation, isp, org, domain, connectionType) + anonymous <- getAnonymousIpLookup(ipAddress) + } yield IpLookupResult(ipLocation, isp, org, domain, connectionType, anonymous) /** * Returns the MaxMind location for this IP address diff --git a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala index 65340b6..0b72768 100644 --- a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala +++ b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala @@ -19,6 +19,7 @@ import cats.effect.Sync import cats.syntax.either._ import com.maxmind.geoip2.DatabaseReader import com.maxmind.geoip2.model.CityResponse +import com.maxmind.geoip2.model.AnonymousIpResponse import model._ @@ -34,6 +35,12 @@ sealed trait SpecializedReader[F[_]] { db: DatabaseReader, ip: InetAddress ): F[Either[Throwable, CityResponse]] + + def getAnonymousValue( + db: DatabaseReader, + ip: InetAddress + ): F[Either[Throwable, AnonymousIpResponse]] + } object SpecializedReader { @@ -50,6 +57,13 @@ object SpecializedReader { ip: InetAddress ): F[Either[Throwable, CityResponse]] = Sync[F].delay { Either.catchNonFatal(db.city(ip)) } + + def getAnonymousValue( + db: DatabaseReader, + ip: InetAddress, + ): F[Either[Throwable, AnonymousIpResponse]] = + Sync[F].delay { Either.catchNonFatal(db.anonymousIp(ip)) } + } implicit def evalSpecializedReader: SpecializedReader[Eval] = new SpecializedReader[Eval] { @@ -65,6 +79,13 @@ object SpecializedReader { ip: InetAddress ): Eval[Either[Throwable, CityResponse]] = Eval.later { Either.catchNonFatal(db.city(ip)) } + + def getAnonymousValue( + db: DatabaseReader, + ip: InetAddress + ): Eval[Either[Throwable, AnonymousIpResponse]] = + Eval.later { Either.catchNonFatal(db.anonymousIp(ip)) } + } implicit def idSpecializedReader: SpecializedReader[Id] = new SpecializedReader[Id] { @@ -80,6 +101,12 @@ object SpecializedReader { ip: InetAddress ): Id[Either[Throwable, CityResponse]] = Either.catchNonFatal(db.city(ip)) + + def getAnonymousValue( + db: DatabaseReader, + ip: InetAddress + ): Id[Either[Throwable, AnonymousIpResponse]] = + Either.catchNonFatal(db.anonymousIp(ip)) } } diff --git a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/model.scala b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/model.scala index 0e0ee32..e0c5b8a 100644 --- a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/model.scala +++ b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/model.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2019 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -23,6 +23,7 @@ import cats.instances.either._ import com.maxmind.geoip2.DatabaseReader import com.maxmind.geoip2.model.CityResponse +import com.maxmind.geoip2.model.AnonymousIpResponse object model { type ReaderFunction = (DatabaseReader, InetAddress) => String @@ -46,6 +47,16 @@ object model { accuracyRadius: Int ) + /** A case class wrapper around the MaxMind AnonymousIp class. */ + final case class AnonymousIp( + ipAddress: String, + isAnonymous: Boolean, + isAnonymousVpn: Boolean, + isHostingProvider: Boolean, + isPublicProxy: Boolean, + isTorExitNode: Boolean + ) + /** Companion class contains a constructor which takes a MaxMind CityResponse. */ object IpLocation { @@ -80,25 +91,55 @@ object model { } } + /** Companion class contains a constructor which takes a MaxMind AnonymousIp. */ + object AnonymousIp { + + /** + * Constructs an AnonymousIp instance from a MaxMind AnonymousIp instance. + * @param anonymousIP MaxMind AnonymousIp object + * @return AnonymousIp + */ + def apply(anonymousIpResponse: AnonymousIpResponse): AnonymousIp = { + + AnonymousIp( + ipAddress = anonymousIpResponse.getIpAddress, + isAnonymous = anonymousIpResponse.isAnonymous, + isAnonymousVpn = anonymousIpResponse.isAnonymousVpn, + isHostingProvider = anonymousIpResponse.isHostingProvider, + isPublicProxy = anonymousIpResponse.isPublicProxy, + isTorExitNode = anonymousIpResponse.isTorExitNode + ) + } + + } + /** Result of MaxMind lookups */ final case class IpLookupResult( ipLocation: Option[Either[Throwable, IpLocation]], isp: Option[Either[Throwable, String]], organization: Option[Either[Throwable, String]], domain: Option[Either[Throwable, String]], - connectionType: Option[Either[Throwable, String]] + connectionType: Option[Either[Throwable, String]], + anonymousIp: Option[Either[Throwable, AnonymousIp]] ) { // Combine all errors if any def results: ValidatedNel[ Throwable, - (Option[IpLocation], Option[String], Option[String], Option[String], Option[String])] = { + ( + Option[IpLocation], + Option[String], + Option[String], + Option[String], + Option[String], + Option[AnonymousIp])] = { val location = ipLocation.sequence[Error, IpLocation].toValidatedNel val provider = isp.sequence[Error, String].toValidatedNel val org = organization.sequence[Error, String].toValidatedNel val dom = domain.sequence[Error, String].toValidatedNel val connection = connectionType.sequence[Error, String].toValidatedNel + val anonymous = anonymousIp.sequence[Error, AnonymousIp].toValidatedNel - (location, provider, org, dom, connection).tupled + (location, provider, org, dom, connection, anonymous).tupled } } } diff --git a/src/test/resources/com/snowplowanalytics/maxmind/iplookups/GeoIP2-Anonymous-IP-Test.mmdb b/src/test/resources/com/snowplowanalytics/maxmind/iplookups/GeoIP2-Anonymous-IP-Test.mmdb new file mode 100644 index 0000000..5ca54e7 Binary files /dev/null and b/src/test/resources/com/snowplowanalytics/maxmind/iplookups/GeoIP2-Anonymous-IP-Test.mmdb differ diff --git a/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala b/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala index 586876f..e0c4f36 100644 --- a/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala +++ b/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala @@ -29,6 +29,7 @@ object IpLookupsTest { val ispFile = getClass.getResource("GeoIP2-ISP-Test.mmdb").getFile val domainFile = getClass.getResource("GeoIP2-Domain-Test.mmdb").getFile val connectionTypeFile = getClass.getResource("GeoIP2-Connection-Type-Test.mmdb").getFile + val anonymousFile = getClass.getResource("GeoIP2-Anonymous-IP-Test.mmdb").getFile def ioIpLookupsFromFiles(memCache: Boolean, lruCache: Int): IpLookups[IO] = CreateIpLookups[IO] @@ -37,6 +38,7 @@ object IpLookupsTest { Some(ispFile), Some(domainFile), Some(connectionTypeFile), + Some(anonymousFile), memCache, lruCache ) @@ -49,6 +51,7 @@ object IpLookupsTest { Some(ispFile), Some(domainFile), Some(connectionTypeFile), + Some(anonymousFile), memCache, lruCache ) @@ -61,6 +64,7 @@ object IpLookupsTest { Some(ispFile), Some(domainFile), Some(connectionTypeFile), + Some(anonymousFile), memCache, lruCache ) @@ -86,7 +90,15 @@ object IpLookupsTest { new AddressNotFoundException("The address 175.16.199.0 is not in the database.").asLeft.some, new AddressNotFoundException("The address 175.16.199.0 is not in the database.").asLeft.some, new AddressNotFoundException("The address 175.16.199.0 is not in the database.").asLeft.some, - "Dialup".asRight.some + "Dialup".asRight.some, + AnonymousIp( + ipAddress = "175.16.199.0", + isAnonymous = false, + isAnonymousVpn = false, + isHostingProvider = false, + isPublicProxy = false, + isTorExitNode = false + ).asRight.some ), "216.160.83.56" -> IpLookupResult( IpLocation( @@ -107,7 +119,15 @@ object IpLookupsTest { "Century Link".asRight.some, "Lariat Software".asRight.some, new AddressNotFoundException("The address 216.160.83.56 is not in the database.").asLeft.some, - new AddressNotFoundException("The address 216.160.83.56 is not in the database.").asLeft.some + new AddressNotFoundException("The address 216.160.83.56 is not in the database.").asLeft.some, + AnonymousIp( + ipAddress = "216.160.83.56", + isAnonymous = false, + isAnonymousVpn = false, + isHostingProvider = false, + isPublicProxy = false, + isTorExitNode = false + ).asRight.some ), "67.43.156.0" -> IpLookupResult( IpLocation( @@ -128,7 +148,30 @@ object IpLookupsTest { "Loud Packet".asRight.some, "zudoarichikito_".asRight.some, "shoesfin.NET".asRight.some, - new AddressNotFoundException("The address 67.43.156.0 is not in the database.").asLeft.some + new AddressNotFoundException("The address 67.43.156.0 is not in the database.").asLeft.some, + AnonymousIp( + ipAddress = "67.43.156.0", + isAnonymous = false, + isAnonymousVpn = false, + isHostingProvider = false, + isPublicProxy = false, + isTorExitNode = false + ).asRight.some + ), + "81.2.69.11" -> IpLookupResult( + new AddressNotFoundException("The address 81.2.69.11 is not in the database.").asLeft.some, + new AddressNotFoundException("The address 81.2.69.11 is not in the database.").asLeft.some, + new AddressNotFoundException("The address 81.2.69.11 is not in the database.").asLeft.some, + "in-addr.arpa".asRight.some, + new AddressNotFoundException("The address 81.2.69.11 is not in the database.").asLeft.some, + AnonymousIp( + ipAddress = "81.2.69.11", + isAnonymous = true, + isAnonymousVpn = true, + isHostingProvider = true, + isPublicProxy = true, + isTorExitNode = true + ).asRight.some ), // Invalid IP address, as per // http://stackoverflow.com/questions/10456044/what-is-a-good-invalid-ip-address-to-use-for-unit-tests @@ -137,6 +180,7 @@ object IpLookupsTest { new AddressNotFoundException("The address 192.0.2.0 is not in the database.").asLeft.some, new AddressNotFoundException("The address 192.0.2.0 is not in the database.").asLeft.some, new AddressNotFoundException("The address 192.0.2.0 is not in the database.").asLeft.some, + new AddressNotFoundException("The address 192.0.2.0 is not in the database.").asLeft.some, new AddressNotFoundException("The address 192.0.2.0 is not in the database.").asLeft.some ) ) @@ -189,6 +233,7 @@ class IpLookupsTest extends Specification with Tables { new UnknownHostException("not: Name or service not known").asLeft.some, new UnknownHostException("not: Name or service not known").asLeft.some, new UnknownHostException("not: Name or service not known").asLeft.some, + new UnknownHostException("not: Name or service not known").asLeft.some, new UnknownHostException("not: Name or service not known").asLeft.some ) val evalExpected = IpLookupResult( @@ -196,6 +241,7 @@ class IpLookupsTest extends Specification with Tables { new UnknownHostException("not").asLeft.some, new UnknownHostException("not").asLeft.some, new UnknownHostException("not").asLeft.some, + new UnknownHostException("not").asLeft.some, new UnknownHostException("not").asLeft.some ) val idExpected = IpLookupResult( @@ -203,6 +249,7 @@ class IpLookupsTest extends Specification with Tables { new UnknownHostException("not").asLeft.some, new UnknownHostException("not").asLeft.some, new UnknownHostException("not").asLeft.some, + new UnknownHostException("not").asLeft.some, new UnknownHostException("not").asLeft.some ) val ioActual = ioIpLookups.performLookups("not").unsafeRunSync @@ -215,17 +262,17 @@ class IpLookupsTest extends Specification with Tables { "providing no files should return Nones" in { val ioActual = (for { - ipLookups <- CreateIpLookups[IO].createFromFiles(None, None, None, None, true, 0) + ipLookups <- CreateIpLookups[IO].createFromFiles(None, None, None, None, None, true, 0) res <- ipLookups.performLookups("67.43.156.0") } yield res).unsafeRunSync val evalActual = (for { - ipLookups <- CreateIpLookups[Eval].createFromFiles(None, None, None, None, true, 0) + ipLookups <- CreateIpLookups[Eval].createFromFiles(None, None, None, None, None, true, 0) res <- ipLookups.performLookups("67.43.156.0") } yield res).value val idActual = CreateIpLookups[Id] - .createFromFiles(None, None, None, None, true, 0) + .createFromFiles(None, None, None, None, None, true, 0) .performLookups("67.43.156.0") - val expected = IpLookupResult(None, None, None, None, None) + val expected = IpLookupResult(None, None, None, None, None, None) matchIpLookupResult(ioActual, expected) matchIpLookupResult(evalActual, expected) matchIpLookupResult(idActual, expected) @@ -238,7 +285,8 @@ class IpLookupsTest extends Specification with Tables { "isp" ! expected.isp ! actual.isp | "organization" ! expected.organization ! actual.organization | "domain" ! expected.domain ! actual.domain | - "connection type" ! expected.connectionType ! actual.connectionType | { (_, e, a) => + "connection type" ! expected.connectionType ! actual.connectionType | + "anonymous" ! expected.anonymousIp ! actual.anonymousIp | { (_, e, a) => matchThrowables(e, a) } }