diff --git a/build.sbt b/build.sbt index ef1029d..e13ad8c 100644 --- a/build.sbt +++ b/build.sbt @@ -33,8 +33,8 @@ lazy val root = project Dependencies.maxmind, Dependencies.catsEffect, Dependencies.cats, - Dependencies.scalaz, - Dependencies.specs2, - Dependencies.catsEffect + Dependencies.lruMap, + Dependencies.scalaCheck, + Dependencies.specs2 ) ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c34d7e6..d7c3f2f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -13,8 +13,10 @@ import sbt._ object Dependencies { - val maxmind = "com.maxmind.geoip2" % "geoip2" % "2.11.0" - val catsEffect = "org.typelevel" %% "cats-effect" % "0.10.1" - val cats = "org.typelevel" %% "cats-core" % "1.1.0" - val specs2 = "org.specs2" %% "specs2-core" % "4.0.3" % "test" + val maxmind = "com.maxmind.geoip2" % "geoip2" % "2.11.0" + val catsEffect = "org.typelevel" %% "cats-effect" % "0.10.1" + val cats = "org.typelevel" %% "cats-core" % "1.1.0" + val lruMap = "com.snowplowanalytics" %% "scala-lru-map" % "0.1.0" + val scalaCheck = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" + val specs2 = "org.specs2" %% "specs2-core" % "4.0.3" % "test" } diff --git a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala index 59dcf1a..d27017f 100644 --- a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala +++ b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/IpLookups.scala @@ -19,6 +19,7 @@ import java.util.{Collections, Map} import com.maxmind.db.CHMCache import com.maxmind.geoip2.model.CityResponse import com.maxmind.geoip2.DatabaseReader +import com.snowplowanalytics.lrumap.LruMap import cats.effect.Sync import cats.syntax.either._ import cats.syntax.flatMap._ @@ -46,9 +47,9 @@ object IpLookups { connectionTypeFile: Option[File] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 - ): F[IpLookups] = + ): F[IpLookups[F]] = ( - if (lruCacheSize >= 0) + if (lruCacheSize > 0) Sync[F].map(LruMap.create[F, String, IpLookupResult](lruCacheSize))(Some(_)) else Sync[F].pure(None) ).flatMap((lruCache) => @@ -80,7 +81,7 @@ object IpLookups { connectionTypeFile: Option[String] = None, memCache: Boolean = true, lruCacheSize: Int = 10000 - ): F[IpLookups] = + ): F[IpLookups[F]] = IpLookups.createFromFiles( geoFile.map(new File(_)), ispFile.map(new File(_)), @@ -104,18 +105,14 @@ object IpLookups { * Inspired by: * https://github.com/jt6211/hadoop-dns-mining/blob/master/src/main/java/io/covert/dns/geo/IpLookups.java */ -class IpLookups private ( +class IpLookups[F[_]: Sync] private ( geoFile: Option[File], ispFile: Option[File], domainFile: Option[File], connectionTypeFile: Option[File], memCache: Boolean, - lruCache: Option[LruMap[String, IpLookupResult]] + lru: Option[LruMap[F, String, IpLookupResult]] ) { - - // Initialise the cache - private val lru = lruCache - // Configure the lookup services private val geoService = getService(geoFile) private val ispService = getService(ispFile).map(SpecializedReader(_, ReaderFunctions.isp)) @@ -141,11 +138,11 @@ class IpLookups private ( } /** - * Creates a Validation from an IPLookup + * Creates an Either from an IPLookup * @param service ISP, domain or connection type LookupService * @return the result of the lookup */ - private def getLookup[F[_]: Sync]( + private def getLookup( ipAddress: Either[Throwable, InetAddress], service: Option[SpecializedReader] ): F[Option[Either[Throwable, String]]] = @@ -163,12 +160,12 @@ class IpLookups private ( * as an IpLocation, or None if MaxMind cannot find * the location. */ - def performLookups[F[_]: Sync](s: String): F[IpLookupResult] = + def performLookups(s: String): F[IpLookupResult] = lru .map(performLookupsWithLruCache(_, s)) .getOrElse(performLookupsWithoutLruCache(s)) - private def getLocationLookup[F[_]: Sync]( + private def getLocationLookup( ipAddress: Either[Throwable, InetAddress] ): F[Option[Either[Throwable, IpLocation]]] = (ipAddress, geoService) match { case (Right(ipA), Some(gs)) => @@ -189,7 +186,7 @@ class IpLookups private ( * @return Tuple containing the results of the * LookupServices */ - private def performLookupsWithoutLruCache[F[_]: Sync](ip: String): F[IpLookupResult] = + private def performLookupsWithoutLruCache(ip: String): F[IpLookupResult] = for { ipAddress <- getIpAddress(ip) @@ -211,26 +208,26 @@ class IpLookups private ( * cache entry could be found), versus an extant cache entry * containing None (meaning that the IP address is unknown). */ - private def performLookupsWithLruCache[F[_]: Sync]( - lru: LruMap[String, IpLookupResult], + private def performLookupsWithLruCache( + lru: LruMap[F, String, IpLookupResult], ip: String ): F[IpLookupResult] = { val lookupAndCache = performLookupsWithoutLruCache(ip).flatMap(result => { - LruMap.put(lru, ip, result).map(_ => result) + lru.put(ip, result).map(_ => result) }) - LruMap - .get(lru, ip) + lru + .get(ip) .map(_.map(Sync[F].pure(_))) .flatMap(_.getOrElse(lookupAndCache)) } /** Transforms a String into an Either[Throwable, InetAddress] */ - private def getIpAddress[F[_]: Sync](ip: String): F[Either[Throwable, InetAddress]] = + private def getIpAddress(ip: String): F[Either[Throwable, InetAddress]] = Sync[F].delay { Either.catchNonFatal(InetAddress.getByName(ip)) } - private def getCityResponse[F[_]: Sync]( + private def getCityResponse( gs: DatabaseReader, ipAddress: InetAddress ): F[Either[Throwable, CityResponse]] = diff --git a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/LruMap.scala b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/LruMap.scala deleted file mode 100644 index 20c743d..0000000 --- a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/LruMap.scala +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2012-2018 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.maxmind.iplookups - -import cats.effect.Sync -import java.{util => ju} -import java.util.LinkedHashMap -import scala.collection.JavaConverters._ -import scala.collection.mutable.{Map, MapLike, SynchronizedMap} - -// Based on com.twitter.util.LruMap -// https://github.com/twitter/util/blob/develop/util-collection/src/main/scala/com/twitter/util/LruMap.scala - -/** - * A wrapper trait for java.util.Map implementations to make them behave as scala Maps. - * This is useful if you want to have more specifically-typed wrapped objects instead - * of the generic maps returned by JavaConverters - */ -trait JMapWrapperLike[A, B, +Repr <: MapLike[A, B, Repr] with Map[A, B]] - extends Map[A, B] - with MapLike[A, B, Repr] { - def underlying: ju.Map[A, B] - - override def size = underlying.size - - override def get(k: A) = underlying.asScala.get(k) - - override def +=(kv: (A, B)): this.type = { underlying.put(kv._1, kv._2); this } - override def -=(key: A): this.type = { underlying remove key; this } - - override def put(k: A, v: B): Option[B] = underlying.asScala.put(k, v) - - override def update(k: A, v: B): Unit = underlying.put(k, v) - - override def remove(k: A): Option[B] = underlying.asScala.remove(k) - - override def clear() = underlying.clear() - - override def empty: Repr = null.asInstanceOf[Repr] - - override def iterator = underlying.asScala.iterator -} - -object LruMap { - def create[F[_]: Sync, K, V]( - size: Int - ): F[LruMap[K, V]] = implicitly[Sync[F]].delay { - new LruMap[K, V](size) - } - - def put[F[_]: Sync, K, V]( - lruMap: LruMap[K, V], - key: K, - value: V - ): F[Unit] = implicitly[Sync[F]].delay { - lruMap.put(key, value) - } - - def get[F[_]: Sync, K, V](lruMap: LruMap[K, V], key: K): F[Option[V]] = - implicitly[Sync[F]].delay { - lruMap.get(key) - } - - // initial capacity and load factor are the normal defaults for LinkedHashMap - def makeUnderlying[K, V](maxSize: Int): ju.Map[K, V] = - new LinkedHashMap[K, V]( - 16, /* initial capacity */ - 0.75f, /* load factor */ - true /* access order (as opposed to insertion order) */ - ) { - override protected def removeEldestEntry(eldest: ju.Map.Entry[K, V]): Boolean = - this.size() > maxSize - } -} - -/** - * A scala `Map` backed by a [[java.util.LinkedHashMap]] - */ -class LruMap[K, V](val maxSize: Int, val underlying: ju.Map[K, V]) - extends JMapWrapperLike[K, V, LruMap[K, V]] { - override def empty: LruMap[K, V] = new LruMap[K, V](maxSize) - def this(maxSize: Int) = this(maxSize, LruMap.makeUnderlying(maxSize)) -} diff --git a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala index 9968030..7255de6 100644 --- a/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala +++ b/src/main/scala/com.snowplowanalytics.maxmind.iplookups/SpecializedReader.scala @@ -24,6 +24,7 @@ import cats.syntax.either._ final case class SpecializedReader(db: DatabaseReader, f: ReaderFunction) { def getValue[F[_]: Sync](ip: InetAddress): F[Either[Throwable, String]] = Sync[F].delay { Either.catchNonFatal(f(db, ip)) } +} object ReaderFunctions { type ReaderFunction = (DatabaseReader, InetAddress) => String diff --git a/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala b/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala index 03512b0..0f61592 100644 --- a/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala +++ b/src/test/scala/com.snowplowanalytics.maxmind.iplookups/IpLookupsTest.scala @@ -16,6 +16,7 @@ import java.net.UnknownHostException import com.maxmind.geoip2.exception.AddressNotFoundException import org.specs2.mutable.Specification +import org.specs2.specification.Tables import cats.syntax.either._ import cats.syntax.option._ import cats.effect.IO @@ -24,7 +25,7 @@ import model._ object IpLookupsTest { - def ipLookupsFromFiles(memCache: Boolean, lruCache: Int): IpLookups = { + def ipLookupsFromFiles(memCache: Boolean, lruCache: Int): IpLookups[IO] = { val geoFile = getClass.getResource("GeoIP2-City-Test.mmdb").getFile val ispFile = getClass.getResource("GeoIP2-ISP-Test.mmdb").getFile val domainFile = getClass.getResource("GeoIP2-Domain-Test.mmdb").getFile @@ -109,7 +110,7 @@ object IpLookupsTest { ) } -class IpLookupsTest extends Specification { +class IpLookupsTest extends Specification with Tables { "Looking up some IP address locations should match their expected locations" should { @@ -135,7 +136,7 @@ class IpLookupsTest extends Specification { testData foreach { case (ip, expected) => formatter(ip, memCache, lruCache) should { - val actual = ipLookups.performLookups[IO](ip).unsafeRunSync() + val actual = ipLookups.performLookups(ip).unsafeRunSync matchIpLookupResult(actual, expected) } } @@ -150,14 +151,14 @@ class IpLookupsTest extends Specification { new UnknownHostException("not: Name or service not known").asLeft.some, new UnknownHostException("not: Name or service not known").asLeft.some ) - val actual = ipLookups.performLookups[IO]("not").unsafeRunSync + val actual = ipLookups.performLookups("not").unsafeRunSync matchIpLookupResult(actual, expected) } "providing no files should return Nones" in { val actual = (for { ipLookups <- IpLookups.createFromFiles[IO](None, None, None, None, true, 0) - res <- ipLookups.performLookups[IO]("67.43.156.0") + res <- ipLookups.performLookups("67.43.156.0") } yield res).unsafeRunSync val expected = IpLookupResult(None, None, None, None, None) matchIpLookupResult(actual, expected) @@ -165,16 +166,13 @@ class IpLookupsTest extends Specification { } private def matchIpLookupResult(actual: IpLookupResult, expected: IpLookupResult) = { - s"have iplocation = ${actual.ipLocation}" in { - matchThrowables(actual.ipLocation, expected.ipLocation) - } - s"have isp = ${actual.isp}" in { matchThrowables(actual.isp, expected.isp) } - s"have org = ${actual.organization}" in { - matchThrowables(actual.organization, expected.organization) - } - s"have domain = ${actual.domain}" in { matchThrowables(actual.domain, expected.domain) } - s"have net speed = ${actual.connectionType}" in { - matchThrowables(actual.connectionType, expected.connectionType) + "field" | "expected" | "actual" |> + "iplocation" ! expected.ipLocation ! actual.ipLocation | + "isp" ! expected.isp ! actual.isp | + "organization" ! expected.organization ! actual.organization | + "domain" ! expected.domain ! actual.domain | + "connection type" ! expected.connectionType ! actual.connectionType | { (_, e, a) => + matchThrowables(e, a) } }