From b88ce97445831867604fa0cf0f47b1700ebacdb0 Mon Sep 17 00:00:00 2001 From: EvgeniyPukanovich Date: Mon, 9 May 2022 16:58:44 +0500 Subject: [PATCH 1/4] two tables --- app/controllers/JobAggregatorController.scala | 12 +- app/model/Job.scala | 10 +- app/model/Salary.scala | 9 ++ app/model/db/DBTables.scala | 1 + app/model/db/JobTable.scala | 27 ++++- app/model/db/SalaryTable.scala | 20 ++++ app/scheduler/Task.scala | 26 +++++ app/scheduler/TasksModule.scala | 6 + app/service/JobAggregatorService.scala | 110 ++++++++++++++++-- conf/application.conf | 8 +- conf/evolutions/default/1.sql | 32 +++++ conf/routes | 2 +- 12 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 app/model/Salary.scala create mode 100644 app/model/db/SalaryTable.scala create mode 100644 app/scheduler/Task.scala create mode 100644 app/scheduler/TasksModule.scala create mode 100644 conf/evolutions/default/1.sql diff --git a/app/controllers/JobAggregatorController.scala b/app/controllers/JobAggregatorController.scala index 12dedf2..1849b98 100644 --- a/app/controllers/JobAggregatorController.scala +++ b/app/controllers/JobAggregatorController.scala @@ -3,13 +3,17 @@ package controllers import javax.inject._ import play.api.mvc._ import service.JobAggregatorService +import scala.util.{Failure, Success} -import scala.concurrent.ExecutionContext +import scala.concurrent.{ExecutionContext, Future} @Singleton -class JobAggregatorController @Inject()(val controllerComponents: ControllerComponents, jobAggregatorService: JobAggregatorService)(implicit ec: ExecutionContext) extends BaseController { +class JobAggregatorController @Inject()(val controllerComponents: ControllerComponents, + jobAggregatorService: JobAggregatorService)(implicit ec: ExecutionContext) extends BaseController { - def index() = Action.async { implicit request: Request[AnyContent] => - jobAggregatorService.addJobTest("Test title").map(_ => Ok("")) + def index(text: String, area: Int) = Action.async { implicit request: Request[AnyContent] => + + jobAggregatorService.parse(text, area).map(_ => Ok("")) + .recover(x => InternalServerError("Some exception has occurred")) } } diff --git a/app/model/Job.scala b/app/model/Job.scala index eb13f2e..4eae55b 100644 --- a/app/model/Job.scala +++ b/app/model/Job.scala @@ -2,4 +2,12 @@ package model import java.util.UUID -case class Job(id: UUID, title: String) +case class Job(id: UUID, + hh_id: Int, + title: Option[String], + requirement: Option[String], + responsibility: Option[String], + salary_id: Option[UUID], + alternate_url: Option[String], + city: Option[String], + key_word: Option[String]) diff --git a/app/model/Salary.scala b/app/model/Salary.scala new file mode 100644 index 0000000..e41967f --- /dev/null +++ b/app/model/Salary.scala @@ -0,0 +1,9 @@ +package model + +import java.util.UUID + +case class Salary(id: UUID, + _from: Option[Int], + _to: Option[Int], + currency: Option[String], + gross: Option[Boolean]) diff --git a/app/model/db/DBTables.scala b/app/model/db/DBTables.scala index 698fd20..ae7ed46 100644 --- a/app/model/db/DBTables.scala +++ b/app/model/db/DBTables.scala @@ -4,4 +4,5 @@ import slick.lifted.TableQuery object DBTables { val jobTable = TableQuery[JobTable] + val salaryTable = TableQuery[SalaryTable] } diff --git a/app/model/db/JobTable.scala b/app/model/db/JobTable.scala index e8313e7..6d0d98e 100644 --- a/app/model/db/JobTable.scala +++ b/app/model/db/JobTable.scala @@ -2,14 +2,33 @@ package model.db import model.Job import slick.jdbc.PostgresProfile.api._ - import java.util.UUID +import model.db.DBTables.salaryTable + class JobTable(tag: Tag) extends Table[Job](tag, "job") { - def id = column[UUID]("id", O.PrimaryKey) - def title = column[String]("title") + def id = column[UUID]("id", O.PrimaryKey) + + def hh_id = column[Int]("hh_id") + + def title = column[Option[String]]("title") + + def requirement = column[Option[String]]("requirement") + + def responsibility = column[Option[String]]("responsibility") + + def salary_id = column[Option[UUID]]("salary_id") + + def alternate_url = column[Option[String]]("alternate_url") + + def city = column[Option[String]]("city") + + def key_word = column[Option[String]]("key_word") + + def salary = foreignKey("fk_1", salary_id, salaryTable)(_.id) - def * = (id,title) <> (Job.tupled, Job.unapply) + def * = + (id, hh_id, title, requirement, responsibility, salary_id, alternate_url, city, key_word) <> (Job.tupled, Job.unapply) } diff --git a/app/model/db/SalaryTable.scala b/app/model/db/SalaryTable.scala new file mode 100644 index 0000000..f0142bb --- /dev/null +++ b/app/model/db/SalaryTable.scala @@ -0,0 +1,20 @@ +package model.db + +import java.util.UUID + +import model.Salary +import slick.jdbc.PostgresProfile.api._ + +class SalaryTable(tag: Tag) extends Table[Salary](tag, "salary") { + def id = column[UUID]("id", O.PrimaryKey) + + def _from = column[Option[Int]]("_from") + + def _to = column[Option[Int]]("_to") + + def currency = column[Option[String]]("currency") + + def gross = column[Option[Boolean]]("gross") + + override def * = (id, _from, _to, currency, gross) <> (Salary.tupled, Salary.unapply) +} diff --git a/app/scheduler/Task.scala b/app/scheduler/Task.scala new file mode 100644 index 0000000..dffe7a6 --- /dev/null +++ b/app/scheduler/Task.scala @@ -0,0 +1,26 @@ +package scheduler + +import javax.inject.Inject +import akka.actor.ActorSystem + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ +import play.api.Configuration +import service.JobAggregatorService + +class Task @Inject()(actorSystem: ActorSystem, + configuration: Configuration, + jobAggregatorService: JobAggregatorService)(implicit executionContext: ExecutionContext) { + + val initialDelay: String = configuration.get[String]("initialDelay") + val interval: String = configuration.get[String]("interval") + val cities: Seq[String] = configuration.get[Seq[String]]("cities") + val keyWords: Seq[String] = configuration.get[Seq[String]]("keyWords") + + actorSystem.scheduler.scheduleAtFixedRate(initialDelay = Duration(initialDelay).asInstanceOf[FiniteDuration], + interval = Duration(interval).asInstanceOf[FiniteDuration]) { () => + jobAggregatorService.parse(keyWords, cities) + println("Scheduled task executed") + //actorSystem.log.info("Executing something...") + } +} diff --git a/app/scheduler/TasksModule.scala b/app/scheduler/TasksModule.scala new file mode 100644 index 0000000..561dfe6 --- /dev/null +++ b/app/scheduler/TasksModule.scala @@ -0,0 +1,6 @@ +package scheduler + +import play.api.inject.SimpleModule +import play.api.inject._ + +class TasksModule extends SimpleModule(bind[Task].toSelf.eagerly()) diff --git a/app/service/JobAggregatorService.scala b/app/service/JobAggregatorService.scala index 198d5c8..d52feff 100644 --- a/app/service/JobAggregatorService.scala +++ b/app/service/JobAggregatorService.scala @@ -1,21 +1,117 @@ package service -import model.Job -import model.db.DBTables.jobTable +import model.{Job, Salary} +import model.db.DBTables.{jobTable, salaryTable} import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} import slick.jdbc.JdbcProfile import slick.jdbc.PostgresProfile.api._ - import java.util.UUID + import javax.inject.{Inject, Singleton} +import play.api.db.evolutions.InvalidDatabaseRevision +import play.api.libs.ws._ +import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup +import play.api.libs.json._ -import scala.concurrent.Future +import scala.concurrent.ExecutionContext.Implicits.global @Singleton -class JobAggregatorService @Inject()(val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] { +class JobAggregatorService @Inject()(ws: WSClient, + val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] { + + + def parse(text: String, area: Int) = { + //&per_page=100&page=0 + val response = ws.url(s"https://api.hh.ru/vacancies?text=$text&area=$area").get() + getRegions().flatMap( + region => response.map(x => parseJobs(x.json, text, region._1(area))).flatMap(x => addToDBSeq(x._1, x._2))) + } + + def parse(keyWords: Seq[String], areas: Seq[String]) = { + val regionsMap = getRegions().map(x => x._2) + + val requests = for { + keyWord <- keyWords + area <- areas + } yield regionsMap.map(x => + (ws.url(s"https://api.hh.ru/vacancies?text=$keyWord&area=${x(area)}"), keyWord, area)) + + val jobsAndSalaries = for { + request <- requests + } yield request.flatMap(x => x._1.get().map(resp => parseJobs(resp.json, x._2, x._3))) + + jobsAndSalaries.foreach(x => x.map(s => addToDBSeq(s._1,s._2))) + } + + private def parseJobs(json: JsValue, keyWord: String, area: String) = { + val per_page = (json \ "per_page").get.toString.toInt + var jobs = Seq[Job]() + var salaries = Seq[Salary]() + + for (n <- 0 until per_page) { + val items = (json \ "items" \ n).get + val id = (items \ "id").as[String].toInt + val name = (items \ "name").asOpt[String] + val alternate_url = (items \ "alternate_url").asOpt[String] + val requirement = (items \ "snippet" \ "requirement").asOpt[String] + val responsibility = (items \ "snippet" \ "responsibility").asOpt[String] - def addJobTest(title: String): Future[Int] = { - db.run(jobTable += Job(UUID.randomUUID(), title)) + val salary = (items \ "salary").get + var uuid: Option[UUID] = None + if (salary != JsNull) { + val currency = (salary \ "currency").asOpt[String] + val from = (salary \ "from").asOpt[Int] + val gross = (salary \ "gross").asOpt[Boolean] + val to = (salary \ "to").asOpt[Int] + val id = UUID.randomUUID() + salaries = salaries :+ Salary(id, from, to, currency, gross) + uuid = Option(id) + } + val job = Job(UUID.randomUUID(), id, name, requirement, responsibility, uuid, alternate_url, Option(area), Option(keyWord)) + job.hashCode() + jobs = jobs :+ job } + (jobs, salaries) + } + + private def getRegions() = { + val resp = ws.url("https://api.hh.ru/areas").get() + val indexToRegion = scala.collection.mutable.Map[Int, String]() + val regionToIndex = scala.collection.mutable.Map[String, Int]() + + def initialParse(json: JsValue) = { + for (n <- json.as[Seq[JsValue]]) { + parseRegions(n) + } + (indexToRegion, regionToIndex) + } + + def parseRegions(regionJson: JsValue): Unit = { + val areas = (regionJson \ "areas").as[Seq[JsValue]] + val id = (regionJson \ "id").asOpt[String] + val name = (regionJson \ "name").asOpt[String] + + if (id.isDefined && name.isDefined) { + indexToRegion += (id.get.toInt -> name.get) + regionToIndex += (name.get -> id.get.toInt) + } + + if (areas.nonEmpty) { + for (n <- areas) + parseRegions(n) + } + } + + resp.map(x => initialParse(x.json)) + } + + private def addToDB(job: Job, salary: Salary) ={ + db.run(DBIO.seq(salaryTable+= salary, jobTable+=job)) + } + + private def addToDBSeq(jobs: Seq[Job], salaries: Seq[Salary]) = { + db.run(DBIO.seq(salaryTable ++= salaries, jobTable ++= jobs)) + //Await.result(db.run(DBIO.seq(salaryTable ++= salaries, jobTable ++= jobs)), Duration.Inf) + } } diff --git a/conf/application.conf b/conf/application.conf index 9ef1931..4924fde 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -1,6 +1,12 @@ slick.dbs.default.driver="slick.driver.PostgresDriver$" slick.dbs.default.db.driver="org.postgresql.Driver" -slick.dbs.default.db.url="jdbc:postgresql://10.106.0.26:5434/postgres" +slick.dbs.default.db.url="jdbc:postgresql://localhost:5434/postgres" slick.dbs.default.db.user="postgres" slick.dbs.default.db.password="12345678" +//play.modules.enabled += "scheduler.TasksModule" + +initialDelay="10 seconds" +interval="10 seconds" +cities=["Москва"] +keyWords=["java"] \ No newline at end of file diff --git a/conf/evolutions/default/1.sql b/conf/evolutions/default/1.sql new file mode 100644 index 0000000..0c7e6e5 --- /dev/null +++ b/conf/evolutions/default/1.sql @@ -0,0 +1,32 @@ +--public schema + +-- !Ups + +CREATE TABLE public.job( +id uuid DEFAULT gen_random_uuid() PRIMARY KEY, +hh_id int, +title text, +requirement text, +responsibility text, +salary_id uuid, +alternate_url text, +city text, +key_word text +); + +CREATE TABLE public.salary( +id uuid PRIMARY KEY, +_from int, +_to int, +currency text, +gross boolean +); + +ALTER TABLE public.job ADD CONSTRAINT fk_1 FOREIGN KEY (salary_id) REFERENCES public.salary(id); +CREATE INDEX ON public.job(city); +CREATE INDEX ON public.job(key_word); + +-- !Downs + +DROP TABLE public.job; +DROP TABLE public.salary; diff --git a/conf/routes b/conf/routes index 949ce1b..dc9c739 100644 --- a/conf/routes +++ b/conf/routes @@ -4,7 +4,7 @@ # ~~~~ # An example controller showing a sample home page -GET / controllers.JobAggregatorController.index() +GET / controllers.JobAggregatorController.index(text: String, area: Int) # Map static resources from the /public folder to the /assets URL path GET /assets/*file controllers.Assets.versioned(path="/public", file: Asset) From bf8de5a5f4edcbfbe3022fced89b1c322c9d6f37 Mon Sep 17 00:00:00 2001 From: EvgeniyPukanovich Date: Tue, 10 May 2022 18:04:31 +0500 Subject: [PATCH 2/4] phase 1 --- app/model/Job.scala | 8 +- app/model/Salary.scala | 9 -- app/model/db/DBTables.scala | 1 - app/model/db/JobTable.scala | 21 +++-- app/model/db/SalaryTable.scala | 20 ---- app/service/JobAggregatorService.scala | 126 ++++++++++++++++--------- conf/application.conf | 6 +- conf/evolutions/default/1.sql | 18 +--- 8 files changed, 103 insertions(+), 106 deletions(-) delete mode 100644 app/model/Salary.scala delete mode 100644 app/model/db/SalaryTable.scala diff --git a/app/model/Job.scala b/app/model/Job.scala index 4eae55b..82b4e2a 100644 --- a/app/model/Job.scala +++ b/app/model/Job.scala @@ -2,12 +2,14 @@ package model import java.util.UUID -case class Job(id: UUID, - hh_id: Int, +case class Job(id: Int, title: Option[String], requirement: Option[String], responsibility: Option[String], - salary_id: Option[UUID], alternate_url: Option[String], + salary_from: Option[Int], + salary_to: Option[Int], + salary_currency: Option[String], + salary_gross: Option[Boolean], city: Option[String], key_word: Option[String]) diff --git a/app/model/Salary.scala b/app/model/Salary.scala deleted file mode 100644 index e41967f..0000000 --- a/app/model/Salary.scala +++ /dev/null @@ -1,9 +0,0 @@ -package model - -import java.util.UUID - -case class Salary(id: UUID, - _from: Option[Int], - _to: Option[Int], - currency: Option[String], - gross: Option[Boolean]) diff --git a/app/model/db/DBTables.scala b/app/model/db/DBTables.scala index ae7ed46..698fd20 100644 --- a/app/model/db/DBTables.scala +++ b/app/model/db/DBTables.scala @@ -4,5 +4,4 @@ import slick.lifted.TableQuery object DBTables { val jobTable = TableQuery[JobTable] - val salaryTable = TableQuery[SalaryTable] } diff --git a/app/model/db/JobTable.scala b/app/model/db/JobTable.scala index 6d0d98e..bbd285d 100644 --- a/app/model/db/JobTable.scala +++ b/app/model/db/JobTable.scala @@ -4,12 +4,8 @@ import model.Job import slick.jdbc.PostgresProfile.api._ import java.util.UUID -import model.db.DBTables.salaryTable - class JobTable(tag: Tag) extends Table[Job](tag, "job") { - def id = column[UUID]("id", O.PrimaryKey) - - def hh_id = column[Int]("hh_id") + def id = column[Int]("id", O.PrimaryKey) def title = column[Option[String]]("title") @@ -17,18 +13,23 @@ class JobTable(tag: Tag) extends Table[Job](tag, "job") { def responsibility = column[Option[String]]("responsibility") - def salary_id = column[Option[UUID]]("salary_id") - def alternate_url = column[Option[String]]("alternate_url") + def salary_from = column[Option[Int]]("salary_from") + + def salary_to = column[Option[Int]]("salary_to") + + def salary_currency = column[Option[String]]("salary_currency") + + def salary_gross = column[Option[Boolean]]("salary_gross") + def city = column[Option[String]]("city") def key_word = column[Option[String]]("key_word") - def salary = foreignKey("fk_1", salary_id, salaryTable)(_.id) - def * = - (id, hh_id, title, requirement, responsibility, salary_id, alternate_url, city, key_word) <> (Job.tupled, Job.unapply) + (id, title, requirement, responsibility, alternate_url, salary_from, salary_to, salary_currency, salary_gross, + city, key_word) <> (Job.tupled, Job.unapply) } diff --git a/app/model/db/SalaryTable.scala b/app/model/db/SalaryTable.scala deleted file mode 100644 index f0142bb..0000000 --- a/app/model/db/SalaryTable.scala +++ /dev/null @@ -1,20 +0,0 @@ -package model.db - -import java.util.UUID - -import model.Salary -import slick.jdbc.PostgresProfile.api._ - -class SalaryTable(tag: Tag) extends Table[Salary](tag, "salary") { - def id = column[UUID]("id", O.PrimaryKey) - - def _from = column[Option[Int]]("_from") - - def _to = column[Option[Int]]("_to") - - def currency = column[Option[String]]("currency") - - def gross = column[Option[Boolean]]("gross") - - override def * = (id, _from, _to, currency, gross) <> (Salary.tupled, Salary.unapply) -} diff --git a/app/service/JobAggregatorService.scala b/app/service/JobAggregatorService.scala index d52feff..b430b2f 100644 --- a/app/service/JobAggregatorService.scala +++ b/app/service/JobAggregatorService.scala @@ -1,80 +1,112 @@ package service -import model.{Job, Salary} -import model.db.DBTables.{jobTable, salaryTable} +import model.Job +import model.db.DBTables.jobTable import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} import slick.jdbc.JdbcProfile import slick.jdbc.PostgresProfile.api._ -import java.util.UUID import javax.inject.{Inject, Singleton} -import play.api.db.evolutions.InvalidDatabaseRevision import play.api.libs.ws._ import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup import play.api.libs.json._ import scala.concurrent.ExecutionContext.Implicits.global +import scala.collection.mutable.ListBuffer +import scala.concurrent.{Await, Future} @Singleton class JobAggregatorService @Inject()(ws: WSClient, val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] { + val perPage: Int = 100 + /** + * добавляет в БД все вакансии по ключевому слову и региону + * @param text ключевое слов + * @param area индекс региона + */ def parse(text: String, area: Int) = { - //&per_page=100&page=0 - val response = ws.url(s"https://api.hh.ru/vacancies?text=$text&area=$area").get() - getRegions().flatMap( - region => response.map(x => parseJobs(x.json, text, region._1(area))).flatMap(x => addToDBSeq(x._1, x._2))) + + ws.url(s"https://api.hh.ru/vacancies?text=$text&area=$area&per_page=$perPage&page=0") + .get() + .flatMap(x => getJobs(x.json, text, area)) + .map(buff => buff.foreach(x => addToDB(x))) } - def parse(keyWords: Seq[String], areas: Seq[String]) = { + /** + * добавляет в БД все вакансии по всем ключевым словам и регионам + * @param keyWords ключевые слова + * @param areas индексы регионов + */ + def parse(keyWords: Seq[String], areas: Seq[String]):Unit = { val regionsMap = getRegions().map(x => x._2) - - val requests = for { + for { keyWord <- keyWords area <- areas - } yield regionsMap.map(x => - (ws.url(s"https://api.hh.ru/vacancies?text=$keyWord&area=${x(area)}"), keyWord, area)) + } regionsMap.map(x => parse(keyWord, x(area))) + } + + /** + * делает несколько запросов к hh.ru, если все вакансии не помещаются в один ответ + * @param firstResp первый ответ от hh.ru + * @return Future от массива всех полученных вакансий + */ + private def getJobs(firstResp: JsValue, keyWord: String, area: Int) = { - val jobsAndSalaries = for { - request <- requests - } yield request.flatMap(x => x._1.get().map(resp => parseJobs(resp.json, x._2, x._3))) + def parseAllPages(areaText: String) = { + val pages = (firstResp \ "pages").get.toString.toInt + val jobs: ListBuffer[Job] = getJobsFromPage(firstResp, keyWord, areaText) - jobsAndSalaries.foreach(x => x.map(s => addToDBSeq(s._1,s._2))) + val futures = for (n <- 1 until pages) + yield ws.url(s"https://api.hh.ru/vacancies?text=$keyWord&area=$area&per_page=$perPage&page=$n") + .get().map(resp => getJobsFromPage(resp.json, keyWord, areaText)) + + Future.foldLeft(futures)(jobs)((acc, e) => acc ++ e) + } + + getRegions().flatMap(x => parseAllPages(x._1(area))) } - private def parseJobs(json: JsValue, keyWord: String, area: String) = { - val per_page = (json \ "per_page").get.toString.toInt - var jobs = Seq[Job]() - var salaries = Seq[Salary]() - - for (n <- 0 until per_page) { - val items = (json \ "items" \ n).get - val id = (items \ "id").as[String].toInt - val name = (items \ "name").asOpt[String] - val alternate_url = (items \ "alternate_url").asOpt[String] - val requirement = (items \ "snippet" \ "requirement").asOpt[String] - val responsibility = (items \ "snippet" \ "responsibility").asOpt[String] - - val salary = (items \ "salary").get - var uuid: Option[UUID] = None + /** + * парсит вакансии в массив DTO Job + * @param json json, содержащий массив вакансий + * @return массив DTO Job + */ + private def getJobsFromPage(json: JsValue, keyWord: String, area: String): ListBuffer[Job] = { + var jobs = ListBuffer[Job]() + val items = (json \ "items").as[Seq[JsValue]] + + for (item <- items) { + val id = (item \ "id").as[String].toInt + val name = (item \ "name").asOpt[String] + val alternate_url = (item \ "alternate_url").asOpt[String] + val requirement = (item \ "snippet" \ "requirement").asOpt[String] + val responsibility = (item \ "snippet" \ "responsibility").asOpt[String] + + val salary = (item \ "salary").get + var from: Option[Int] = None + var to: Option[Int] = None + var currency: Option[String] = None + var gross: Option[Boolean] = None + if (salary != JsNull) { - val currency = (salary \ "currency").asOpt[String] - val from = (salary \ "from").asOpt[Int] - val gross = (salary \ "gross").asOpt[Boolean] - val to = (salary \ "to").asOpt[Int] - val id = UUID.randomUUID() - salaries = salaries :+ Salary(id, from, to, currency, gross) - uuid = Option(id) + currency = (salary \ "currency").asOpt[String] + from = (salary \ "from").asOpt[Int] + gross = (salary \ "gross").asOpt[Boolean] + to = (salary \ "to").asOpt[Int] } - val job = Job(UUID.randomUUID(), id, name, requirement, responsibility, uuid, alternate_url, Option(area), Option(keyWord)) - job.hashCode() - jobs = jobs :+ job + val job = + Job(id, name, requirement, responsibility, alternate_url, from, to, currency, gross, Option(area), Option(keyWord)) + jobs += job } - (jobs, salaries) + jobs } + /** + * @return возвращает два словаря: (id -> название_региона) и (название_региона -> id) + */ private def getRegions() = { val resp = ws.url("https://api.hh.ru/areas").get() val indexToRegion = scala.collection.mutable.Map[Int, String]() @@ -106,12 +138,12 @@ class JobAggregatorService @Inject()(ws: WSClient, resp.map(x => initialParse(x.json)) } - private def addToDB(job: Job, salary: Salary) ={ - db.run(DBIO.seq(salaryTable+= salary, jobTable+=job)) + private def addToDB(job: Job) = { + db.run(DBIO.seq(jobTable += job)) } - private def addToDBSeq(jobs: Seq[Job], salaries: Seq[Salary]) = { - db.run(DBIO.seq(salaryTable ++= salaries, jobTable ++= jobs)) + private def addToDBSeq(jobs: Seq[Job]) = { + db.run(DBIO.seq(jobTable ++= jobs)) //Await.result(db.run(DBIO.seq(salaryTable ++= salaries, jobTable ++= jobs)), Duration.Inf) } } diff --git a/conf/application.conf b/conf/application.conf index 4924fde..dbc1321 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -7,6 +7,6 @@ slick.dbs.default.db.password="12345678" //play.modules.enabled += "scheduler.TasksModule" initialDelay="10 seconds" -interval="10 seconds" -cities=["Москва"] -keyWords=["java"] \ No newline at end of file +interval="30 seconds" +cities=["Москва","Екатеринбург"] +keyWords=["java","scala"] \ No newline at end of file diff --git a/conf/evolutions/default/1.sql b/conf/evolutions/default/1.sql index 0c7e6e5..806f880 100644 --- a/conf/evolutions/default/1.sql +++ b/conf/evolutions/default/1.sql @@ -3,30 +3,22 @@ -- !Ups CREATE TABLE public.job( -id uuid DEFAULT gen_random_uuid() PRIMARY KEY, -hh_id int, +id int PRIMARY KEY, title text, requirement text, responsibility text, -salary_id uuid, alternate_url text, +salary_from int, +salary_to int, +salary_currency text, +salary_gross boolean, city text, key_word text ); -CREATE TABLE public.salary( -id uuid PRIMARY KEY, -_from int, -_to int, -currency text, -gross boolean -); - -ALTER TABLE public.job ADD CONSTRAINT fk_1 FOREIGN KEY (salary_id) REFERENCES public.salary(id); CREATE INDEX ON public.job(city); CREATE INDEX ON public.job(key_word); -- !Downs DROP TABLE public.job; -DROP TABLE public.salary; From c01447fddc780f888aa683769f243b2b4a56fe82 Mon Sep 17 00:00:00 2001 From: EvgeniyPukanovich Date: Sat, 21 May 2022 15:00:25 +0500 Subject: [PATCH 3/4] fixed problems --- app/ DTOs/HolderDTO.scala | 10 ++ app/ DTOs/RegionDTO.scala | 11 ++ app/ DTOs/SalaryDTO.scala | 12 ++ app/ DTOs/SnippetDTO.scala | 10 ++ app/ DTOs/VacancyDTO.scala | 13 ++ app/controllers/JobAggregatorController.scala | 4 +- app/model/Job.scala | 14 +- app/model/db/JobTable.scala | 17 +- app/scheduler/Task.scala | 35 ++-- app/service/JobAggregatorService.scala | 169 +++++++++++------- conf/application.conf | 3 +- 11 files changed, 201 insertions(+), 97 deletions(-) create mode 100644 app/ DTOs/HolderDTO.scala create mode 100644 app/ DTOs/RegionDTO.scala create mode 100644 app/ DTOs/SalaryDTO.scala create mode 100644 app/ DTOs/SnippetDTO.scala create mode 100644 app/ DTOs/VacancyDTO.scala diff --git a/app/ DTOs/HolderDTO.scala b/app/ DTOs/HolderDTO.scala new file mode 100644 index 0000000..af76c36 --- /dev/null +++ b/app/ DTOs/HolderDTO.scala @@ -0,0 +1,10 @@ +package DTOs + +import play.api.libs.json.{Json, Reads} + +case class HolderDTO(pages: Int, + items: Option[Seq[VacancyDTO]]) + +object HolderDTO { + implicit val holderDtoReader: Reads[HolderDTO] = Json.reads[HolderDTO] +} diff --git a/app/ DTOs/RegionDTO.scala b/app/ DTOs/RegionDTO.scala new file mode 100644 index 0000000..e5159eb --- /dev/null +++ b/app/ DTOs/RegionDTO.scala @@ -0,0 +1,11 @@ +package DTOs + +import play.api.libs.json.{Json, Reads} + +case class RegionDTO(id: String, + name: String, + areas: Seq[RegionDTO]) + +object RegionDTO { + implicit val regionDtoReader: Reads[RegionDTO] = Json.reads[RegionDTO] +} \ No newline at end of file diff --git a/app/ DTOs/SalaryDTO.scala b/app/ DTOs/SalaryDTO.scala new file mode 100644 index 0000000..7ad7bd3 --- /dev/null +++ b/app/ DTOs/SalaryDTO.scala @@ -0,0 +1,12 @@ +package DTOs + +import play.api.libs.json.{Json, Reads} + +case class SalaryDTO(currency: Option[String], + from: Option[Int], + gross: Option[Boolean], + to: Option[Int]) + +object SalaryDTO { + implicit val salaryDtoReader: Reads[SalaryDTO] = Json.reads[SalaryDTO] +} diff --git a/app/ DTOs/SnippetDTO.scala b/app/ DTOs/SnippetDTO.scala new file mode 100644 index 0000000..2c81b9f --- /dev/null +++ b/app/ DTOs/SnippetDTO.scala @@ -0,0 +1,10 @@ +package DTOs + +import play.api.libs.json.{Json, Reads} + +case class SnippetDTO(requirement: Option[String], + responsibility: Option[String]) + +object SnippetDTO { + implicit val snippetDtoReader: Reads[SnippetDTO] = Json.reads[SnippetDTO] +} diff --git a/app/ DTOs/VacancyDTO.scala b/app/ DTOs/VacancyDTO.scala new file mode 100644 index 0000000..d88fa43 --- /dev/null +++ b/app/ DTOs/VacancyDTO.scala @@ -0,0 +1,13 @@ +package DTOs + +import play.api.libs.json.{Json, Reads} + +case class VacancyDTO(id: String, + name: Option[String], + alternate_url: Option[String], + snippet: Option[SnippetDTO], + salary: Option[SalaryDTO]) + +object VacancyDTO { + implicit val vacancyDtoReader: Reads[VacancyDTO] = Json.reads[VacancyDTO] +} diff --git a/app/controllers/JobAggregatorController.scala b/app/controllers/JobAggregatorController.scala index 1849b98..2a3d0b0 100644 --- a/app/controllers/JobAggregatorController.scala +++ b/app/controllers/JobAggregatorController.scala @@ -13,7 +13,7 @@ class JobAggregatorController @Inject()(val controllerComponents: ControllerComp def index(text: String, area: Int) = Action.async { implicit request: Request[AnyContent] => - jobAggregatorService.parse(text, area).map(_ => Ok("")) - .recover(x => InternalServerError("Some exception has occurred")) + jobAggregatorService.aggregateData(text, area).map(_ => Ok("")) + .recover(exception => InternalServerError("Following exception has occurred: " + exception.getMessage)) } } diff --git a/app/model/Job.scala b/app/model/Job.scala index 82b4e2a..a8f151c 100644 --- a/app/model/Job.scala +++ b/app/model/Job.scala @@ -1,15 +1,13 @@ package model -import java.util.UUID - case class Job(id: Int, title: Option[String], requirement: Option[String], responsibility: Option[String], - alternate_url: Option[String], - salary_from: Option[Int], - salary_to: Option[Int], - salary_currency: Option[String], - salary_gross: Option[Boolean], + alternateUrl: Option[String], + salaryFrom: Option[Int], + salaryTo: Option[Int], + salaryCurrency: Option[String], + salaryGross: Option[Boolean], city: Option[String], - key_word: Option[String]) + keyWord: Option[String]) diff --git a/app/model/db/JobTable.scala b/app/model/db/JobTable.scala index bbd285d..3cc2f75 100644 --- a/app/model/db/JobTable.scala +++ b/app/model/db/JobTable.scala @@ -2,7 +2,6 @@ package model.db import model.Job import slick.jdbc.PostgresProfile.api._ -import java.util.UUID class JobTable(tag: Tag) extends Table[Job](tag, "job") { def id = column[Int]("id", O.PrimaryKey) @@ -13,23 +12,23 @@ class JobTable(tag: Tag) extends Table[Job](tag, "job") { def responsibility = column[Option[String]]("responsibility") - def alternate_url = column[Option[String]]("alternate_url") + def alternateUrl = column[Option[String]]("alternate_url") - def salary_from = column[Option[Int]]("salary_from") + def salaryFrom = column[Option[Int]]("salary_from") - def salary_to = column[Option[Int]]("salary_to") + def salaryTo = column[Option[Int]]("salary_to") - def salary_currency = column[Option[String]]("salary_currency") + def salaryCurrency = column[Option[String]]("salary_currency") - def salary_gross = column[Option[Boolean]]("salary_gross") + def salaryGross = column[Option[Boolean]]("salary_gross") def city = column[Option[String]]("city") - def key_word = column[Option[String]]("key_word") + def keyWord = column[Option[String]]("key_word") def * = - (id, title, requirement, responsibility, alternate_url, salary_from, salary_to, salary_currency, salary_gross, - city, key_word) <> (Job.tupled, Job.unapply) + (id, title, requirement, responsibility, alternateUrl, salaryFrom, salaryTo, salaryCurrency, salaryGross, + city, keyWord) <> (Job.tupled, Job.unapply) } diff --git a/app/scheduler/Task.scala b/app/scheduler/Task.scala index dffe7a6..53c3ce6 100644 --- a/app/scheduler/Task.scala +++ b/app/scheduler/Task.scala @@ -5,22 +5,37 @@ import akka.actor.ActorSystem import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -import play.api.Configuration +import play.api.{Configuration, Logger, Mode} import service.JobAggregatorService +import scala.util.Try + class Task @Inject()(actorSystem: ActorSystem, configuration: Configuration, jobAggregatorService: JobAggregatorService)(implicit executionContext: ExecutionContext) { - val initialDelay: String = configuration.get[String]("initialDelay") - val interval: String = configuration.get[String]("interval") - val cities: Seq[String] = configuration.get[Seq[String]]("cities") - val keyWords: Seq[String] = configuration.get[Seq[String]]("keyWords") + if (!configuration.has("initialDelay") || !configuration.has("interval") + || !configuration.has("cities") || !configuration.has("keyWords")) + throw new NoSuchFieldException("Configuration doesn't have some of these paths:" + + " initialDelay, interval, cities, keyWords") + + val initialDelay: Option[String] = configuration.getOptional[String]("initialDelay") + val interval: Option[String] = configuration.getOptional[String]("interval") + val cities: Option[Seq[String]] = configuration.getOptional[Seq[String]]("cities") + val keyWords: Option[Seq[String]] = configuration.getOptional[Seq[String]]("keyWords") + + if (initialDelay.isEmpty || interval.isEmpty || cities.isEmpty || keyWords.isEmpty) + throw new ClassCastException("Some of these paths have wrong type: initialDelay, interval, cities, keyWords") + + val initDelay: Try[FiniteDuration] = Try(Duration(initialDelay.get).asInstanceOf[FiniteDuration]) + val interv: Try[FiniteDuration] = Try(Duration(interval.get).asInstanceOf[FiniteDuration]) + + if(initDelay.isFailure || interv.isFailure) + throw new ClassCastException("Initial delay or interval have wrong format") - actorSystem.scheduler.scheduleAtFixedRate(initialDelay = Duration(initialDelay).asInstanceOf[FiniteDuration], - interval = Duration(interval).asInstanceOf[FiniteDuration]) { () => - jobAggregatorService.parse(keyWords, cities) - println("Scheduled task executed") - //actorSystem.log.info("Executing something...") + actorSystem.scheduler.scheduleAtFixedRate(initialDelay = initDelay.get, + interval = interv.get) { () => + jobAggregatorService.aggregateData(keyWords.get, cities.get) + Logger("play").info("Scheduled task executed") } } diff --git a/app/service/JobAggregatorService.scala b/app/service/JobAggregatorService.scala index b430b2f..737d2ca 100644 --- a/app/service/JobAggregatorService.scala +++ b/app/service/JobAggregatorService.scala @@ -1,32 +1,41 @@ package service +import DTOs._ import model.Job import model.db.DBTables.jobTable import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} import slick.jdbc.JdbcProfile import slick.jdbc.PostgresProfile.api._ - import javax.inject.{Inject, Singleton} +import play.api.{Configuration, Logger, Mode} import play.api.libs.ws._ -import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup import play.api.libs.json._ +import scala.util.{Failure, Success, Try} import scala.concurrent.ExecutionContext.Implicits.global import scala.collection.mutable.ListBuffer import scala.concurrent.{Await, Future} + @Singleton class JobAggregatorService @Inject()(ws: WSClient, + configuration: Configuration, val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] { - val perPage: Int = 100 + val perPage: Int = configuration.getOptional[Int]("perPage") match { + case Some(value) => value + case None => + Logger("play").warn("perPage is not configured. The default value(100) will be used") + 100 + } /** - * добавляет в БД все вакансии по ключевому слову и региону + * агрегирует вакансии по ключевому слову и региону + * * @param text ключевое слов * @param area индекс региона */ - def parse(text: String, area: Int) = { + def aggregateData(text: String, area: Int) = { ws.url(s"https://api.hh.ru/vacancies?text=$text&area=$area&per_page=$perPage&page=0") .get() @@ -35,73 +44,105 @@ class JobAggregatorService @Inject()(ws: WSClient, } /** - * добавляет в БД все вакансии по всем ключевым словам и регионам + * агрегирует вакансии по ключевому слову и региону + * * @param keyWords ключевые слова - * @param areas индексы регионов + * @param areas индексы регионов */ - def parse(keyWords: Seq[String], areas: Seq[String]):Unit = { - val regionsMap = getRegions().map(x => x._2) - for { - keyWord <- keyWords - area <- areas - } regionsMap.map(x => parse(keyWord, x(area))) + def aggregateData(keyWords: Seq[String], areas: Seq[String]): Unit = { + getRegions() match { + case Success(value) => { + val regionsMap = value.map(x => x._2) + for { + keyWord <- keyWords + area <- areas + } regionsMap.map(x => aggregateData(keyWord, x(area))) + } + case Failure(exception) => Logger("play").error(exception.getMessage, exception) + } } /** * делает несколько запросов к hh.ru, если все вакансии не помещаются в один ответ + * * @param firstResp первый ответ от hh.ru * @return Future от массива всех полученных вакансий */ - private def getJobs(firstResp: JsValue, keyWord: String, area: Int) = { + private def getJobs(firstResp: JsValue, keyWord: String, area: Int): Future[List[Job]] = { + + def handleJobs(jobs: Try[List[Job]]) = { + jobs match { + case Success(value) => value + case Failure(exception) => Logger("play").error(exception.getMessage, exception) + Nil + } + } def parseAllPages(areaText: String) = { - val pages = (firstResp \ "pages").get.toString.toInt - val jobs: ListBuffer[Job] = getJobsFromPage(firstResp, keyWord, areaText) + val pages = firstResp.asOpt[HolderDTO] match { + case Some(holder) => holder.pages + case None => throw new ClassCastException("json can't be parsed: " + firstResp.toString) + } + val jobs: List[Job] = handleJobs(getJobsFromPage(firstResp, keyWord, areaText)) - val futures = for (n <- 1 until pages) + val requests = for (n <- 1 until pages) yield ws.url(s"https://api.hh.ru/vacancies?text=$keyWord&area=$area&per_page=$perPage&page=$n") - .get().map(resp => getJobsFromPage(resp.json, keyWord, areaText)) - Future.foldLeft(futures)(jobs)((acc, e) => acc ++ e) + requests.foldLeft(Future.successful(jobs))((fut, req) => + req.get().map(resp => handleJobs(getJobsFromPage(resp.json, keyWord, areaText))) + .flatMap(newJobs => fut.map(oldJobs => oldJobs ++ newJobs))) } - getRegions().flatMap(x => parseAllPages(x._1(area))) + getRegions() match { + case Success(value) => value.flatMap(x => parseAllPages(x._1(area))) + case Failure(exception) => Logger("play").error(exception.getMessage, exception) + Future.successful(Nil) + } } /** * парсит вакансии в массив DTO Job + * * @param json json, содержащий массив вакансий * @return массив DTO Job */ - private def getJobsFromPage(json: JsValue, keyWord: String, area: String): ListBuffer[Job] = { + private def getJobsFromPage(json: JsValue, keyWord: String, area: String): Try[List[Job]] = { var jobs = ListBuffer[Job]() - val items = (json \ "items").as[Seq[JsValue]] - - for (item <- items) { - val id = (item \ "id").as[String].toInt - val name = (item \ "name").asOpt[String] - val alternate_url = (item \ "alternate_url").asOpt[String] - val requirement = (item \ "snippet" \ "requirement").asOpt[String] - val responsibility = (item \ "snippet" \ "responsibility").asOpt[String] - - val salary = (item \ "salary").get - var from: Option[Int] = None - var to: Option[Int] = None - var currency: Option[String] = None - var gross: Option[Boolean] = None - - if (salary != JsNull) { - currency = (salary \ "currency").asOpt[String] - from = (salary \ "from").asOpt[Int] - gross = (salary \ "gross").asOpt[Boolean] - to = (salary \ "to").asOpt[Int] + + def addJob(item: VacancyDTO): Unit = { + var requirement: Option[String] = None + var responsibility: Option[String] = None + + item.snippet match { + case Some(value) => { + requirement = value.requirement + responsibility = value.responsibility + } + case _ => + } + + Try(item.id.toInt).toOption match { + case Some(id) => item.salary match { + case Some(salary) => jobs += Job(id, item.name, requirement, responsibility, + item.alternate_url, salary.from, salary.to, salary.currency, salary.gross, Option(area), Option(keyWord)) + case None => jobs += Job(id, item.name, requirement, responsibility, + item.alternate_url, None, None, None, None, Option(area), Option(keyWord)) + } + case None => throw new NumberFormatException("id value can't be converted to int" + json.toString()) } - val job = - Job(id, name, requirement, responsibility, alternate_url, from, to, currency, gross, Option(area), Option(keyWord)) - jobs += job } - jobs + Try { + json.asOpt[HolderDTO] match { + case Some(holder) => holder.items match { + case Some(items) => items.foreach(addJob) + case None => throw new RuntimeException("Items doesn't exist" + json.toString()) + } + case None => throw new RuntimeException("Can't be parsed to HolderDTO" + json.toString()) + } + + jobs.toList + } } /** @@ -112,38 +153,32 @@ class JobAggregatorService @Inject()(ws: WSClient, val indexToRegion = scala.collection.mutable.Map[Int, String]() val regionToIndex = scala.collection.mutable.Map[String, Int]() + def initialParse(json: JsValue) = { - for (n <- json.as[Seq[JsValue]]) { - parseRegions(n) + json.asOpt[Seq[RegionDTO]] match { + case Some(regions) => regions.foreach(parseRegions) + case None => throw new ClassCastException("Can't be parsed to Seq[RegionDTO]: " + json.toString()) } - (indexToRegion, regionToIndex) + (indexToRegion.toMap, regionToIndex.toMap) } - def parseRegions(regionJson: JsValue): Unit = { - val areas = (regionJson \ "areas").as[Seq[JsValue]] - val id = (regionJson \ "id").asOpt[String] - val name = (regionJson \ "name").asOpt[String] - - if (id.isDefined && name.isDefined) { - indexToRegion += (id.get.toInt -> name.get) - regionToIndex += (name.get -> id.get.toInt) - } - - if (areas.nonEmpty) { - for (n <- areas) - parseRegions(n) + def parseRegions(regionDTO: RegionDTO): Unit = { + Try(regionDTO.id.toInt).toOption match { + case Some(id) => { + indexToRegion += (id -> regionDTO.name) + regionToIndex += (regionDTO.name -> id) + if (regionDTO.areas.nonEmpty) { + regionDTO.areas.foreach(area => parseRegions(area)) + } + } + case None => throw new NumberFormatException("region id can't be converted to int") } } - resp.map(x => initialParse(x.json)) + Try(resp.map(x => initialParse(x.json))) } private def addToDB(job: Job) = { db.run(DBIO.seq(jobTable += job)) } - - private def addToDBSeq(jobs: Seq[Job]) = { - db.run(DBIO.seq(jobTable ++= jobs)) - //Await.result(db.run(DBIO.seq(salaryTable ++= salaries, jobTable ++= jobs)), Duration.Inf) - } } diff --git a/conf/application.conf b/conf/application.conf index dbc1321..d274c80 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -9,4 +9,5 @@ slick.dbs.default.db.password="12345678" initialDelay="10 seconds" interval="30 seconds" cities=["Москва","Екатеринбург"] -keyWords=["java","scala"] \ No newline at end of file +keyWords=["java","scala"] +perPage=100 \ No newline at end of file From 9bfb096c5456c756db65e4648cd961201f6fa206 Mon Sep 17 00:00:00 2001 From: EvgeniyPukanovich Date: Thu, 30 Jun 2022 20:08:51 +0500 Subject: [PATCH 4/4] final changes --- app/controllers/JobAggregatorController.scala | 2 +- app/scheduler/Task.scala | 38 ++++++++++--------- app/service/JobAggregatorService.scala | 19 +++++----- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/app/controllers/JobAggregatorController.scala b/app/controllers/JobAggregatorController.scala index 2a3d0b0..79e90c6 100644 --- a/app/controllers/JobAggregatorController.scala +++ b/app/controllers/JobAggregatorController.scala @@ -14,6 +14,6 @@ class JobAggregatorController @Inject()(val controllerComponents: ControllerComp def index(text: String, area: Int) = Action.async { implicit request: Request[AnyContent] => jobAggregatorService.aggregateData(text, area).map(_ => Ok("")) - .recover(exception => InternalServerError("Following exception has occurred: " + exception.getMessage)) + .recover(exception => InternalServerError(s"Following exception has occurred: ${exception.getMessage}")) } } diff --git a/app/scheduler/Task.scala b/app/scheduler/Task.scala index 53c3ce6..4a0cc1d 100644 --- a/app/scheduler/Task.scala +++ b/app/scheduler/Task.scala @@ -6,6 +6,7 @@ import akka.actor.ActorSystem import scala.concurrent.ExecutionContext import scala.concurrent.duration._ import play.api.{Configuration, Logger, Mode} +import play.libs.XML.Constants import service.JobAggregatorService import scala.util.Try @@ -14,28 +15,29 @@ class Task @Inject()(actorSystem: ActorSystem, configuration: Configuration, jobAggregatorService: JobAggregatorService)(implicit executionContext: ExecutionContext) { - if (!configuration.has("initialDelay") || !configuration.has("interval") - || !configuration.has("cities") || !configuration.has("keyWords")) - throw new NoSuchFieldException("Configuration doesn't have some of these paths:" + - " initialDelay, interval, cities, keyWords") + val logger: Logger = Logger("Scheduler") - val initialDelay: Option[String] = configuration.getOptional[String]("initialDelay") - val interval: Option[String] = configuration.getOptional[String]("interval") - val cities: Option[Seq[String]] = configuration.getOptional[Seq[String]]("cities") - val keyWords: Option[Seq[String]] = configuration.getOptional[Seq[String]]("keyWords") + val initialDelay: String = configuration.getOptional[String]("initialDelay") + .getOrElse(sys.error("No \"initialDelay\" field found or it has wrong type")) - if (initialDelay.isEmpty || interval.isEmpty || cities.isEmpty || keyWords.isEmpty) - throw new ClassCastException("Some of these paths have wrong type: initialDelay, interval, cities, keyWords") + val interval: String = configuration.getOptional[String]("interval") + .getOrElse(sys.error("No \"interval\" field found or it has wrong type")) - val initDelay: Try[FiniteDuration] = Try(Duration(initialDelay.get).asInstanceOf[FiniteDuration]) - val interv: Try[FiniteDuration] = Try(Duration(interval.get).asInstanceOf[FiniteDuration]) + val cities: Seq[String] = configuration.getOptional[Seq[String]]("cities") + .getOrElse(sys.error("No \"cities\" field found or it has wrong type")) - if(initDelay.isFailure || interv.isFailure) - throw new ClassCastException("Initial delay or interval have wrong format") + val keyWords: Seq[String] = configuration.getOptional[Seq[String]]("keyWords") + .getOrElse(sys.error("No \"keyWords\" field found or it has wrong type")) - actorSystem.scheduler.scheduleAtFixedRate(initialDelay = initDelay.get, - interval = interv.get) { () => - jobAggregatorService.aggregateData(keyWords.get, cities.get) - Logger("play").info("Scheduled task executed") + + val initDelay: FiniteDuration = Try(Duration(initialDelay).asInstanceOf[FiniteDuration]) + .getOrElse(sys.error("\"initialDelay\" field has wrong format")) + val interv: FiniteDuration = Try(Duration(interval).asInstanceOf[FiniteDuration]) + .getOrElse(sys.error("\"interval\" field has wrong format")) + + actorSystem.scheduler.scheduleAtFixedRate(initialDelay = initDelay, + interval = interv) { () => + jobAggregatorService.aggregateData(keyWords, cities) + logger.info("Scheduled task executed") } } diff --git a/app/service/JobAggregatorService.scala b/app/service/JobAggregatorService.scala index 737d2ca..9c3220b 100644 --- a/app/service/JobAggregatorService.scala +++ b/app/service/JobAggregatorService.scala @@ -22,10 +22,11 @@ class JobAggregatorService @Inject()(ws: WSClient, configuration: Configuration, val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] { + val logger: Logger = Logger("JobAggregatorService") val perPage: Int = configuration.getOptional[Int]("perPage") match { case Some(value) => value case None => - Logger("play").warn("perPage is not configured. The default value(100) will be used") + logger.warn("perPage is not configured. The default value(100) will be used") 100 } @@ -39,8 +40,8 @@ class JobAggregatorService @Inject()(ws: WSClient, ws.url(s"https://api.hh.ru/vacancies?text=$text&area=$area&per_page=$perPage&page=0") .get() - .flatMap(x => getJobs(x.json, text, area)) - .map(buff => buff.foreach(x => addToDB(x))) + .flatMap(response => getJobs(response.json, text, area)) + .map(buff => buff.foreach(job => addToDB(job))) } /** @@ -58,7 +59,7 @@ class JobAggregatorService @Inject()(ws: WSClient, area <- areas } regionsMap.map(x => aggregateData(keyWord, x(area))) } - case Failure(exception) => Logger("play").error(exception.getMessage, exception) + case Failure(exception) => logger.error(exception.getMessage, exception) } } @@ -73,15 +74,15 @@ class JobAggregatorService @Inject()(ws: WSClient, def handleJobs(jobs: Try[List[Job]]) = { jobs match { case Success(value) => value - case Failure(exception) => Logger("play").error(exception.getMessage, exception) + case Failure(exception) => logger.error(exception.getMessage, exception) Nil } } - def parseAllPages(areaText: String) = { + def parseAllPages(areaText: String): Future[List[Job]] = { val pages = firstResp.asOpt[HolderDTO] match { case Some(holder) => holder.pages - case None => throw new ClassCastException("json can't be parsed: " + firstResp.toString) + case None => return Future.failed(new ClassCastException("json can't be parsed: " + firstResp.toString)) } val jobs: List[Job] = handleJobs(getJobsFromPage(firstResp, keyWord, areaText)) @@ -95,8 +96,8 @@ class JobAggregatorService @Inject()(ws: WSClient, getRegions() match { case Success(value) => value.flatMap(x => parseAllPages(x._1(area))) - case Failure(exception) => Logger("play").error(exception.getMessage, exception) - Future.successful(Nil) + case Failure(exception) => logger.error(exception.getMessage, exception) + Future.failed(exception) } }