Skip to content

Commit

Permalink
fix: Ipad 크롤링 오류 해결
Browse files Browse the repository at this point in the history
  • Loading branch information
cookienc committed Jun 21, 2024
1 parent 97a48be commit d00ac05
Show file tree
Hide file tree
Showing 6 changed files with 112 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class CrawlService(

fun crawlIpad(): List<Ipad> {
val url = getCrawlUrl(CrawlTargetCategory.IPAD)
val products = crawler.crawl(url)
val products = crawler.crawlWithClick(url)
return crawlMapper.toIpad(products)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import org.springframework.stereotype.Component
private const val IPAD_AIR_5_GEN = "iPad Air 5세대"

@Component
class IpadAirMapper : IpadMappingComponent {
class IpadAir5Mapper : IpadMappingComponent {

override fun supports(subCategory: String): Boolean {
return IPAD_AIR_5_GEN == subCategory
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package backend.itracker.crawl.service.mapper.ipad

import backend.itracker.crawl.ipad.domain.Ipad
import backend.itracker.crawl.ipad.domain.IpadCategory
import backend.itracker.crawl.service.response.IpadCrawlResponse
import backend.itracker.crawl.service.vo.DefaultProduct
import org.springframework.stereotype.Component

@Component
class IpadAirM2Mapper : IpadMappingComponent {

override fun supports(subCategory: String): Boolean {
return subCategory.contains("iPad Air M2")
}

override fun toDomain(product: DefaultProduct): Ipad {
val names = product.name.split(",")
.map { it.trim() }
.toList()

val title = names[0].split(" ")
val company = title[0]
val releaseYear = title[2]
val category = IpadCategory.I_PAD_AIR
val chip = "M2"
val size = title[5]
val generation = "6"
val color = names[1]
val storage = names[2]
val isCellular = names[3].contains("Cellular")

return IpadCrawlResponse(
coupangId = product.productId,
company = company,
name = product.name,
category = category,
chip = chip,
storage = storage,
color = color,
size = size,
releaseYear = releaseYear.toInt(),
isCellular = isCellular,
generation = generation,
productLink = product.productLink,
thumbnail = product.thumbnailLink,
basePrice = product.price.basePrice,
discountPercentage = product.price.discountPercentage,
currentPrice = product.price.discountPrice,
isOutOfStock = product.price.isOutOfStock
).toDomain()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package backend.itracker.crawl.service.mapper.ipad

import backend.itracker.crawl.ipad.domain.Ipad
import backend.itracker.crawl.ipad.domain.IpadCategory
import backend.itracker.crawl.service.response.IpadCrawlResponse
import backend.itracker.crawl.service.vo.DefaultProduct
import org.springframework.stereotype.Component

@Component
class IpadProM4Mapper : IpadMappingComponent {

override fun supports(subCategory: String): Boolean {
return subCategory.contains("iPad Pro M4")
}

override fun toDomain(product: DefaultProduct): Ipad {
val names = product.name.split(",")
.map { it.trim() }
.toList()

val title = names[0].split(" ")
val company = title[0]
val releaseYear = title[2]
val category = IpadCategory.I_PAD_PRO
val chip = "M4"
val size = title[5]
val generation = "5"
val glass = title[7]
val color = names[1]
val storage = names[2]
val isCellular = names[3].contains("Cellular")

return IpadCrawlResponse(
coupangId = product.productId,
company = company,
name = product.name,
category = category,
chip = chip,
storage = storage,
color = color,
size = size,
glass = glass,
releaseYear = releaseYear.toInt(),
isCellular = isCellular,
generation = generation,
productLink = product.productLink,
thumbnail = product.thumbnailLink,
basePrice = product.price.basePrice,
discountPercentage = product.price.discountPercentage,
currentPrice = product.price.discountPrice,
isOutOfStock = product.price.isOutOfStock
).toDomain()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ data class IpadCrawlResponse(
val storage: String,
val color: String,
val size: String,
val glass: String = "",
val releaseYear: Int,
val isCellular: Boolean,
val generation: String,
Expand All @@ -33,6 +34,7 @@ data class IpadCrawlResponse(
storage: String,
color: String,
size: String,
glass: String = "",
releaseYear: String,
isCellular: Boolean,
generation: String,
Expand All @@ -51,6 +53,7 @@ data class IpadCrawlResponse(
storage = storage,
color = color,
size = size,
glass = glass,
releaseYear = releaseYear.toInt(),
isCellular = isCellular,
generation = generation,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class Crawler(
val targetButton = buttons.firstOrNull {
helper.findGrandParentElement(it)
.findElement(By.className("carousel-header__title")).text == subcategory
} ?: throw CrawlException("해당 subcategory의 버튼을 찾을 수 없습니다. subcategory: $subcategory")
} ?: return products

var clickCount = 0
while (clickCount++ < maxClicks) {
Expand Down

0 comments on commit d00ac05

Please sign in to comment.