diff --git a/build.gradle b/build.gradle index 46de497..080c8ab 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,10 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-batch' // spring-batch-core 최신버전 추가 implementation 'org.springframework.batch:spring-batch-core:5.1.0' + // Webflux + implementation 'org.springframework.boot:spring-boot-starter-webflux' + // mac netty 설정 + implementation 'io.netty:netty-resolver-dns-native-macos:4.1.68.Final:osx-aarch_64' // QueryDSL - 4개 implementation 'com.querydsl:querydsl-jpa:5.0.0:jakarta' annotationProcessor "com.querydsl:querydsl-apt:${dependencyManagement.importedProperties['querydsl.version']}:jakarta" @@ -55,6 +59,7 @@ dependencies { annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.springframework.security:spring-security-test' + testImplementation 'org.springframework:spring-webflux' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } diff --git a/src/main/java/wanted/ribbon/genrestrt/component/DataFetchTasklet.java b/src/main/java/wanted/ribbon/datapipe/component/DataFetchTasklet.java similarity index 96% rename from src/main/java/wanted/ribbon/genrestrt/component/DataFetchTasklet.java rename to src/main/java/wanted/ribbon/datapipe/component/DataFetchTasklet.java index 4e38eed..21cd9a1 100644 --- a/src/main/java/wanted/ribbon/genrestrt/component/DataFetchTasklet.java +++ b/src/main/java/wanted/ribbon/datapipe/component/DataFetchTasklet.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.component; +package wanted.ribbon.datapipe.component; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,10 +11,10 @@ import org.springframework.stereotype.Component; import wanted.ribbon.exception.BaseException; import wanted.ribbon.exception.ErrorCode; -import wanted.ribbon.genrestrt.dto.RawData; -import wanted.ribbon.genrestrt.mapper.RawDataRowMapper; -import wanted.ribbon.genrestrt.mapper.StoreDataRowMapper; -import wanted.ribbon.genrestrt.service.DataProcessor; +import wanted.ribbon.datapipe.dto.RawData; +import wanted.ribbon.datapipe.mapper.RawDataRowMapper; +import wanted.ribbon.datapipe.mapper.StoreDataRowMapper; +import wanted.ribbon.datapipe.service.DataProcessor; import wanted.ribbon.store.domain.Store; import java.util.ArrayList; diff --git a/src/main/java/wanted/ribbon/genrestrt/component/DataPipeJobScheduler.java b/src/main/java/wanted/ribbon/datapipe/component/DataPipeJobScheduler.java similarity index 92% rename from src/main/java/wanted/ribbon/genrestrt/component/DataPipeJobScheduler.java rename to src/main/java/wanted/ribbon/datapipe/component/DataPipeJobScheduler.java index 45d3eee..2cb60aa 100644 --- a/src/main/java/wanted/ribbon/genrestrt/component/DataPipeJobScheduler.java +++ b/src/main/java/wanted/ribbon/datapipe/component/DataPipeJobScheduler.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.component; +package wanted.ribbon.datapipe.component; import lombok.RequiredArgsConstructor; import org.springframework.batch.core.JobParametersBuilder; @@ -9,7 +9,7 @@ import org.springframework.batch.core.repository.JobRestartException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import wanted.ribbon.genrestrt.config.DataPipeJobConfig; +import wanted.ribbon.datapipe.config.DataPipeJobConfig; @Component @RequiredArgsConstructor diff --git a/src/main/java/wanted/ribbon/genrestrt/component/DataPipeTasklet.java b/src/main/java/wanted/ribbon/datapipe/component/DataPipeTasklet.java similarity index 95% rename from src/main/java/wanted/ribbon/genrestrt/component/DataPipeTasklet.java rename to src/main/java/wanted/ribbon/datapipe/component/DataPipeTasklet.java index e40d1c8..7d8f208 100644 --- a/src/main/java/wanted/ribbon/genrestrt/component/DataPipeTasklet.java +++ b/src/main/java/wanted/ribbon/datapipe/component/DataPipeTasklet.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.component; +package wanted.ribbon.datapipe.component; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,9 +11,9 @@ import org.springframework.stereotype.Component; import wanted.ribbon.exception.BaseException; import wanted.ribbon.exception.ErrorCode; -import wanted.ribbon.genrestrt.dto.RawData; -import wanted.ribbon.genrestrt.mapper.RawDataRowMapper; -import wanted.ribbon.genrestrt.service.DataProcessor; +import wanted.ribbon.datapipe.dto.RawData; +import wanted.ribbon.datapipe.mapper.RawDataRowMapper; +import wanted.ribbon.datapipe.service.DataProcessor; import wanted.ribbon.store.domain.Store; import java.util.List; diff --git a/src/main/java/wanted/ribbon/datapipe/config/AppConfig.java b/src/main/java/wanted/ribbon/datapipe/config/AppConfig.java new file mode 100644 index 0000000..712493c --- /dev/null +++ b/src/main/java/wanted/ribbon/datapipe/config/AppConfig.java @@ -0,0 +1,44 @@ +package wanted.ribbon.datapipe.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.util.DefaultUriBuilderFactory; + +@Configuration +public class AppConfig { + @Value("${spring.public-api.base-url}") + private String baseUrl; + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + /* + * WebClient의 UriComponentsBuilder.encode() 방식의 인코딩 방지 + * key 값이 달라지는 것을 방지하기 위해 DefaultUriBuilderFactory 생성 + * encoding 모드 지정 + * */ + @Bean + public DefaultUriBuilderFactory builderFactory() { + DefaultUriBuilderFactory builderFactory = new DefaultUriBuilderFactory(); + builderFactory.setEncodingMode(DefaultUriBuilderFactory.EncodingMode.VALUES_ONLY); + return builderFactory; + } + + @Bean + public WebClient webClient() { + return WebClient.builder() + .uriBuilderFactory(builderFactory()) + .build(); + } +} diff --git a/src/main/java/wanted/ribbon/genrestrt/config/DataPipeJobConfig.java b/src/main/java/wanted/ribbon/datapipe/config/DataPipeJobConfig.java similarity index 93% rename from src/main/java/wanted/ribbon/genrestrt/config/DataPipeJobConfig.java rename to src/main/java/wanted/ribbon/datapipe/config/DataPipeJobConfig.java index f531f0f..9e83662 100644 --- a/src/main/java/wanted/ribbon/genrestrt/config/DataPipeJobConfig.java +++ b/src/main/java/wanted/ribbon/datapipe/config/DataPipeJobConfig.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.config; +package wanted.ribbon.datapipe.config; import lombok.RequiredArgsConstructor; import org.springframework.batch.core.Job; @@ -12,8 +12,8 @@ import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.transaction.PlatformTransactionManager; -import wanted.ribbon.genrestrt.component.DataFetchTasklet; -import wanted.ribbon.genrestrt.component.DataPipeTasklet; +import wanted.ribbon.datapipe.component.DataFetchTasklet; +import wanted.ribbon.datapipe.component.DataPipeTasklet; @RequiredArgsConstructor @Configuration diff --git a/src/main/java/wanted/ribbon/datapipe/controller/OpenApiController.java b/src/main/java/wanted/ribbon/datapipe/controller/OpenApiController.java new file mode 100644 index 0000000..c13a33b --- /dev/null +++ b/src/main/java/wanted/ribbon/datapipe/controller/OpenApiController.java @@ -0,0 +1,38 @@ +package wanted.ribbon.datapipe.controller; + + +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import reactor.core.publisher.Mono; +import wanted.ribbon.datapipe.dto.GyeongGiList; +import wanted.ribbon.datapipe.service.GenrestrtService; +import wanted.ribbon.datapipe.service.RawDataService; + +@RestController +@RequestMapping("/api/datapipes") +@RequiredArgsConstructor +public class OpenApiController { + private final GenrestrtService genrestrtService; + private final RawDataService rawDataService; + + // 경기도 맛집 데이터 수집 API (RestTemplate), responsebody 사용으로 PostMapping으로 진행 + @PostMapping("/fetch-data") + public ResponseEntity fetchData(@RequestParam("serviceName") String serviceName) { + // openAPI 호출 + genrestrtService.fetchAndSaveData(serviceName); + return ResponseEntity.ok(serviceName + "가 db에 성공적으로 저장됐습니다."); + } + + // 경기도 맛집 데이터 수집 API (WebClient, 모든 경기도 맛집 api 가능) + /** + * 데이터 조회로 GetMapping 사용 + * 비동기 처리를 위한 Mono 사용 + */ + @GetMapping("/fetch-and-save") + public Mono> fetchAndSaveData(@RequestParam("serviceName") String serviceName) { + return rawDataService.getAndSaveByServiceName(serviceName) + .map(ResponseEntity::ok) + .defaultIfEmpty(ResponseEntity.notFound().build()); + } +} diff --git a/src/main/java/wanted/ribbon/genrestrt/domain/Genrestrt.java b/src/main/java/wanted/ribbon/datapipe/domain/Genrestrt.java similarity index 97% rename from src/main/java/wanted/ribbon/genrestrt/domain/Genrestrt.java rename to src/main/java/wanted/ribbon/datapipe/domain/Genrestrt.java index 672107d..62f069b 100644 --- a/src/main/java/wanted/ribbon/genrestrt/domain/Genrestrt.java +++ b/src/main/java/wanted/ribbon/datapipe/domain/Genrestrt.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.domain; +package wanted.ribbon.datapipe.domain; import jakarta.persistence.*; import lombok.*; diff --git a/src/main/java/wanted/ribbon/genrestrt/dto/GenrestrtApiResponse.java b/src/main/java/wanted/ribbon/datapipe/dto/GenrestrtApiResponse.java similarity index 98% rename from src/main/java/wanted/ribbon/genrestrt/dto/GenrestrtApiResponse.java rename to src/main/java/wanted/ribbon/datapipe/dto/GenrestrtApiResponse.java index 80cef78..c8d28ee 100644 --- a/src/main/java/wanted/ribbon/genrestrt/dto/GenrestrtApiResponse.java +++ b/src/main/java/wanted/ribbon/datapipe/dto/GenrestrtApiResponse.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.dto; +package wanted.ribbon.datapipe.dto; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; diff --git a/src/main/java/wanted/ribbon/datapipe/dto/GyeongGiList.java b/src/main/java/wanted/ribbon/datapipe/dto/GyeongGiList.java new file mode 100644 index 0000000..ca82489 --- /dev/null +++ b/src/main/java/wanted/ribbon/datapipe/dto/GyeongGiList.java @@ -0,0 +1,32 @@ +package wanted.ribbon.datapipe.dto; + +import java.util.Date; +import java.util.List; + +public record GyeongGiList(String message, + Long total, + List gyeongGiApiResponses) { + public record GyeongGiApiResponse(String sigunNm, + String sigunCd, + String bizplcNm, + Date licensgDe, + String bsnStateNm, + Date clsbizDe, + Double locplcAr, + String gradFacltDivNm, + Long maleEnflpsnCnt, + Integer yy, + String multiUseBizestblYn, + String gradDivNm, + Double totFacltScale, + Long femaleEnflpsnCnt, + String bsnsiteCircumfrDivNm, + String sanittnIndutypeNm, + String sanittnBizcondNm, + Long totEmplyCnt, + String refineRoadnmAddr, + String refineLotnoAddr, + String refineZipCd, + Double refineWgs84Lat, + Double refineWgs84Logt){} +} diff --git a/src/main/java/wanted/ribbon/genrestrt/dto/RawData.java b/src/main/java/wanted/ribbon/datapipe/dto/RawData.java similarity index 92% rename from src/main/java/wanted/ribbon/genrestrt/dto/RawData.java rename to src/main/java/wanted/ribbon/datapipe/dto/RawData.java index 0cf030b..27c8227 100644 --- a/src/main/java/wanted/ribbon/genrestrt/dto/RawData.java +++ b/src/main/java/wanted/ribbon/datapipe/dto/RawData.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.dto; +package wanted.ribbon.datapipe.dto; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/src/main/java/wanted/ribbon/genrestrt/mapper/RawDataRowMapper.java b/src/main/java/wanted/ribbon/datapipe/mapper/RawDataRowMapper.java similarity index 90% rename from src/main/java/wanted/ribbon/genrestrt/mapper/RawDataRowMapper.java rename to src/main/java/wanted/ribbon/datapipe/mapper/RawDataRowMapper.java index 853d2bb..db99264 100644 --- a/src/main/java/wanted/ribbon/genrestrt/mapper/RawDataRowMapper.java +++ b/src/main/java/wanted/ribbon/datapipe/mapper/RawDataRowMapper.java @@ -1,7 +1,7 @@ -package wanted.ribbon.genrestrt.mapper; +package wanted.ribbon.datapipe.mapper; import org.springframework.jdbc.core.RowMapper; -import wanted.ribbon.genrestrt.dto.RawData; +import wanted.ribbon.datapipe.dto.RawData; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/src/main/java/wanted/ribbon/genrestrt/mapper/StoreDataRowMapper.java b/src/main/java/wanted/ribbon/datapipe/mapper/StoreDataRowMapper.java similarity index 95% rename from src/main/java/wanted/ribbon/genrestrt/mapper/StoreDataRowMapper.java rename to src/main/java/wanted/ribbon/datapipe/mapper/StoreDataRowMapper.java index a0ae99f..6b16f61 100644 --- a/src/main/java/wanted/ribbon/genrestrt/mapper/StoreDataRowMapper.java +++ b/src/main/java/wanted/ribbon/datapipe/mapper/StoreDataRowMapper.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.mapper; +package wanted.ribbon.datapipe.mapper; import org.springframework.jdbc.core.RowMapper; import wanted.ribbon.store.domain.Category; diff --git a/src/main/java/wanted/ribbon/genrestrt/repository/GenrestrtRepository.java b/src/main/java/wanted/ribbon/datapipe/repository/GenrestrtRepository.java similarity index 60% rename from src/main/java/wanted/ribbon/genrestrt/repository/GenrestrtRepository.java rename to src/main/java/wanted/ribbon/datapipe/repository/GenrestrtRepository.java index 5a1c5fd..bd3e5c0 100644 --- a/src/main/java/wanted/ribbon/genrestrt/repository/GenrestrtRepository.java +++ b/src/main/java/wanted/ribbon/datapipe/repository/GenrestrtRepository.java @@ -1,7 +1,7 @@ -package wanted.ribbon.genrestrt.repository; +package wanted.ribbon.datapipe.repository; import org.springframework.data.jpa.repository.JpaRepository; -import wanted.ribbon.genrestrt.domain.Genrestrt; +import wanted.ribbon.datapipe.domain.Genrestrt; public interface GenrestrtRepository extends JpaRepository { } diff --git a/src/main/java/wanted/ribbon/genrestrt/service/DataProcessor.java b/src/main/java/wanted/ribbon/datapipe/service/DataProcessor.java similarity index 95% rename from src/main/java/wanted/ribbon/genrestrt/service/DataProcessor.java rename to src/main/java/wanted/ribbon/datapipe/service/DataProcessor.java index ff9663b..f838692 100644 --- a/src/main/java/wanted/ribbon/genrestrt/service/DataProcessor.java +++ b/src/main/java/wanted/ribbon/datapipe/service/DataProcessor.java @@ -1,8 +1,8 @@ -package wanted.ribbon.genrestrt.service; +package wanted.ribbon.datapipe.service; import org.springframework.batch.item.ItemProcessor; import org.springframework.stereotype.Component; -import wanted.ribbon.genrestrt.dto.RawData; +import wanted.ribbon.datapipe.dto.RawData; import wanted.ribbon.store.domain.Category; import wanted.ribbon.store.domain.Store; diff --git a/src/main/java/wanted/ribbon/genrestrt/service/GenrestrtService.java b/src/main/java/wanted/ribbon/datapipe/service/GenrestrtService.java similarity index 98% rename from src/main/java/wanted/ribbon/genrestrt/service/GenrestrtService.java rename to src/main/java/wanted/ribbon/datapipe/service/GenrestrtService.java index f04c019..54327f1 100644 --- a/src/main/java/wanted/ribbon/genrestrt/service/GenrestrtService.java +++ b/src/main/java/wanted/ribbon/datapipe/service/GenrestrtService.java @@ -1,4 +1,4 @@ -package wanted.ribbon.genrestrt.service; +package wanted.ribbon.datapipe.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -10,9 +10,9 @@ import org.springframework.web.client.RestTemplate; import wanted.ribbon.exception.ErrorCode; import wanted.ribbon.exception.GenrestrtException; -import wanted.ribbon.genrestrt.domain.Genrestrt; -import wanted.ribbon.genrestrt.dto.GenrestrtApiResponse; -import wanted.ribbon.genrestrt.repository.GenrestrtRepository; +import wanted.ribbon.datapipe.domain.Genrestrt; +import wanted.ribbon.datapipe.dto.GenrestrtApiResponse; +import wanted.ribbon.datapipe.repository.GenrestrtRepository; import java.text.ParseException; import java.text.SimpleDateFormat; diff --git a/src/main/java/wanted/ribbon/datapipe/service/RawDataService.java b/src/main/java/wanted/ribbon/datapipe/service/RawDataService.java new file mode 100644 index 0000000..8ab35d0 --- /dev/null +++ b/src/main/java/wanted/ribbon/datapipe/service/RawDataService.java @@ -0,0 +1,236 @@ +package wanted.ribbon.datapipe.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; +import wanted.ribbon.datapipe.domain.Genrestrt; +import wanted.ribbon.datapipe.dto.GyeongGiList; +import wanted.ribbon.datapipe.repository.GenrestrtRepository; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Service +@Slf4j +@Transactional +public class RawDataService { + private final GenrestrtRepository genrestrtRepository; + private final WebClient webClient; + + @Value("${spring.public-api.gyeong-gi-url}") + private String baseUrl; + @Value("${spring.public-api.service-key}") + private String serviceKey; + + /** + * 주어진 서비스 이름에 따라 데이터를 가져오고 데이터베이스에 저장합니다. + * 이 메소드는 페이지네이션을 사용하여 모든 데이터를 가져옵니다. + * 병렬처리로 성능 향상 + * @param serviceName API 서비스 이름 + * @return 처리된 데이터를 포함한 GyeongGiList의 Mono + */ + public Mono getAndSaveByServiceName(String serviceName) { + return Mono.defer(() -> { + // 페이지 인덱스를 위한 AtomicInteger 초기화 + AtomicInteger pageIndex = new AtomicInteger(1); + // 한 페이지 당 가져올 데이터 수 + int pageSize = 100; + // 응답 저장할 리스트 + List allResponses = new ArrayList<>(); + + return Mono.just(true) + // 1. 모든 페이지의 데이터를 순차적으로 가져오기 + .expand(hasMore -> { + if (!hasMore) { + return Mono.empty(); // 더이상 데이터가 없으면 종료 + } + return fetchAndProcessPage(serviceName, pageIndex.getAndIncrement(), pageSize) + .doOnNext(allResponses::addAll) // 응답 데이터 추가 + .map(responses -> responses.size() == pageSize); // 다음 페이지 존재 여부 확인 + }) + // 2. true인 경우 계속 진행 + .takeWhile(Boolean::booleanValue) + // 3. 데이터베이스 저장 및 서비스이름 전달 + .then(Mono.defer(() -> { + GyeongGiList finalList = new GyeongGiList("모든 데이터가 성공적으로 처리되었습니다.", (long) allResponses.size(), allResponses); + return saveToDatabase(finalList,serviceName); + })) + .doOnSuccess(result -> log.info("데이터 처리 완료: {} 개의 항목 처리됨", result.total())) + .doOnError(error -> log.error("데이터 처리 중 오류 발생", error)); + }); + } + + /** + * 맛집API 페이지의 데이터를 가져와 처리합니다. + * @param serviceName API 서비스 이름 + * @param pageIndex 현재 페이지 인덱스 + * @param pageSize 페이지당 데이터 수 + * @return 처리된 응답 리스트의 Mono + */ + private Mono> fetchAndProcessPage(String serviceName, int pageIndex, int pageSize) { + return webClient.get() + .uri(uriBuilder -> uriBuilder + .scheme("https") + .host(baseUrl) + .path("/" + serviceName) + .queryParam("KEY", serviceKey) + .queryParam("Type", "json") + .queryParam("pIndex", pageIndex) + .queryParam("pSize", pageSize) + .build()) + .retrieve() + .bodyToMono(String.class) + .map(responseBody -> parseStringResponse(responseBody, serviceName)) + .map(GyeongGiList::gyeongGiApiResponses) + .doOnNext(responses -> log.info("페이지 {} 처리 완료: {} 개의 항목", pageIndex, responses.size())); + } + + private GyeongGiList parseStringResponse(String responseBody, String serviceName) { + log.info("API 응답: {}", responseBody); + try { + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(responseBody); + return parseJsonResponse(jsonNode,serviceName); + } catch (JsonProcessingException e) { + log.error("JSON 파싱 중 오류 발생: {}", e.getMessage()); + return new GyeongGiList("JSON 파싱 실패", 0L, Collections.emptyList()); + } + } + + /** + * JSON 응답을 파싱하여 GyeongGiList 객체로 변환합니다. + * @param jsonNode 파싱할 JSON 노드 + * @return 파싱된 GyeongGiList 객체 + */ + private GyeongGiList parseJsonResponse(JsonNode jsonNode, String serviceName) { + List responses = new ArrayList<>(); + JsonNode serviceNode = jsonNode.path(serviceName); + + if (serviceNode.isMissingNode()) { + log.warn("{} 노드를 찾을 수 없습니다.", serviceName); + return new GyeongGiList("데이터 구조 오류", 0L, responses); + } + JsonNode rowsNode = serviceNode.path(1).path("row"); + if (rowsNode.isMissingNode() || !rowsNode.isArray()) { + log.warn("row 배열을 찾을 수 없습니다."); + return new GyeongGiList("데이터 구조 오류", 0L, responses); + } + for (JsonNode row : rowsNode) { + // 각 행의 데이터를 GyeongGiApiResponse 객체로 변환 + try { + GyeongGiList.GyeongGiApiResponse response = new GyeongGiList.GyeongGiApiResponse( + row.path("SIGUN_NM").asText(), + row.path("SIGUN_CD").asText(), + row.path("BIZPLC_NM").asText(), + parseDate(row.path("LICENSG_DE").asText()), + row.path("BSN_STATE_NM").asText(), + parseDate(row.path("CLSBIZ_DE").asText()), + row.path("LOCPLC_AR").asDouble(), + row.path("GRAD_FACLT_DIV_NM").asText(), + row.path("MALE_ENFLPSN_CNT").asLong(), + row.path("YY").asInt(), + row.path("MULTI_USE_BIZESTBL_YN").asText(), + row.path("GRAD_DIV_NM").asText(), + row.path("TOT_FACLT_SCALE").asDouble(), + row.path("FEMALE_ENFLPSN_CNT").asLong(), + row.path("BSNSITE_CIRCUMFR_DIV_NM").asText(), + row.path("SANITTN_INDUTYPE_NM").asText(), + row.path("SANITTN_BIZCOND_NM").asText(), + row.path("TOT_EMPLY_CNT").asLong(), + row.path("REFINE_LOTNO_ADDR").asText(), + row.path("REFINE_ROADNM_ADDR").asText(), + row.path("REFINE_ZIP_CD").asText(), + row.path("REFINE_WGS84_LOGT").asDouble(), + row.path("REFINE_WGS84_LAT").asDouble() + ); + responses.add(response); + } catch (Exception e) { + log.error("행 파싱 중 오류 발생: {}", e.getMessage()); + } + } + return new GyeongGiList("성공적으로 데이터를 파싱했습니다.", (long) responses.size(), responses); + } + + /** + * 데이터를 비동기적으로 데이터베이스에 저장합니다. + * @param gyeongGiList 저장할 GyeongGiList 객체 + * @return 저장된 GyeongGiList의 Mono + */ + private Mono saveToDatabase(GyeongGiList gyeongGiList, String serviceName) { + return Mono.fromCallable(() -> { + List genrestrts = gyeongGiList.gyeongGiApiResponses().stream() + .map(this::convertToGenrestrt) + .collect(Collectors.toList()); + List savedGenrestrts = genrestrtRepository.saveAll(genrestrts); + int savedCount = savedGenrestrts.size(); + String message = serviceName + "의 원본데이터 " + savedCount + "개가 DB에 성공적으로 저장됐습니다."; + return new GyeongGiList(message, (long) savedCount, gyeongGiList.gyeongGiApiResponses()); + }).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * GyeongGiApiResponse 객체를 Genrestrt 엔티티로 변환합니다. + * @param response 변환할 GyeongGiApiResponse 객체 + * @return 변환된 Genrestrt 엔티티 + */ + private Genrestrt convertToGenrestrt(GyeongGiList.GyeongGiApiResponse response) { + return Genrestrt.builder() + .sigunNm(response.sigunNm()) + .sigunCd(response.sigunCd()) + .bizplcNm(response.bizplcNm()) + .licensgDe(response.licensgDe()) + .bsnStateNm(response.bsnStateNm()) + .clsbizDe(response.clsbizDe()) + .locplcAr(response.locplcAr()) + .gradFacltDivNm(response.gradFacltDivNm()) + .maleEnflpsnCnt(response.maleEnflpsnCnt()) + .yy(response.yy()) + .multiUseBizestblYn(response.multiUseBizestblYn()) + .gradDivNm(response.gradDivNm()) + .totFacltScale(response.totFacltScale()) + .femaleEnflpsnCnt(response.femaleEnflpsnCnt()) + .bsnsiteCircumfrDivNm(response.bsnsiteCircumfrDivNm()) + .sanittnIndutypeNm(response.sanittnIndutypeNm()) + .sanittnBizcondNm(response.sanittnBizcondNm()) + .totEmplyCnt(response.totEmplyCnt()) + .refineRoadnmAddr(response.refineRoadnmAddr()) + .refineLotnoAddr(response.refineLotnoAddr()) + .refineZipCd(response.refineZipCd()) + .refineWgs84Lat(response.refineWgs84Lat()) + .refineWgs84Logt(response.refineWgs84Logt()) + .build(); + } + + /** + * 문자열을 Date 객체로 파싱합니다. + * @param dateString 파싱할 날짜 문자열 + * @return 파싱된 Date 객체, 파싱 실패 시 null + */ + private Date parseDate(String dateString) { + if (dateString == null || dateString.isEmpty()) { + return null; + } + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + try { + return format.parse(dateString); + } catch (ParseException e) { + log.error("날짜 파싱 오류: {}", dateString, e); + return null; + } + } +} \ No newline at end of file diff --git a/src/main/java/wanted/ribbon/genrestrt/config/AppConfig.java b/src/main/java/wanted/ribbon/genrestrt/config/AppConfig.java deleted file mode 100644 index 0090b32..0000000 --- a/src/main/java/wanted/ribbon/genrestrt/config/AppConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package wanted.ribbon.genrestrt.config; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; - -@Configuration -public class AppConfig { - - @Bean - public RestTemplate restTemplate() { - return new RestTemplate(); - } - - @Bean - public ObjectMapper objectMapper() { - return new ObjectMapper(); - } -} diff --git a/src/main/java/wanted/ribbon/genrestrt/controller/GenrestrtController.java b/src/main/java/wanted/ribbon/genrestrt/controller/GenrestrtController.java deleted file mode 100644 index 8b37dc5..0000000 --- a/src/main/java/wanted/ribbon/genrestrt/controller/GenrestrtController.java +++ /dev/null @@ -1,23 +0,0 @@ -package wanted.ribbon.genrestrt.controller; - - -import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import wanted.ribbon.genrestrt.service.GenrestrtService; - -@RestController -@RequestMapping("/api/genrestrts") -@RequiredArgsConstructor -public class GenrestrtController { - private final GenrestrtService genrestrtService; - - // 경기도 맛집 데이터 수집 API - @PostMapping("/fetch-data") - public ResponseEntity fetchData(@RequestParam("serviceName") String serviceName) { - // openAPI 호출 - genrestrtService.fetchAndSaveData(serviceName); - return ResponseEntity.ok(serviceName + "가 db에 성공적으로 저장됐습니다."); - } - -} diff --git a/src/main/java/wanted/ribbon/user/config/WebSecurityConfig.java b/src/main/java/wanted/ribbon/user/config/WebSecurityConfig.java index 0b848bb..36dbc39 100644 --- a/src/main/java/wanted/ribbon/user/config/WebSecurityConfig.java +++ b/src/main/java/wanted/ribbon/user/config/WebSecurityConfig.java @@ -26,6 +26,7 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { .csrf(AbstractHttpConfigurer::disable) .authorizeRequests(auth -> auth .requestMatchers("/api/users/login", "/api/users/signup").permitAll() + .requestMatchers("/api/datapipes/**").permitAll() // 데이터파이프라인 모든 권한 허용 .anyRequest().authenticated()) .formLogin(AbstractHttpConfigurer::disable) .addFilterBefore(new TokenAuthenticationFilter(tokenProvider), UsernamePasswordAuthenticationFilter.class);