Merge pull request #29 from 2024-pre-onboarding-backend-F/feat/create…

…_datapipe

[feat] 데이터파이프라인 공공데이터 수집 기능 고도화

build.gradle

@@ -44,6 +44,10 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-batch'
     // spring-batch-core 최신버전 추가
     implementation 'org.springframework.batch:spring-batch-core:5.1.0'
+    // Webflux
+    implementation 'org.springframework.boot:spring-boot-starter-webflux'
+    // mac netty 설정
+    implementation 'io.netty:netty-resolver-dns-native-macos:4.1.68.Final:osx-aarch_64'
     // QueryDSL - 4개
     implementation 'com.querydsl:querydsl-jpa:5.0.0:jakarta'
     annotationProcessor "com.querydsl:querydsl-apt:${dependencyManagement.importedProperties['querydsl.version']}:jakarta"
@@ -55,6 +59,7 @@ dependencies {
     annotationProcessor 'org.projectlombok:lombok'
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
     testImplementation 'org.springframework.security:spring-security-test'
+    testImplementation 'org.springframework:spring-webflux'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
 }
 

src/main/java/wanted/ribbon/genrestrt/component/DataFetchTasklet.java → src/main/java/wanted/ribbon/datapipe/component/DataFetchTasklet.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.component;
+package wanted.ribbon.datapipe.component;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -11,10 +11,10 @@
 import org.springframework.stereotype.Component;
 import wanted.ribbon.exception.BaseException;
 import wanted.ribbon.exception.ErrorCode;
-import wanted.ribbon.genrestrt.dto.RawData;
-import wanted.ribbon.genrestrt.mapper.RawDataRowMapper;
-import wanted.ribbon.genrestrt.mapper.StoreDataRowMapper;
-import wanted.ribbon.genrestrt.service.DataProcessor;
+import wanted.ribbon.datapipe.dto.RawData;
+import wanted.ribbon.datapipe.mapper.RawDataRowMapper;
+import wanted.ribbon.datapipe.mapper.StoreDataRowMapper;
+import wanted.ribbon.datapipe.service.DataProcessor;
 import wanted.ribbon.store.domain.Store;
 
 import java.util.ArrayList;

src/main/java/wanted/ribbon/genrestrt/component/DataPipeJobScheduler.java → src/main/java/wanted/ribbon/datapipe/component/DataPipeJobScheduler.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.component;
+package wanted.ribbon.datapipe.component;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.batch.core.JobParametersBuilder;
@@ -9,7 +9,7 @@
 import org.springframework.batch.core.repository.JobRestartException;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
-import wanted.ribbon.genrestrt.config.DataPipeJobConfig;
+import wanted.ribbon.datapipe.config.DataPipeJobConfig;
 
 @Component
 @RequiredArgsConstructor

src/main/java/wanted/ribbon/genrestrt/component/DataPipeTasklet.java → src/main/java/wanted/ribbon/datapipe/component/DataPipeTasklet.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.component;
+package wanted.ribbon.datapipe.component;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -11,9 +11,9 @@
 import org.springframework.stereotype.Component;
 import wanted.ribbon.exception.BaseException;
 import wanted.ribbon.exception.ErrorCode;
-import wanted.ribbon.genrestrt.dto.RawData;
-import wanted.ribbon.genrestrt.mapper.RawDataRowMapper;
-import wanted.ribbon.genrestrt.service.DataProcessor;
+import wanted.ribbon.datapipe.dto.RawData;
+import wanted.ribbon.datapipe.mapper.RawDataRowMapper;
+import wanted.ribbon.datapipe.service.DataProcessor;
 import wanted.ribbon.store.domain.Store;
 
 import java.util.List;

src/main/java/wanted/ribbon/datapipe/config/AppConfig.java

@@ -0,0 +1,44 @@
+package wanted.ribbon.datapipe.config;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.util.DefaultUriBuilderFactory;
+
+@Configuration
+public class AppConfig {
+    @Value("${spring.public-api.base-url}")
+    private String baseUrl;
+
+    @Bean
+    public RestTemplate restTemplate() {
+        return new RestTemplate();
+    }
+
+    @Bean
+    public ObjectMapper objectMapper() {
+        return new ObjectMapper();
+    }
+
+    /*
+     * WebClient의 UriComponentsBuilder.encode() 방식의 인코딩 방지
+     * key 값이 달라지는 것을 방지하기 위해 DefaultUriBuilderFactory 생성
+     * encoding 모드 지정
+     * */
+    @Bean
+    public DefaultUriBuilderFactory builderFactory() {
+        DefaultUriBuilderFactory builderFactory = new DefaultUriBuilderFactory();
+        builderFactory.setEncodingMode(DefaultUriBuilderFactory.EncodingMode.VALUES_ONLY);
+        return builderFactory;
+    }
+
+    @Bean
+    public WebClient webClient() {
+        return WebClient.builder()
+                .uriBuilderFactory(builderFactory())
+                .build();
+    }
+}

src/main/java/wanted/ribbon/genrestrt/config/DataPipeJobConfig.java → src/main/java/wanted/ribbon/datapipe/config/DataPipeJobConfig.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.config;
+package wanted.ribbon.datapipe.config;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.batch.core.Job;
@@ -12,8 +12,8 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.transaction.PlatformTransactionManager;
-import wanted.ribbon.genrestrt.component.DataFetchTasklet;
-import wanted.ribbon.genrestrt.component.DataPipeTasklet;
+import wanted.ribbon.datapipe.component.DataFetchTasklet;
+import wanted.ribbon.datapipe.component.DataPipeTasklet;
 
 @RequiredArgsConstructor
 @Configuration

src/main/java/wanted/ribbon/datapipe/controller/OpenApiController.java

@@ -0,0 +1,38 @@
+package wanted.ribbon.datapipe.controller;
+
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import reactor.core.publisher.Mono;
+import wanted.ribbon.datapipe.dto.GyeongGiList;
+import wanted.ribbon.datapipe.service.GenrestrtService;
+import wanted.ribbon.datapipe.service.RawDataService;
+
+@RestController
+@RequestMapping("/api/datapipes")
+@RequiredArgsConstructor
+public class OpenApiController {
+    private final GenrestrtService genrestrtService;
+    private final RawDataService rawDataService;
+
+    // 경기도 맛집 데이터 수집 API (RestTemplate), responsebody 사용으로 PostMapping으로 진행
+    @PostMapping("/fetch-data")
+    public ResponseEntity<String> fetchData(@RequestParam("serviceName") String serviceName) {
+        // openAPI 호출
+        genrestrtService.fetchAndSaveData(serviceName);
+        return ResponseEntity.ok(serviceName + "가 db에 성공적으로 저장됐습니다.");
+    }
+
+    // 경기도 맛집 데이터 수집 API (WebClient, 모든 경기도 맛집 api 가능)
+    /**
+     * 데이터 조회로 GetMapping 사용
+     * 비동기 처리를 위한 Mono 사용
+     */
+    @GetMapping("/fetch-and-save")
+    public Mono<ResponseEntity<GyeongGiList>> fetchAndSaveData(@RequestParam("serviceName") String serviceName) {
+        return rawDataService.getAndSaveByServiceName(serviceName)
+                .map(ResponseEntity::ok)
+                .defaultIfEmpty(ResponseEntity.notFound().build());
+    }
+}

src/main/java/wanted/ribbon/genrestrt/domain/Genrestrt.java → src/main/java/wanted/ribbon/datapipe/domain/Genrestrt.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.domain;
+package wanted.ribbon.datapipe.domain;
 
 import jakarta.persistence.*;
 import lombok.*;

src/main/java/wanted/ribbon/genrestrt/dto/GenrestrtApiResponse.java → src/main/java/wanted/ribbon/datapipe/dto/GenrestrtApiResponse.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.dto;
+package wanted.ribbon.datapipe.dto;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;

src/main/java/wanted/ribbon/datapipe/dto/GyeongGiList.java

@@ -0,0 +1,32 @@
+package wanted.ribbon.datapipe.dto;
+
+import java.util.Date;
+import java.util.List;
+
+public record GyeongGiList(String message,
+                           Long total,
+                           List<GyeongGiApiResponse> gyeongGiApiResponses) {
+    public record GyeongGiApiResponse(String sigunNm,
+                                      String sigunCd,
+                                      String bizplcNm,
+                                      Date licensgDe,
+                                      String bsnStateNm,
+                                      Date clsbizDe,
+                                      Double locplcAr,
+                                      String gradFacltDivNm,
+                                      Long maleEnflpsnCnt,
+                                      Integer yy,
+                                      String multiUseBizestblYn,
+                                      String gradDivNm,
+                                      Double totFacltScale,
+                                      Long femaleEnflpsnCnt,
+                                      String bsnsiteCircumfrDivNm,
+                                      String sanittnIndutypeNm,
+                                      String sanittnBizcondNm,
+                                      Long totEmplyCnt,
+                                      String refineRoadnmAddr,
+                                      String refineLotnoAddr,
+                                      String refineZipCd,
+                                      Double refineWgs84Lat,
+                                      Double refineWgs84Logt){}
+}

src/main/java/wanted/ribbon/genrestrt/dto/RawData.java → src/main/java/wanted/ribbon/datapipe/dto/RawData.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.dto;
+package wanted.ribbon.datapipe.dto;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

src/main/java/wanted/ribbon/genrestrt/mapper/RawDataRowMapper.java → src/main/java/wanted/ribbon/datapipe/mapper/RawDataRowMapper.java

@@ -1,7 +1,7 @@
-package wanted.ribbon.genrestrt.mapper;
+package wanted.ribbon.datapipe.mapper;
 
 import org.springframework.jdbc.core.RowMapper;
-import wanted.ribbon.genrestrt.dto.RawData;
+import wanted.ribbon.datapipe.dto.RawData;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;

src/main/java/wanted/ribbon/genrestrt/mapper/StoreDataRowMapper.java → src/main/java/wanted/ribbon/datapipe/mapper/StoreDataRowMapper.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.mapper;
+package wanted.ribbon.datapipe.mapper;
 
 import org.springframework.jdbc.core.RowMapper;
 import wanted.ribbon.store.domain.Category;

src/main/java/wanted/ribbon/genrestrt/repository/GenrestrtRepository.java → src/main/java/wanted/ribbon/datapipe/repository/GenrestrtRepository.java

@@ -1,7 +1,7 @@
-package wanted.ribbon.genrestrt.repository;
+package wanted.ribbon.datapipe.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
-import wanted.ribbon.genrestrt.domain.Genrestrt;
+import wanted.ribbon.datapipe.domain.Genrestrt;
 
 public interface GenrestrtRepository extends JpaRepository<Genrestrt, Long> {
 }

src/main/java/wanted/ribbon/genrestrt/service/DataProcessor.java → src/main/java/wanted/ribbon/datapipe/service/DataProcessor.java

@@ -1,8 +1,8 @@
-package wanted.ribbon.genrestrt.service;
+package wanted.ribbon.datapipe.service;
 
 import org.springframework.batch.item.ItemProcessor;
 import org.springframework.stereotype.Component;
-import wanted.ribbon.genrestrt.dto.RawData;
+import wanted.ribbon.datapipe.dto.RawData;
 import wanted.ribbon.store.domain.Category;
 import wanted.ribbon.store.domain.Store;
 

src/main/java/wanted/ribbon/genrestrt/service/GenrestrtService.java → src/main/java/wanted/ribbon/datapipe/service/GenrestrtService.java

@@ -1,4 +1,4 @@
-package wanted.ribbon.genrestrt.service;
+package wanted.ribbon.datapipe.service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -10,9 +10,9 @@
 import org.springframework.web.client.RestTemplate;
 import wanted.ribbon.exception.ErrorCode;
 import wanted.ribbon.exception.GenrestrtException;
-import wanted.ribbon.genrestrt.domain.Genrestrt;
-import wanted.ribbon.genrestrt.dto.GenrestrtApiResponse;
-import wanted.ribbon.genrestrt.repository.GenrestrtRepository;
+import wanted.ribbon.datapipe.domain.Genrestrt;
+import wanted.ribbon.datapipe.dto.GenrestrtApiResponse;
+import wanted.ribbon.datapipe.repository.GenrestrtRepository;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;