feat : 점심 수동 크롤링 기능 추가

* feat : 수동 스크래핑 서비스 로직 추가 * refactor : 점심 크롤링 리팩토링 - 스케줄러의 비즈니스 로직을 서비스단으로 분리 * feat : 수동 스크래핑 컨트롤러 추가 * refactor : 크롤링 관련 패키지 구조 변경 * refactor : 추상화 리팩토링
SSAF-SOUND · Nov 15, 2023 · 3ce38a9 · 3ce38a9
1 parent 1e4a553
commit 3ce38a9
Show file tree

Hide file tree

Showing 18 changed files with 320 additions and 221 deletions.
diff --git a/src/main/java/com/ssafy/ssafsound/domain/auth/exception/AuthErrorInfo.java b/src/main/java/com/ssafy/ssafsound/domain/auth/exception/AuthErrorInfo.java
@@ -9,7 +9,8 @@ public enum AuthErrorInfo {
     AUTH_SERVER_PARSING_ERROR("702", "서버에서 파싱하는데 문제가 발생했습니다."),
     AUTH_TOKEN_INVALID("705", "토큰이 유효하지 않습니다."),
     AUTH_TOKEN_EXPIRED("706", "토큰이 만료됐습니다."),
-    AUTH_TOKEN_SERVICE_ERROR("707", "서버에서 토큰을 처리하는 과정에서 문제가 발생했습니다.");
+    AUTH_TOKEN_SERVICE_ERROR("707", "서버에서 토큰을 처리하는 과정에서 문제가 발생했습니다."),
+    UNAUTHORIZED_ERROR("708", "권한이 없습니다.");
 
     private final String code;
     private final String message;

diff --git a/src/main/java/com/ssafy/ssafsound/domain/lunch/controller/LunchController.java b/src/main/java/com/ssafy/ssafsound/domain/lunch/controller/LunchController.java
@@ -6,6 +6,7 @@
 import com.ssafy.ssafsound.domain.lunch.dto.GetLunchListResDto;
 import com.ssafy.ssafsound.domain.lunch.dto.PostLunchPollResDto;
 import com.ssafy.ssafsound.domain.lunch.service.LunchPollService;
+import com.ssafy.ssafsound.domain.lunch.service.LunchScrapService;
 import com.ssafy.ssafsound.domain.lunch.service.LunchService;
 import com.ssafy.ssafsound.global.common.response.EnvelopeResponse;
 import lombok.RequiredArgsConstructor;
@@ -25,6 +26,7 @@ public class LunchController {
 
     private final LunchService lunchService;
     private final LunchPollService lunchPollService;
+    private final LunchScrapService lunchScrapService;
 
     @GetMapping
     public EnvelopeResponse<GetLunchListResDto> getLunchesByCampusAndDate(@Authentication AuthenticatedMember member, @Valid GetLunchListReqDto getLunchListReqDto) {
@@ -50,4 +52,11 @@ public EnvelopeResponse<PostLunchPollResDto> revertLunchPoll(@Authentication Aut
                 .build();
     }
 
+    @PostMapping("/manual-scrap")
+    public EnvelopeResponse<Void> scrapWelstoryLunchManually(@Authentication AuthenticatedMember member, @Valid GetLunchListReqDto getLunchListReqDto) {
+
+        lunchScrapService.scrapLunchManually(member, getLunchListReqDto);
+        return EnvelopeResponse.<Void>builder().build();
+    }
+
 }
diff --git a/...unch/task/domain/FreshmealProperties.java → ...ain/lunch/domain/FreshmealProperties.java b/...unch/task/domain/FreshmealProperties.java → ...ain/lunch/domain/FreshmealProperties.java
@@ -1,4 +1,4 @@
-package com.ssafy.ssafsound.domain.lunch.task.domain;
+package com.ssafy.ssafsound.domain.lunch.domain;
 
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;

diff --git a/...lunch/task/domain/WelstoryProperties.java → ...main/lunch/domain/WelstoryProperties.java b/...lunch/task/domain/WelstoryProperties.java → ...main/lunch/domain/WelstoryProperties.java
@@ -1,4 +1,4 @@
-package com.ssafy.ssafsound.domain.lunch.task.domain;
+package com.ssafy.ssafsound.domain.lunch.domain;
 
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;

diff --git a/...in/lunch/task/dto/GetFreshmealResDto.java → .../domain/lunch/dto/GetFreshmealResDto.java b/...in/lunch/task/dto/GetFreshmealResDto.java → .../domain/lunch/dto/GetFreshmealResDto.java
@@ -1,15 +1,18 @@
-package com.ssafy.ssafsound.domain.lunch.task.dto;
+package com.ssafy.ssafsound.domain.lunch.dto;
 
 import com.fasterxml.jackson.annotation.JsonAnySetter;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.ssafy.ssafsound.domain.lunch.domain.Lunch;
+import com.ssafy.ssafsound.domain.lunch.exception.LunchErrorInfo;
+import com.ssafy.ssafsound.domain.lunch.exception.LunchException;
 import com.ssafy.ssafsound.domain.meta.domain.MetaData;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -21,6 +24,29 @@ public class GetFreshmealResDto implements GetScrapResDto{
     @JsonProperty("data")
     private FreshmealBodyData data;
 
+    @Override
+    public List<Lunch> getLunches(MetaData campus) {
+        List<String> daysOfWeek = Arrays.asList("mo","tu","we","th","fr");
+        List<Lunch> lunches = new ArrayList<>();
+
+        for (String day : daysOfWeek) {
+            this.data.getDailyMeal().get(day).getMeals().forEach(
+                    meal -> {
+                        try {
+                            Lunch lunch = meal.toEntity(campus);
+                            if (lunch != null) {
+                                lunches.add(lunch);
+                            }
+                        } catch (Exception e) {
+                            throw new LunchException(LunchErrorInfo.SCRAPING_ERROR);
+                        }
+                    }
+            );
+        }
+
+        return lunches;
+    }
+
     @Data
     @NoArgsConstructor
     public static class FreshmealBodyData {

diff --git a/...domain/lunch/task/dto/GetScrapReqDto.java → ...ound/domain/lunch/dto/GetScrapReqDto.java b/...domain/lunch/task/dto/GetScrapReqDto.java → ...ound/domain/lunch/dto/GetScrapReqDto.java
@@ -1,4 +1,4 @@
-package com.ssafy.ssafsound.domain.lunch.task.dto;
+package com.ssafy.ssafsound.domain.lunch.dto;
 
 import com.ssafy.ssafsound.domain.meta.domain.MetaData;
 import lombok.Builder;

diff --git a/src/main/java/com/ssafy/ssafsound/domain/lunch/dto/GetScrapResDto.java b/src/main/java/com/ssafy/ssafsound/domain/lunch/dto/GetScrapResDto.java
@@ -0,0 +1,10 @@
+package com.ssafy.ssafsound.domain.lunch.dto;
+
+import com.ssafy.ssafsound.domain.lunch.domain.Lunch;
+import com.ssafy.ssafsound.domain.meta.domain.MetaData;
+
+import java.util.List;
+
+public interface GetScrapResDto {
+    List<Lunch> getLunches(MetaData campus);
+}
diff --git a/...ain/lunch/task/dto/GetWelstoryResDto.java → ...d/domain/lunch/dto/GetWelstoryResDto.java b/...ain/lunch/task/dto/GetWelstoryResDto.java → ...d/domain/lunch/dto/GetWelstoryResDto.java
@@ -1,20 +1,43 @@
-package com.ssafy.ssafsound.domain.lunch.task.dto;
+package com.ssafy.ssafsound.domain.lunch.dto;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.ssafy.ssafsound.domain.lunch.domain.Lunch;
+import com.ssafy.ssafsound.domain.lunch.exception.LunchErrorInfo;
+import com.ssafy.ssafsound.domain.lunch.exception.LunchException;
 import com.ssafy.ssafsound.domain.meta.domain.MetaData;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
 import java.util.List;
 
 @Data
 @NoArgsConstructor
 public class GetWelstoryResDto implements GetScrapResDto{
     @JsonProperty("data")
-    private WelstoryBodyData welstoryBodyData;
+    private WelstoryBodyData data;
+
+    public List<Lunch> getLunches(MetaData campus) {
+        List<Lunch> lunches = new ArrayList<>();
+
+        this.data.getMealList().forEach(
+                meal -> {
+                    try {
+                        Lunch lunch = meal.toEntity(campus);
+
+                        if (lunch != null) {
+                            lunches.add(lunch);
+                        }
+                    } catch (Exception e) {
+                        throw new LunchException(LunchErrorInfo.SCRAPING_ERROR);
+                    }
+                }
+        );
+
+        return lunches;
+    }
 
     @Data
     @NoArgsConstructor

diff --git a/src/main/java/com/ssafy/ssafsound/domain/lunch/service/FreshmealInfoProvider.java b/src/main/java/com/ssafy/ssafsound/domain/lunch/service/FreshmealInfoProvider.java
@@ -0,0 +1,91 @@
+package com.ssafy.ssafsound.domain.lunch.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.ssafy.ssafsound.domain.lunch.domain.FreshmealProperties;
+import com.ssafy.ssafsound.domain.lunch.domain.Lunch;
+import com.ssafy.ssafsound.domain.lunch.dto.GetFreshmealResDto;
+import com.ssafy.ssafsound.domain.lunch.dto.GetScrapReqDto;
+import com.ssafy.ssafsound.global.common.json.JsonParser;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class FreshmealInfoProvider implements ScrapInfoProvider {
+
+    private final RestTemplate restTemplate;
+
+    private final FreshmealProperties freshmealProperties;
+
+    @Override
+    public List<Lunch> scrapLunchInfo(List<GetScrapReqDto> getScrapReqDtos) {
+
+        List<Lunch> lunches = new ArrayList<>();
+
+        for (GetScrapReqDto getScrapReqDto : getScrapReqDtos) {
+            Map<String, String> parameters = makeScrapParameters();
+
+            String url = makeScrapUri(this.freshmealProperties.getUrl(), parameters);
+
+            ResponseEntity<String> response = restTemplate.exchange(
+                    url,
+                    HttpMethod.GET,
+                    makeHeader(),
+                    String.class
+            );
+
+            try {
+                lunches.addAll(JsonParser.getMapper()
+                        .readValue(response.getBody(), GetFreshmealResDto.class)
+                        .getLunches(getScrapReqDto.getCampus()));
+            } catch (JsonProcessingException e) {
+                log.error(e.getMessage());
+                e.printStackTrace();
+
+                throw new RuntimeException();
+            }
+        }
+
+        return lunches;
+    }
+
+    private HttpEntity makeHeader() {
+        HttpHeaders header = new HttpHeaders();
+        header.add("Content-Type", "application/json");
+        header.add("Accept", MediaType.APPLICATION_JSON_VALUE);
+        return new HttpEntity(header);
+    }
+
+    private Map<String, String> makeScrapParameters() {
+        Map<String, String> parameters = new HashMap<>();
+
+        parameters.put("storeIdx", this.freshmealProperties.getStoreIdx());
+        parameters.put("weekType", this.freshmealProperties.getWeekType());
+
+        return parameters;
+    }
+
+    private String makeScrapUri(String baseUri, Map<String, String> parameters) {
+
+        UriComponentsBuilder uri = UriComponentsBuilder.fromUriString(baseUri);
+        for (Map.Entry<String, String> entry : parameters.entrySet()) {
+            uri.queryParam(entry.getKey(), entry.getValue());
+        }
+
+        return uri.toUriString();
+    }
+}
diff --git a/src/main/java/com/ssafy/ssafsound/domain/lunch/service/LunchScrapService.java b/src/main/java/com/ssafy/ssafsound/domain/lunch/service/LunchScrapService.java
@@ -0,0 +1,83 @@
+package com.ssafy.ssafsound.domain.lunch.service;
+
+import com.ssafy.ssafsound.domain.auth.dto.AuthenticatedMember;
+import com.ssafy.ssafsound.domain.auth.exception.AuthErrorInfo;
+import com.ssafy.ssafsound.domain.auth.exception.AuthException;
+import com.ssafy.ssafsound.domain.lunch.dto.GetLunchListReqDto;
+import com.ssafy.ssafsound.domain.lunch.dto.GetScrapReqDto;
+import com.ssafy.ssafsound.domain.lunch.exception.LunchErrorInfo;
+import com.ssafy.ssafsound.domain.lunch.exception.LunchException;
+import com.ssafy.ssafsound.domain.lunch.repository.LunchRepository;
+import com.ssafy.ssafsound.domain.member.exception.MemberErrorInfo;
+import com.ssafy.ssafsound.domain.meta.domain.Campus;
+import com.ssafy.ssafsound.domain.meta.domain.MetaData;
+import com.ssafy.ssafsound.domain.meta.service.MetaDataConsumer;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@RequiredArgsConstructor
+@Service
+public class LunchScrapService {
+
+    private final LunchRepository lunchRepository;
+    private final MetaDataConsumer metaDataConsumer;
+    private final ScrapInfoProviderFactory scrapInfoProviderFactory;
+
+    @Transactional(readOnly = false)
+    public void scrapLunchManually(AuthenticatedMember authenticatedMember, GetLunchListReqDto getLunchListReqDto) {
+
+        String memberRole = Objects.requireNonNull(authenticatedMember.getMemberRole(), MemberErrorInfo.MEMBER_ROLE_TYPE_NOT_FOUND.getMessage());
+
+        if (!memberRole.equals("admin")) {
+            throw new AuthException(AuthErrorInfo.UNAUTHORIZED_ERROR);
+        }
+
+        MetaData campus = metaDataConsumer.getMetaData("CAMPUS", getLunchListReqDto.getCampus());
+
+        if (getLunchListReqDto.getCampus().equals(Campus.DAEJEON.getName())) {
+            throw new LunchException(LunchErrorInfo.SCRAPING_ERROR);
+        }
+
+        if (getLunchListReqDto.getCampus().equals(Campus.GWANGJU.getName())) {
+            scrapFreshmealLunch(campus);
+        } else {
+            scrapWelstoryLunch(getLunchListReqDto.getDate(), campus);
+        }
+    }
+
+    @Transactional(readOnly = false)
+    public void scrapWelstoryLunch(LocalDate date, MetaData... campuses) {
+
+        ScrapInfoProvider scraper = scrapInfoProviderFactory.getProviderFrom("welstory");
+
+        List<GetScrapReqDto> requests = Arrays.stream(campuses)
+                .map(campus -> GetScrapReqDto.builder()
+                        .campus(campus)
+                        .menuDt(date)
+                        .build())
+                .collect(Collectors.toList());
+
+        lunchRepository.saveAll(scraper.scrapLunchInfo(requests));
+    }
+
+    @Transactional(readOnly = false)
+    public void scrapFreshmealLunch(MetaData... campuses) {
+
+        ScrapInfoProvider scraper = scrapInfoProviderFactory.getProviderFrom("freshmeal");
+
+        List<GetScrapReqDto> requests = Arrays.stream(campuses)
+                .map(campus -> GetScrapReqDto.builder()
+                        .campus(campus)
+                        .build())
+                .collect(Collectors.toList());
+
+        lunchRepository.saveAll(scraper.scrapLunchInfo(requests));
+    }
+}
diff --git a/src/main/java/com/ssafy/ssafsound/domain/lunch/service/ScrapInfoProvider.java b/src/main/java/com/ssafy/ssafsound/domain/lunch/service/ScrapInfoProvider.java
@@ -0,0 +1,10 @@
+package com.ssafy.ssafsound.domain.lunch.service;
+
+import com.ssafy.ssafsound.domain.lunch.domain.Lunch;
+import com.ssafy.ssafsound.domain.lunch.dto.GetScrapReqDto;
+
+import java.util.List;
+
+public interface ScrapInfoProvider {
+    List<Lunch> scrapLunchInfo(List<GetScrapReqDto> getScrapReqDtos);
+}
diff --git a/...task/domain/ScrapInfoProviderFactory.java → ...nch/service/ScrapInfoProviderFactory.java b/...task/domain/ScrapInfoProviderFactory.java → ...nch/service/ScrapInfoProviderFactory.java
@@ -1,5 +1,8 @@
-package com.ssafy.ssafsound.domain.lunch.task.domain;
+package com.ssafy.ssafsound.domain.lunch.service;
 
+import com.ssafy.ssafsound.domain.lunch.service.FreshmealInfoProvider;
+import com.ssafy.ssafsound.domain.lunch.service.ScrapInfoProvider;
+import com.ssafy.ssafsound.domain.lunch.service.WelstoryInfoProvider;
 import org.springframework.stereotype.Component;
 
 import java.util.HashMap;