Skip to content

Commit

Permalink
Merge branch 'release/1.3'
Browse files Browse the repository at this point in the history
  • Loading branch information
@junhaesung
junhaesung committed Mar 9, 2023
2 parents 7c05709 + 11d5658 commit 57bdbe4
Show file tree
Hide file tree
Showing 12 changed files with 347 additions and 126 deletions.
141 changes: 141 additions & 0 deletions module-batch/src/main/java/inspiration/application/member/count/MemberCountJobConfig.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package inspiration.application.member.count;

import inspiration.application.slack.SlackService;
import inspiration.domain.member.Member;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.item.database.builder.JpaPagingItemReaderBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import javax.persistence.EntityManagerFactory;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.YearMonth;
import java.util.Map;
import java.util.stream.Collectors;

@Slf4j
@ConditionalOnProperty(
name = "spring.batch.job.names",
havingValue = MemberCountJobConfig.JOB_NAME
)
@Configuration
@RequiredArgsConstructor
public class MemberCountJobConfig {
static final String JOB_NAME = "member-count-job";
static final String STEP_NAME = "member-count-step";
private static final int CHUNK_SIZE = 1000;
private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final SlackService slackService;

@Value("ygtang.temporary-directory-path")
private String temporaryDirectoryPath;

@Bean
public Job memberDailyCountJob(Step memberDailyCountStep) {
return jobBuilderFactory.get(JOB_NAME)
.incrementer(new RunIdIncrementer())
.start(memberDailyCountStep)
.build();
}

@Bean
@JobScope
public Step memberDailyCountStep(
EntityManagerFactory entityManagerFactory
) {
return stepBuilderFactory.get(STEP_NAME)
.<Member, Member>chunk(CHUNK_SIZE)
.reader(new JpaPagingItemReaderBuilder<Member>()
.name("memberItemReader")
.entityManagerFactory(entityManagerFactory)
.queryString("SELECT m FROM Member m")
.pageSize(CHUNK_SIZE)
.build())
.writer(items -> {
LocalDateTime now = LocalDateTime.now();

// daily
Map<LocalDate, Integer> dailyCountMap = items.stream()
.collect(Collectors.toMap(
it -> it.getCreatedDateTime().toLocalDate(),
it -> 1,
Integer::sum
));
slackService.sendCsv(
toDailyCsvFile(dailyCountMap),
"Member Daily Count at " + now,
"member_daily_count_" + now.toLocalDate() + ".csv"
);

// monthly
Map<YearMonth, Integer> monthlyCountMap = items.stream()
.collect(Collectors.toMap(
it -> YearMonth.from(it.getCreatedDateTime()),
it -> 1,
Integer::sum
));
slackService.sendCsv(
toMonthlyCsvFile(monthlyCountMap),
"Member Monthly Count at " + now,
"member_monthly_count_" + now.toLocalDate() + ".csv"
);
})
.build();
}

private File toDailyCsvFile(Map<LocalDate, Integer> dailyCountMap) throws IOException {
File file = File.createTempFile("memberDailyCount", "csv", new File(temporaryDirectoryPath));
file.deleteOnExit();
FileWriter out = new FileWriter(file);
CSVFormat csvFormat = CSVFormat.Builder.create()
.setHeader("date", "count")
.build();
try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) {
dailyCountMap.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(it -> {
try {
printer.printRecord(it.getKey(), it.getValue());
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
);
}
return file;
}

private File toMonthlyCsvFile(Map<YearMonth, Integer> monthlyCountMap) throws IOException {
File file = File.createTempFile("memberMonthlyCount", "csv", new File(temporaryDirectoryPath));
file.deleteOnExit();
FileWriter out = new FileWriter(file);
CSVFormat csvFormat = CSVFormat.Builder.create()
.setHeader("yearMonth", "count")
.build();
try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) {
monthlyCountMap.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(it -> {
try {
printer.printRecord(it.getKey(), it.getValue());
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
);
}
return file;
}
}
7 changes: 7 additions & 0 deletions module-batch/src/main/java/inspiration/application/slack/SlackService.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package inspiration.application.slack;

import java.io.File;

public interface SlackService {
void sendCsv(File file, String title, String filename);
}
40 changes: 40 additions & 0 deletions module-batch/src/main/java/inspiration/application/slack/SlackServiceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package inspiration.application.slack;

import com.slack.api.Slack;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.request.files.FilesUploadRequest;
import com.slack.api.methods.response.files.FilesUploadResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.File;
import java.util.Collections;

@Slf4j
@Service
public class SlackServiceImpl implements SlackService {
@Value("${ygtang.slack.token.bot}")
private String botToken;
@Value("${ygtang.slack.channel.report}")
private String reportChannel;

@Override
public void sendCsv(File file, String title, String filename) {
try (Slack slack = Slack.getInstance()) {
MethodsClient methods = slack.methods(botToken);
FilesUploadResponse response = methods.filesUpload(
FilesUploadRequest.builder()
.channels(Collections.singletonList(reportChannel))
.title(title)
.filename(filename)
.filetype("text/csv")
.file(file)
.build()
);
log.info("response: {}", response);
} catch (Exception e) {
log.error("Failed to send file to slack", e);
}
}
}
131 changes: 12 additions & 119 deletions module-batch/src/main/java/inspiration/application/tag/TagRankingConfig.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,9 @@
package inspiration.application.tag;

import com.slack.api.Slack;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.request.files.FilesUploadRequest;
import com.slack.api.methods.response.files.FilesUploadResponse;
import inspiration.domain.inspiration_tag.InspirationTag;
import inspiration.application.slack.SlackService;
import inspiration.domain.inspiration_tag.InspirationTagRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
Expand All @@ -18,22 +12,11 @@
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Slf4j
@ConditionalOnProperty(
name = "spring.batch.job.names",
Expand All @@ -46,17 +29,8 @@ public class TagRankingConfig {
static final String JOB_NAME = "tag-ranking-job";
private static final String STEP_NAME = "tag-ranking-step";

private static final DateTimeFormatter FORMATTER_YYYYMMDD = DateTimeFormatter.ofPattern("yyyyMMdd");

private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final InspirationTagRepository inspirationTagRepository;
private final TagGroupService googleSheetTagGroupService;

@Value("${ygtang.slack.token.bot}")
private String botToken;
@Value("${ygtang.slack.channel.report}")
private String reportChannel;

@Bean
public Job tagRankingJob() {
Expand All @@ -70,103 +44,22 @@ public Job tagRankingJob() {
@JobScope
public Step tagRankingStep() {
return stepBuilderFactory.get(STEP_NAME)
.tasklet(tagRankingTasklet())
.tasklet(tagRankingTasklet(null, null, null))
.transactionManager(new ResourcelessTransactionManager())
.build();
}

@Bean
@StepScope
public Tasklet tagRankingTasklet() {
return (contribution, chunkContext) -> {
List<TagGroup> tagGroups = googleSheetTagGroupService.getTagGroups();
List<TagRankingVo> tagRankingVoList = getTagRankingVoList(tagGroups);
File csvFile = toCsvFile(tagRankingVoList);
sendFileToSlack(csvFile);
return RepeatStatus.FINISHED;
};
public Tasklet tagRankingTasklet(
InspirationTagRepository inspirationTagRepository,
TagGroupService googleSheetTagGroupService,
SlackService slackService
) {
return new TagRankingTasklet(
inspirationTagRepository,
googleSheetTagGroupService,
slackService
);
}

private List<TagRankingVo> getTagRankingVoList(List<TagGroup> tagGroups) {
List<InspirationTag> inspirationTags = inspirationTagRepository.findAll();

Map<String, Set<Long>> contentTagIdSetMap =
inspirationTags.stream()
.collect(Collectors.toMap(
it -> resolveTagName(it.getTag().getContent(), tagGroups),
it -> Stream.of(it.getTag().getId()).collect(Collectors.toSet()),
(a, b) -> {
Set<Long> c = new HashSet<>(a);
c.addAll(b);
return c;
}
));
Map<String, Set<Long>> contentInspirationIdSetMap =
inspirationTags.stream()
.collect(Collectors.toMap(
it -> resolveTagName(it.getTag().getContent(), tagGroups),
it -> Stream.of(it.getInspiration().getId()).collect(Collectors.toSet()),
(a, b) -> {
Set<Long> c = new HashSet<>(a);
c.addAll(b);
return c;
}
));
return contentInspirationIdSetMap.keySet()
.stream()
.map(content -> new TagRankingVo(
content,
contentTagIdSetMap.get(content).size(),
contentInspirationIdSetMap.get(content).size()
))
.sorted(Comparator.comparing(TagRankingVo::getInspirationCount)
.thenComparing(TagRankingVo::getTagCount)
.reversed())
.filter(it -> it.getTagCount() >= 2 || it.getInspirationCount() >= 2)
.collect(Collectors.toList());
}

private String resolveTagName(String tagName, List<TagGroup> tagGroups) {
for (TagGroup tagGroup : tagGroups) {
if (tagGroup.contains(tagName)) {
return tagGroup.getName();
}
}
return tagName;
}

private File toCsvFile(List<TagRankingVo> tagRankingVoList) throws IOException {
File file = File.createTempFile("tagRanking", "csv");
file.deleteOnExit();
FileWriter out = new FileWriter(file);
CSVFormat csvFormat = CSVFormat.Builder.create()
.setHeader(TagRankingVo.getCsvHeaders())
.build();
try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) {
for (TagRankingVo tagRankingVo : tagRankingVoList) {
printer.printRecord(tagRankingVo.getCsvValues());
}
}
return file;
}

private void sendFileToSlack(File file) {
LocalDate today = LocalDate.now();
try (Slack slack = Slack.getInstance()) {
MethodsClient methods = slack.methods(botToken);
FilesUploadResponse response = methods.filesUpload(
FilesUploadRequest.builder()
.channels(Collections.singletonList(reportChannel))
.title("Tag ranking at " + today)
.filename("tagRanking_" + today.format(FORMATTER_YYYYMMDD) + ".csv")
.filetype("text/csv")
.file(file)
.build()
);
log.info("response: {}", response);
} catch (Exception e) {
log.error("Failed to send file to slack", e);
}
}

}
Loading

0 comments on commit 57bdbe4

Please sign in to comment.