diff --git a/pom.xml b/pom.xml index 1f26e57..d89c0e1 100644 --- a/pom.xml +++ b/pom.xml @@ -127,7 +127,16 @@ 4.0.0 - + + + de.dnb + oaiharvester + 3.0.0 + + + + + diff --git a/src/main/java/io/academic/controller/SearchController.java b/src/main/java/io/academic/controller/SearchController.java index ffdfbf7..74ed419 100644 --- a/src/main/java/io/academic/controller/SearchController.java +++ b/src/main/java/io/academic/controller/SearchController.java @@ -1,12 +1,9 @@ package io.academic.controller; +import io.academic.service.AcademicSearchService; import io.academic.service.OaiService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.ui.Model; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.io.IOException; @@ -16,21 +13,27 @@ @RestController @RequestMapping("/api/elastic") public class SearchController { - private final OaiService service; + private final AcademicSearchService service; @Autowired - public SearchController(OaiService service) { + public SearchController(AcademicSearchService service) { this.service = service; } @RequestMapping(method = GET, value = "/_search") public String search(@RequestParam(value= "q") String term) throws IOException { - return service.search(term); + return service.searchPretty(term); } @RequestMapping(method = GET) public String allData(@RequestParam(defaultValue = "") String query) throws IOException { - return service.getAll(); + return service.getAllPretty(); + } + + //search doing by article criterias such as authors,body,date, keywords, publisher, title + @RequestMapping(method = GET, value = "/_search/{criteria}") + public @ResponseBody String searchByCriteria(@RequestParam(value= "q") String term, @PathVariable(value = "criteria") String criteria) throws IOException { + return service.searchPrettyByCriteria(term,criteria); } diff --git a/src/main/java/io/academic/controller/SearchformController.java b/src/main/java/io/academic/controller/SearchformController.java index 815a423..865c7c5 100644 --- a/src/main/java/io/academic/controller/SearchformController.java +++ b/src/main/java/io/academic/controller/SearchformController.java @@ -2,6 +2,7 @@ import io.academic.dao.SearchDao; +import io.academic.service.AcademicSearchService; import io.academic.service.OaiService; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; @@ -15,9 +16,9 @@ @Controller public class SearchformController { - private final OaiService service; + private final AcademicSearchService service; - public SearchformController(OaiService service) { + public SearchformController(AcademicSearchService service) { this.service = service; } @@ -31,9 +32,20 @@ public String searchForm(Model model) { @PostMapping("/searchform") public String greetingSubmit(@ModelAttribute SearchDao searchDao) throws IOException { System.out.println("inside post"); - System.out.println(searchDao.getValue()); - System.out.println(service.searchForm(searchDao.getValue())); - searchDao.setResult(service.searchForm(searchDao.getValue())); + String value = searchDao.getValue(); + String result = ""; + String criteria = searchDao.getCriteria(); + if (criteria.equals("all")) + { + result = service.searchForm(searchDao.getValue()); + } + else + { + result = service.searchFormByCriteria(searchDao.getValue(),criteria); + } + System.out.println(value); + System.out.println(result); + searchDao.setResult(result); return "searchresult"; } diff --git a/src/main/java/io/academic/dao/SearchDao.java b/src/main/java/io/academic/dao/SearchDao.java index c102dba..482e560 100644 --- a/src/main/java/io/academic/dao/SearchDao.java +++ b/src/main/java/io/academic/dao/SearchDao.java @@ -4,6 +4,7 @@ public class SearchDao { private String value; + private String criteria; private String result; public String getValue() { @@ -14,6 +15,13 @@ public void setValue(String value) { this.value = value; } + public String getCriteria() { + return criteria; + } + + public void setCriteria(String criteria) { + this.criteria = criteria; + } public String getResult() { return result; diff --git a/src/main/java/io/academic/entity/Article.java b/src/main/java/io/academic/entity/Article.java index e5ab3ad..cc80b8e 100644 --- a/src/main/java/io/academic/entity/Article.java +++ b/src/main/java/io/academic/entity/Article.java @@ -6,6 +6,7 @@ import javax.persistence.Entity; import javax.persistence.Table; + @Table(name = "articles") @Entity public class Article extends AbstractAuditingEntity { @@ -30,48 +31,79 @@ public class Article extends AbstractAuditingEntity { @Type(type = "text") private String dc; + @Column + @Type(type = "text") + private String publisher; + + @Column + @Type(type = "text") + private String date; + + @Column + @Type(type = "text") + private String type; + public String getTitle() { return title; } - public Article setTitle(String title) { + public void setTitle(String title) { this.title = title; - return this; } public String getBody() { return body; } - public Article setBody(String body) { + public void setBody(String body) { this.body = body; - return this; } public String getKeywords() { return keywords; } - public Article setKeywords(String keywords) { + public void setKeywords(String keywords) { this.keywords = keywords; - return this; } public String getAuthors() { return authors; } - public Article setAuthors(String authors) { + public void setAuthors(String authors) { this.authors = authors; - return this; } public String getDc() { return dc; } - public Article setDc(String dc) { + public void setDc(String dc) { this.dc = dc; - return this; + } + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public String getDate() { + return date; + } + + public void setDate(String date) { + this.date = date; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; } } diff --git a/src/main/java/io/academic/entity/ArticleRepository.java b/src/main/java/io/academic/entity/ArticleRepository.java index 5e4cac4..96a2412 100644 --- a/src/main/java/io/academic/entity/ArticleRepository.java +++ b/src/main/java/io/academic/entity/ArticleRepository.java @@ -1,5 +1,6 @@ package io.academic.entity; +import org.elasticsearch.action.search.*; import org.springframework.data.repository.PagingAndSortingRepository; import java.util.List; @@ -9,4 +10,5 @@ public interface ArticleRepository extends PagingAndSortingRepository findByTitle(String title); + } diff --git a/src/main/java/io/academic/service/AcademicSearchService.java b/src/main/java/io/academic/service/AcademicSearchService.java new file mode 100644 index 0000000..357de46 --- /dev/null +++ b/src/main/java/io/academic/service/AcademicSearchService.java @@ -0,0 +1,184 @@ +package io.academic.service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import org.elasticsearch.action.search.*; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.IOException; + +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; + +@Service +public class AcademicSearchService { + + @Autowired + private OaiService oaiService; + + public AcademicSearchService() { + } + + public String search(String q) throws IOException { + + SearchRequest searchRequest = new SearchRequest("harvester"); + searchRequest.source(buildSource("term","dc",q,false)); + + //this values are necessary if we need scrollable results (in other words if our result have more than 10 hits) + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1)); + searchRequest.scroll(scroll); + + SearchResponse searchResponse = oaiService.getRestClient().search(searchRequest); + String result = scrollableSearch(searchResponse,scroll); + + return result; + + } + + + public String searchBy(String q, String criteria) throws IOException { + + SearchRequest searchRequest = new SearchRequest("harvester"); + searchRequest.source(buildSource("match",criteria,q,false)); + + //this values are necessary if we need scrollable results (in other words if our result have more than 10 hits) + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1)); + searchRequest.scroll(scroll); + + SearchResponse searchResponse = oaiService.getRestClient().search(searchRequest); + String result = scrollableSearch(searchResponse,scroll); + +// System.out.println(result); + + return result; + + } + + + public String getAll() throws IOException { + + SearchRequest searchRequest = new SearchRequest("harvester"); + searchRequest.source(buildSource("matchAll","","",true)); + + //this values are necessary if we need scrollable results (in other words if our result have more than 10 hits) + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1)); + searchRequest.scroll(scroll); + + SearchResponse searchResponse = oaiService.getRestClient().search(searchRequest); + String result = scrollableSearch(searchResponse,scroll); + +// System.out.println(result); + + return result; + } + + + + public String searchPretty(String q) throws IOException { + + + return "
"+search(q)+"
"; //pre tag for json, otherwise it didnt show pretty in browser + + } + + public String getAllPretty() throws IOException { + + + return "
"+getAll()+"
"; //pre tag for json, otherwise it didnt show pretty in browser + + } + + public String searchPrettyByCriteria(String q, String criteria) throws IOException { + + return "
"+searchBy(q,criteria)+"
"; //pre tag for json, otherwise it didnt show pretty in browser + + } + + public String searchForm(String q) throws IOException { + + return search(q); + + } + + public String searchFormByCriteria(String q, String criteria) throws IOException { + + return searchBy(q,criteria); + + } + + //Helper methods + + //converts normal string to pretty Json String + public String toJson(String nonJsonString){ + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + JsonParser jp = new JsonParser(); + JsonElement je = jp.parse(nonJsonString); + String jsonString = gson.toJson(je); + return jsonString; + } + + + + public SearchSourceBuilder buildSource(String queryType, String criteria, String q, Boolean showAllFields){ + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + if (queryType.equals("match")) + { + searchSourceBuilder.query(matchQuery(criteria,q)); + } + else if (queryType.equals("term")) + { + searchSourceBuilder.query(QueryBuilders.termQuery(criteria,q)); + } + else + { + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + } + //we are allways sorting according to title, if it is needed this can be changed to related criteria + searchSourceBuilder.sort(new FieldSortBuilder("title.keyword").order(SortOrder.DESC)); + if (!showAllFields) + { + String[] includeFields = new String[] {"title",criteria}; + String[] excludeFields = new String[] {""}; + searchSourceBuilder.fetchSource(includeFields,excludeFields); + searchSourceBuilder.fetchSource(true); + } + + return searchSourceBuilder; + } + + public String scrollableSearch(SearchResponse searchResponse, Scroll scroll) throws IOException { + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + String result=""; + + //right now we are returning all results by concatenating results so it is not an elegant solution + while (searchHits != null && searchHits.length > 0) { + result+=toJson(searchResponse.toString()); + + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = oaiService.getRestClient().searchScroll(scrollRequest); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + + } + + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = oaiService.getRestClient().clearScroll(clearScrollRequest); + boolean succeeded = clearScrollResponse.isSucceeded(); + return result; + } + + +} diff --git a/src/main/java/io/academic/service/OaiService.java b/src/main/java/io/academic/service/OaiService.java index b380306..8b09a8d 100644 --- a/src/main/java/io/academic/service/OaiService.java +++ b/src/main/java/io/academic/service/OaiService.java @@ -11,21 +11,14 @@ import com.google.gson.JsonParser; import eu.luminis.elastic.document.DocumentService; import eu.luminis.elastic.document.IndexRequest; +import eu.luminis.elastic.document.UpdateRequest; import eu.luminis.elastic.index.IndexService; import eu.luminis.elastic.search.SearchService; import io.academic.dao.DcDao; -import io.academic.entity.Article; -import io.academic.entity.ArticleRepository; -import io.academic.entity.OaiRecord; -import io.academic.entity.OaiRecordRepository; -import org.apache.http.HttpEntity; +import io.academic.entity.*; import org.apache.http.HttpHost; -import org.apache.http.entity.ContentType; -import org.apache.http.nio.entity.NStringEntity; -import org.apache.http.util.EntityUtils; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.search.*; -import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; @@ -76,6 +69,10 @@ public class OaiService { private DcParseService dcParseService; + public RestHighLevelClient getRestClient() { + return restClient; + } + RestHighLevelClient restClient = new RestHighLevelClient( RestClient.builder( new HttpHost("localhost", 9200, "http"), new HttpHost("localhost", 9201, "http"))); @@ -140,6 +137,9 @@ public void saveRecords(List recordTypes) { article.setAuthors(parts[1].split("::")[1]); article.setKeywords(parts[2].split("::")[1]); article.setBody(parts[3].split("::")[1]); + article.setPublisher(parts[4].split("::")[1]); + article.setDate(parts[5].split("::")[1]); + article.setType(parts[6].split("::")[1]); article.setDc(parsedDc.getDc()); articles.add(article); @@ -183,98 +183,9 @@ private LocalDateTime parseDateTime(String string) { } - - public String search(String q) throws IOException { - - SearchRequest searchRequest = new SearchRequest("harvester"); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.termQuery("dc",q)); - searchSourceBuilder.sort(new FieldSortBuilder("title.keyword").order(SortOrder.DESC)); - searchSourceBuilder.fetchSource("title",""); - searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = restClient.search(searchRequest); - String result = searchResponse.toString(); - - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - JsonParser jp = new JsonParser(); - JsonElement je = jp.parse(result); - String prettyJsonString = gson.toJson(je); - - System.out.println(prettyJsonString); - - return "
"+prettyJsonString+"
"; //pre tag for json, otherwise it didnt show pretty in browser - - } - - public String searchForm(String q) throws IOException { - - SearchRequest searchRequest = new SearchRequest("harvester"); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.termQuery("dc",q)); - searchSourceBuilder.sort(new FieldSortBuilder("title.keyword").order(SortOrder.DESC)); - searchSourceBuilder.fetchSource("title",""); - searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = restClient.search(searchRequest); - String result = searchResponse.toString(); - - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - JsonParser jp = new JsonParser(); - JsonElement je = jp.parse(result); - String prettyJsonString = gson.toJson(je); - - System.out.println(prettyJsonString); - - return prettyJsonString; //pre tag for json, otherwise it didnt show pretty in browser - - } - - public String getAll() throws IOException { - - - final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); - - SearchRequest searchRequest = new SearchRequest("harvester"); - searchRequest.scroll(scroll); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchRequest.source(searchSourceBuilder); - - SearchResponse searchResponse = restClient.search(searchRequest); - String scrollId = searchResponse.getScrollId(); - SearchHit[] searchHits = searchResponse.getHits().getHits(); - String result=""; - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - JsonParser jp = new JsonParser(); - JsonElement je ; - String prettyJsonString ; - while (searchHits != null && searchHits.length > 0) { - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - searchResponse = restClient.searchScroll(scrollRequest); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); - - - je = jp.parse(searchResponse.toString()); - prettyJsonString = gson.toJson(je); - result+=prettyJsonString; - - - - } - - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = restClient.clearScroll(clearScrollRequest); - boolean succeeded = clearScrollResponse.isSucceeded(); - - - return "
"+result+"
"; //pre tag for json, otherwise it didnt show pretty in browser - } - public void delete() throws IOException { - + //TODO:check if there is any indices with that name DeleteIndexRequest request = new DeleteIndexRequest("harvester"); restClient.indices().deleteIndex(request); diff --git a/src/main/java/io/academic/service/ProcessorService.java b/src/main/java/io/academic/service/ProcessorService.java index 6a2ca86..bd66d5c 100644 --- a/src/main/java/io/academic/service/ProcessorService.java +++ b/src/main/java/io/academic/service/ProcessorService.java @@ -35,7 +35,7 @@ public void startProcessors() { taskExecutor.execute(recordTypeProcessor); taskExecutor.execute(urlProcessor); -// oaiDataProviderService.queue(new OaiDataProvider("Acta Medica Anatolia","http://dergipark.gov.tr/api/public/oai/","dergipark.ulakbim.gov.tr" )); + oaiDataProviderService.queue(new OaiDataProvider("Acta Medica Anatolia","http://dergipark.gov.tr/api/public/oai/","dergipark.ulakbim.gov.tr" )); // oaiDataProviderService.queue(new OaiDataProvider("http://export.arxiv.org/oai2")); try { oaiService.delete(); diff --git a/src/main/resources/templates/searchform.html b/src/main/resources/templates/searchform.html index 91b9cd1..b18121a 100644 --- a/src/main/resources/templates/searchform.html +++ b/src/main/resources/templates/searchform.html @@ -10,6 +10,16 @@

OAI Harvester Search Form

Search:

+

Criteria:

diff --git a/src/main/resources/templates/searchresult.html b/src/main/resources/templates/searchresult.html index 99a9691..b43f52a 100644 --- a/src/main/resources/templates/searchresult.html +++ b/src/main/resources/templates/searchresult.html @@ -9,6 +9,7 @@

Result

+

Search moooore