Skip to content

Commit

Permalink
Merge branch 'EMC-73-big-ttl' into 'develop'
Browse files Browse the repository at this point in the history 
Factor out Turtle file generation and expose endpoint via WholeCatalogueTurtleController

Closes EMC-73

See merge request eip/catalogue!571
  • Loading branch information
@rodscott
rodscott committed Feb 6, 2024
2 parents ef0e243 + e975304 commit 2a9ab09
Show file tree
Hide file tree
Showing 7 changed files with 320 additions and 110 deletions.
38 changes: 38 additions & 0 deletions ...src/main/java/uk/ac/ceh/gateway/catalogue/controllers/WholeCatalogueTurtleController.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package uk.ac.ceh.gateway.catalogue.controllers;

import lombok.SneakyThrows;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpEntity;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import uk.ac.ceh.gateway.catalogue.services.DocumentsToTurtleService;

@Slf4j
@ToString
@Controller
public class WholeCatalogueTurtleController {

private final DocumentsToTurtleService docsToTurtle;

public WholeCatalogueTurtleController(DocumentsToTurtleService docsToTurtle) {
this.docsToTurtle = docsToTurtle;
}

@GetMapping("{catalogueId}/catalogue.ttl")
@SneakyThrows
public HttpEntity<String> getTtl(@PathVariable String catalogueId) {
return docsToTurtle.getBigTtl(catalogueId).map(ttl -> {
log.info("serving big turtle for {}", catalogueId);
return ResponseEntity.ok()
.contentType(MediaType.valueOf("text/turtle"))
.body(ttl);
}).orElseGet(() -> {
log.info("not serving big turtle for unknown catalogue {}", catalogueId);
return ResponseEntity.notFound().build();
});
}
}
107 changes: 107 additions & 0 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/services/CatalogueToTurtleService.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package uk.ac.ceh.gateway.catalogue.services;

import com.google.common.collect.ImmutableSet;
import freemarker.template.Configuration;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
import uk.ac.ceh.gateway.catalogue.catalogue.Catalogue;
import uk.ac.ceh.gateway.catalogue.catalogue.CatalogueService;
import uk.ac.ceh.gateway.catalogue.model.MetadataDocument;
import uk.ac.ceh.gateway.catalogue.repository.DocumentRepository;

import java.util.*;
import java.util.stream.Collectors;

@Slf4j
@Service
public class CatalogueToTurtleService implements DocumentsToTurtleService {
private static final Set<String> REQUIRED_TYPES = ImmutableSet.of(
"service",
"dataset",
"signpost"
);
private final CatalogueService catalogueService;
private final Configuration configuration;
private final DocumentRepository documentRepository;
private final MetadataListingService listing;
private final String baseUri;

public CatalogueToTurtleService(
CatalogueService catalogueService,
DocumentRepository documentRepository,
Configuration configuration,
MetadataListingService listing,
@Value("${documents.baseUri}") String baseUri
) {
this.catalogueService = catalogueService;
this.configuration = configuration;
this.documentRepository = documentRepository;
this.baseUri = baseUri;
this.listing = listing;
}

@Override
public Optional<String> getBigTtl(String catalogueId) {
return Optional.ofNullable(catalogueService.retrieve(catalogueId)).map(catalogue -> {
List<String> ids = getRequiredIds(catalogueId);
String catalogueTtl = generateCatalogueTtl(getCatalogueModel(catalogue, ids));
List<String> recordsTtl = getRecordsTtl(ids);

String bigTtl = catalogueTtl.concat(String.join("\n", recordsTtl));
log.debug("Big turtle to send: {}", bigTtl);
return bigTtl;
});
}

private List<String> getRequiredIds(String catalogueId) {
try {
List<String> ids = listing.getPublicDocumentsOfCatalogue(catalogueId);

return ids.stream()
.map(this::getMetadataDocument)
.filter(doc -> REQUIRED_TYPES.contains(doc.getType()))
.map(MetadataDocument::getId)
.collect(Collectors.toList());
} catch (NullPointerException e) {
// no git commits
return Collections.emptyList();
}
}

@SneakyThrows
private String generateCatalogueTtl(Map<String, Object> model) {
val freemarkerTemplate = configuration.getTemplate("rdf/catalogue.ttl.ftlh");
return FreeMarkerTemplateUtils.processTemplateIntoString(freemarkerTemplate, model);
}

private Map<String, Object> getCatalogueModel(Catalogue catalogue, List<String> ids) {
Map<String, Object> model = new HashMap<>();
model.put("records", ids);
model.put("catalogue", catalogue.getId());
model.put("title", catalogue.getTitle());
model.put("baseUri", baseUri);
return model;
}

private List<String> getRecordsTtl(List<String> ids) {
return ids.stream()
.map(this::getMetadataDocument)
.map(this::docToString)
.collect(Collectors.toList());
}

@SneakyThrows
private MetadataDocument getMetadataDocument(String id) {
return documentRepository.read(id);
}

@SneakyThrows
private String docToString(MetadataDocument model) {
val freemarkerTemplate = configuration.getTemplate("rdf/ttlUnprefixed.ftlh");
return FreeMarkerTemplateUtils.processTemplateIntoString(freemarkerTemplate, model);
}
}
7 changes: 7 additions & 0 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/services/DocumentsToTurtleService.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package uk.ac.ceh.gateway.catalogue.services;

import java.util.Optional;

public interface DocumentsToTurtleService {
Optional<String> getBigTtl(String catalogueId);
}
133 changes: 33 additions & 100 deletions java/src/main/java/uk/ac/ceh/gateway/catalogue/services/FusekiExportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,145 +4,78 @@
import lombok.SneakyThrows;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.http.*;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
import org.springframework.web.client.RestClientResponseException;
import org.springframework.web.client.RestTemplate;
import uk.ac.ceh.gateway.catalogue.TimeConstants;
import uk.ac.ceh.gateway.catalogue.catalogue.Catalogue;
import uk.ac.ceh.gateway.catalogue.catalogue.CatalogueService;
import uk.ac.ceh.gateway.catalogue.exports.CatalogueExportService;
import uk.ac.ceh.gateway.catalogue.model.MetadataDocument;
import uk.ac.ceh.gateway.catalogue.repository.DocumentRepository;

import java.util.*;
import java.util.stream.Collectors;

import static uk.ac.ceh.gateway.catalogue.util.Headers.withBasicAuth;

@Profile("exports")
@Slf4j
@Service
@ToString
public class FusekiExportService implements CatalogueExportService {
private final Configuration configuration;
private final DocumentRepository documentRepository;
private final MetadataListingService listing;
private final RestTemplate restTemplate;
private final String baseUri;
private final String fusekiUrl;
private final String fusekiUsername;
private final String fusekiPassword;

private final String catalogueId;
private final String catalogueTitle;

private final DocumentsToTurtleService documentsToTurtleService;

public FusekiExportService(
CatalogueService catalogueService,
Configuration configuration,
DocumentRepository documentRepository,
MetadataListingService listing,
@Qualifier("normal") RestTemplate restTemplate,
@Value("${documents.baseUri}") String baseUri,
@Value("${fuseki.url}/ds") String fusekiUrl,
@Value("${fuseki.username}") String fusekiUsername,
@Value("${fuseki.password}") String fusekiPassword
CatalogueService catalogueService,
Configuration configuration,
DocumentRepository documentRepository,
MetadataListingService listing,
@Qualifier("normal") RestTemplate restTemplate,
@Value("${documents.baseUri}") String baseUri,
@Value("${fuseki.url}/ds") String fusekiUrl,
@Value("${fuseki.username}") String fusekiUsername,
@Value("${fuseki.password}") String fusekiPassword
) {
log.info("Creating");

this.configuration = configuration;
this.documentRepository = documentRepository;
this.listing = listing;
this.restTemplate = restTemplate;
this.baseUri = baseUri;
this.fusekiUrl = fusekiUrl;
this.fusekiUsername = fusekiUsername;
this.fusekiPassword = fusekiPassword;

Catalogue defaultCatalogue = catalogueService.defaultCatalogue();
this.catalogueId = defaultCatalogue.getId();
this.catalogueTitle = defaultCatalogue.getTitle();
this.catalogueId = catalogueService.defaultCatalogue().getId();

this.documentsToTurtleService = new CatalogueToTurtleService(
catalogueService,
documentRepository,
configuration,
listing,
baseUri
);
}

@Scheduled(initialDelay = TimeConstants.ONE_MINUTE, fixedDelay = TimeConstants.ONE_DAY)
@SneakyThrows
public void runExport() {
post(getBigTtl());
log.info("Posted public metadata documents as ttl to {}", fusekiUrl);
}

private String getBigTtl() {
List<String> ids = getRequiredIds();
String catalogueTtl = generateCatalogueTtl(getCatalogueModel(ids));
List<String> recordsTtl = getRecordsTtl(ids);

String bigTtl = catalogueTtl.concat(String.join("\n", recordsTtl));
log.debug("Big turtle to send: {}", bigTtl);
return bigTtl;
}

private List<String> getRequiredIds(){
try {
List<String> ids = listing.getPublicDocumentsOfCatalogue(catalogueId);

return ids.stream()
.map(this::getMetadataDocument)
.filter(this::isRequired)
.map(MetadataDocument::getId)
.collect(Collectors.toList());
} catch(NullPointerException e) {
// no git commits
return new ArrayList<>();
}
}

@SneakyThrows
public String generateCatalogueTtl(Map<String, Object> model){
val freemarkerTemplate = configuration.getTemplate("rdf/catalogue.ttl.ftlh");
return FreeMarkerTemplateUtils.processTemplateIntoString(freemarkerTemplate, model);
}

private Map<String, Object> getCatalogueModel(List<String> ids){
Map<String, Object> model = new HashMap<>();
model.put("records", ids);
model.put("catalogue", catalogueId);
model.put("title", catalogueTitle);
model.put("baseUri", baseUri);
return model;
}

private List<String> getRecordsTtl(List<String> ids){
return ids.stream()
.map(this::getMetadataDocument)
.map(this::docToString)
.collect(Collectors.toList());
}

@SneakyThrows
private MetadataDocument getMetadataDocument(String id) {
return documentRepository.read(id);
}

private boolean isRequired(MetadataDocument doc) {
String[] requiredTypes = {"service","dataset","signpost"};
return Arrays.asList(requiredTypes).contains(doc.getType());
}

@SneakyThrows
public String docToString(MetadataDocument model){
val freemarkerTemplate = configuration.getTemplate("rdf/ttlUnprefixed.ftlh");
return FreeMarkerTemplateUtils.processTemplateIntoString(freemarkerTemplate, model);
documentsToTurtleService.getBigTtl(catalogueId).ifPresent(ttl -> {
post(ttl);
log.info("Posted public metadata documents as ttl to {}", fusekiUrl);
});
}

private void post(String data){
private void post(String data) {
String graphName = baseUri; //this is from the first line after the prefixes in the big.ttl - which we've set to be baseUri that is injected into the template earlier in this code
String serverUrl = new StringBuilder().append(fusekiUrl).append("?graph=").append(graphName).toString();
String serverUrl = fusekiUrl + "?graph=" + graphName;

try {
// PUT the data - this works if there's no graph and if there's an existing graph, in which case it's updated
Expand All @@ -152,11 +85,11 @@ private void post(String data){
restTemplate.put(serverUrl, request);
} catch (RestClientResponseException ex) {
log.error(
"Error communicating with supplied URL: (statusCode={}, status={}, headers={}, body={})",
ex.getRawStatusCode(),
ex.getStatusText(),
ex.getResponseHeaders(),
ex.getResponseBodyAsString()
"Error communicating with supplied URL: (statusCode={}, status={}, headers={}, body={})",
ex.getRawStatusCode(),
ex.getStatusText(),
ex.getResponseHeaders(),
ex.getResponseBodyAsString()
);
throw ex;
}
Expand Down
Loading

0 comments on commit 2a9ab09

Please sign in to comment.