Merge branch 'feature/EMC-27-big-ttl' into 'develop'

EMC-26 EMC-27 Fuseki publishing Closes EMC-27 See merge request eip/catalogue!544
NERC-CEH · Oct 11, 2023 · fd24087 · fd24087
2 parents f007f5c + ba2bbc3
commit fd24087
Show file tree

Hide file tree

Showing 16 changed files with 465 additions and 142 deletions.
diff --git a/README.md b/README.md
@@ -59,6 +59,7 @@ crowd.password=
 plone.password=
 doi.password=
 hubbub.password=
+fuseki.password=
 ```
 
 ## Getting started

diff --git a/docs/profiles.md b/docs/profiles.md
@@ -41,8 +41,9 @@ Choice of:
 * `upload:hubbub` Upload managed through EIDC Hubbub server
 * `upload:simple` Upload to directory
 
-## Imports
+## Imports and Exports
 * `imports` Enable import of records from external services (see [imports documentation](imports.md))
+* `exports` Enable exporting records to a Fuseki instance
 
 ## Search
 Configures how search works

diff --git a/fixtures/datastore/REV-1/c88921ba-f871-44c3-9339-51c5bee4024a.raw b/fixtures/datastore/REV-1/c88921ba-f871-44c3-9339-51c5bee4024a.raw
diff --git a/java/src/main/java/uk/ac/ceh/gateway/catalogue/TimeConstants.java b/java/src/main/java/uk/ac/ceh/gateway/catalogue/TimeConstants.java
@@ -2,5 +2,6 @@
 
 public class TimeConstants {
     public static final int ONE_MINUTE = 60000;
+    public static final int ONE_DAY = 86400000;
     public static final int SEVEN_DAYS = 604800000;
 }
diff --git a/java/src/main/java/uk/ac/ceh/gateway/catalogue/exports/CatalogueExportService.java b/java/src/main/java/uk/ac/ceh/gateway/catalogue/exports/CatalogueExportService.java
@@ -0,0 +1,8 @@
+package uk.ac.ceh.gateway.catalogue.exports;
+
+import org.springframework.context.annotation.Profile;
+
+@Profile("exports")
+public interface CatalogueExportService {
+    void runExport();
+}
diff --git a/java/src/main/java/uk/ac/ceh/gateway/catalogue/services/FusekiExportService.java b/java/src/main/java/uk/ac/ceh/gateway/catalogue/services/FusekiExportService.java
@@ -0,0 +1,160 @@
+package uk.ac.ceh.gateway.catalogue.services;
+
+import freemarker.template.Configuration;
+import lombok.SneakyThrows;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Profile;
+import org.springframework.http.*;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
+import org.springframework.web.client.RestClientResponseException;
+import org.springframework.web.client.RestTemplate;
+import uk.ac.ceh.gateway.catalogue.TimeConstants;
+import uk.ac.ceh.gateway.catalogue.catalogue.Catalogue;
+import uk.ac.ceh.gateway.catalogue.catalogue.CatalogueService;
+import uk.ac.ceh.gateway.catalogue.exports.CatalogueExportService;
+import uk.ac.ceh.gateway.catalogue.model.MetadataDocument;
+import uk.ac.ceh.gateway.catalogue.repository.DocumentRepository;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static uk.ac.ceh.gateway.catalogue.util.Headers.withBasicAuth;
+
+@Profile("exports")
+@Slf4j
+@Service
+@ToString
+public class FusekiExportService implements CatalogueExportService {
+    private final Configuration configuration;
+    private final DocumentRepository documentRepository;
+    private final MetadataListingService listing;
+    private final RestTemplate restTemplate;
+    private final String baseUri;
+    private final String fusekiUrl;
+    private final String fusekiUsername;
+    private final String fusekiPassword;
+
+    private final String catalogueId;
+    private final String catalogueTitle;
+
+    public FusekiExportService(
+            CatalogueService catalogueService,
+            Configuration configuration,
+            DocumentRepository documentRepository,
+            MetadataListingService listing,
+            @Qualifier("normal") RestTemplate restTemplate,
+            @Value("${documents.baseUri}") String baseUri,
+            @Value("${fuseki.url}/ds") String fusekiUrl,
+            @Value("${fuseki.username}") String fusekiUsername,
+            @Value("${fuseki.password}") String fusekiPassword
+    ) {
+        log.info("Creating");
+
+        this.configuration = configuration;
+        this.documentRepository = documentRepository;
+        this.listing = listing;
+        this.restTemplate = restTemplate;
+        this.baseUri = baseUri;
+        this.fusekiUrl = fusekiUrl;
+        this.fusekiUsername = fusekiUsername;
+        this.fusekiPassword = fusekiPassword;
+
+        Catalogue defaultCatalogue = catalogueService.defaultCatalogue();
+        this.catalogueId = defaultCatalogue.getId();
+        this.catalogueTitle = defaultCatalogue.getTitle();
+    }
+
+    @Scheduled(initialDelay = TimeConstants.ONE_MINUTE, fixedDelay = TimeConstants.ONE_DAY)
+    @SneakyThrows
+    public void runExport() {
+        post(getBigTtl());
+        log.info("Posted public metadata documents as ttl to {}", fusekiUrl);
+    }
+
+    private String getBigTtl() {
+        List<String> ids = getRequiredIds();
+        String catalogueTtl = generateCatalogueTtl(getCatalogueModel(ids));
+        List<String> recordsTtl = getRecordsTtl(ids);
+
+        String bigTtl = catalogueTtl.concat(String.join("\n", recordsTtl));
+        log.debug("Big turtle to send: ", bigTtl);
+        return bigTtl;
+    }
+
+    private List<String> getRequiredIds(){
+        List<String> ids = listing.getPublicDocumentsOfCatalogue(catalogueId);
+        return ids.stream()
+                .map(this::getMetadataDocument)
+                .filter(this::isRequired)
+                .map(MetadataDocument::getId)
+                .collect(Collectors.toList());
+    }
+
+    @SneakyThrows
+    public String generateCatalogueTtl(Map<String, Object> model){
+        val freemarkerTemplate = configuration.getTemplate("rdf/catalogue.ttl.ftlh");
+        return FreeMarkerTemplateUtils.processTemplateIntoString(freemarkerTemplate, model);
+    }
+
+    private Map<String, Object> getCatalogueModel(List<String> ids){
+        Map<String, Object> model = new HashMap<>();
+        model.put("records", ids);
+        model.put("catalogue", catalogueId);
+        model.put("title", catalogueTitle);
+        model.put("baseUri", baseUri);
+        return model;
+    }
+
+    private List<String> getRecordsTtl(List<String> ids){
+        return ids.stream()
+                .map(this::getMetadataDocument)
+                .map(this::docToString)
+                .collect(Collectors.toList());
+    }
+
+    @SneakyThrows
+    private MetadataDocument getMetadataDocument(String id) {
+        return documentRepository.read(id);
+    }
+
+    private boolean isRequired(MetadataDocument doc) {
+        String[] requiredTypes = {"service","dataset"};
+        return Arrays.asList(requiredTypes).contains(doc.getType());
+    }
+
+    @SneakyThrows
+    public String docToString(MetadataDocument model){
+        val freemarkerTemplate = configuration.getTemplate("rdf/ttlUnprefixed.ftlh");
+        return FreeMarkerTemplateUtils.processTemplateIntoString(freemarkerTemplate, model);
+    }
+
+    private void post(String data){
+        String graphName = baseUri; //this is from the first line after the prefixes in the big.ttl - which we've set to be baseUri that is injected into the template earlier in this code
+        String serverUrl = new StringBuilder().append(fusekiUrl).append("?graph=").append(graphName).toString();
+
+        try {
+            HttpHeaders headers = withBasicAuth(fusekiUsername, fusekiPassword);
+            headers.add(HttpHeaders.CONTENT_TYPE, "text/turtle");
+
+            HttpEntity<String> request = new HttpEntity<>(data, headers);
+            ResponseEntity<String> response = restTemplate.postForEntity(serverUrl, request, String.class);
+            log.info("Status code: {}", response.getStatusCode());
+            log.info("Response {}", response);
+        } catch (RestClientResponseException ex) {
+            log.error(
+                    "Error communicating with supplied URL: (statusCode={}, status={}, headers={}, body={})",
+                    ex.getRawStatusCode(),
+                    ex.getStatusText(),
+                    ex.getResponseHeaders(),
+                    ex.getResponseBodyAsString()
+            );
+            throw ex;
+        }
+    }
+}
diff --git a/java/src/main/resources/application.properties b/java/src/main/resources/application.properties
@@ -45,3 +45,5 @@ spring.servlet.multipart.location=/var/ceh-catalogue/dropbox
 spring.servlet.multipart.max-file-size=2GB
 spring.servlet.multipart.max-request-size=2GB
 upload.simple.datastore=/var/upload/datastore
+fuseki.url=https://eidc-fuseki.staging.ceh.ac.uk
+fuseki.username=admin
diff --git a/java/src/test/java/uk/ac/ceh/gateway/catalogue/services/FusekiExportServiceTest.java b/java/src/test/java/uk/ac/ceh/gateway/catalogue/services/FusekiExportServiceTest.java
@@ -0,0 +1,100 @@
+package uk.ac.ceh.gateway.catalogue.services;
+
+import freemarker.template.Configuration;
+
+import lombok.SneakyThrows;
+
+import java.io.File;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.test.web.client.MockRestServiceServer;
+import org.springframework.web.client.RestTemplate;
+
+import uk.ac.ceh.components.datastore.DataRepositoryException;
+import uk.ac.ceh.gateway.catalogue.catalogue.Catalogue;
+import uk.ac.ceh.gateway.catalogue.catalogue.CatalogueService;
+import uk.ac.ceh.gateway.catalogue.repository.DocumentRepository;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Mockito.verify;
+
+import static org.springframework.test.web.client.match.MockRestRequestMatchers.*;
+import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess;
+
+@ExtendWith(MockitoExtension.class)
+public class FusekiExportServiceTest {
+    private FusekiExportService service;
+
+    @Mock private CatalogueService catalogueService;
+    @Mock private DocumentRepository documentRepository;
+    @Mock private MetadataListingService listing;
+    private RestTemplate restTemplate;
+    private MockRestServiceServer mockServer;
+
+    private static final String BASE_URI = "http://catalogue.invalid/";
+    private static final String FUSEKI_URL = "http://fuseki.invalid/";
+    private static final String FUSEKI_USERNAME = "username";
+    private static final String FUSEKI_PASSWORD = "password";
+    private static final String CATALOGUE_ID = "testId";
+    private static final String CATALOGUE_TITLE = "Test catalogue title";
+
+    private Configuration configuration = new Configuration();
+
+    private Catalogue catalogue = Catalogue.builder()
+        .id(CATALOGUE_ID)
+        .title(CATALOGUE_TITLE)
+        .url("n/a")
+        .contactUrl("n/a")
+        .logo("n/a")
+        .build();
+
+    @SneakyThrows
+    @BeforeEach
+    public void setup() {
+        restTemplate = new RestTemplate();
+        mockServer = MockRestServiceServer.bindTo(restTemplate).build();
+        configuration.setDirectoryForTemplateLoading(new File("../templates"));
+        // called in constructor of FusekiExportService
+        given(catalogueService.defaultCatalogue())
+            .willReturn(catalogue);
+
+        service = new FusekiExportService(
+                catalogueService,
+                configuration,
+                documentRepository,
+                listing,
+                restTemplate,
+                BASE_URI,
+                FUSEKI_URL,
+                FUSEKI_USERNAME,
+                FUSEKI_PASSWORD
+                );
+    }
+
+    @Test
+    @SneakyThrows
+    public void exportDocuments() throws DataRepositoryException {
+        // given
+        mockServer
+                .expect(requestTo(equalTo(FUSEKI_URL + "?graph=" + BASE_URI)))
+                .andExpect(method(HttpMethod.POST))
+                .andExpect(header(HttpHeaders.CONTENT_TYPE, "text/turtle"))
+                .andExpect(header(HttpHeaders.AUTHORIZATION, "Basic dXNlcm5hbWU6cGFzc3dvcmQ="))
+                .andRespond(withSuccess());
+        // when
+        service.runExport();
+
+        // then
+        mockServer.verify();
+    }
+}
diff --git a/templates/rdf/_body.ftlh b/templates/rdf/_body.ftlh
@@ -0,0 +1,82 @@
+:${id}
+  dct:title "${title}" ;
+  <#if description?has_content>
+  dct:description "${description?replace("\n", " ")}" ;
+  </#if>
+  <#if boundingBoxes?has_content && boundingBoxes?has_content>
+    <#list boundingBoxes as extent>
+       dct:spatial "${extent.wkt}"^^geo:wktLiteral ;
+    </#list>
+  </#if>
+  <#if temporalExtents?has_content>
+    <#list temporalExtents as extent>
+       dct:temporal "${(extent.begin?date)!''}/${(extent.end?date)!''}"^^dct:PeriodOfTime ;
+    </#list>
+  </#if>
+
+  <#--Points of contact2-->
+  <#if pointsOfContact?has_content>
+    dcat:contactPoint <@contactList pointsOfContact "c" />  ;
+  </#if>
+
+  <#--Publisher-->
+  <#if publishers?has_content>
+    dct:publisher  <@contactList publishers "pub" /> ;
+  </#if>
+
+  dct:language "eng" ;
+  <#assign rel_memberOf = jena.relationships(uri, "https://vocabs.ceh.ac.uk/eidc#memberOf")>
+  <#if rel_memberOf?has_content && rel_memberOf?size gt 0>
+  dct:isPartOf
+    <#list rel_memberOf as item>
+      <${item.href}><#sep>,
+    </#list>
+  ;
+  </#if>
+
+  <#-- Keywords -->
+  <#assign allKeywords = []>
+  <#if descriptiveKeywords?has_content>
+    <#list descriptiveKeywords as descriptiveKeyword>
+        <#assign allKeywords = allKeywords + descriptiveKeyword.keywords >
+    </#list>
+  </#if>
+
+  <#if allKeywords?has_content>
+  dct:subject
+    <#list allKeywords as keyword>
+      <#if keyword.uri?has_content>
+        <#if keyword.value?has_content>
+          <${keyword.uri}>, "${keyword.value}"<#t>
+        <#else>
+          "${keyword.uri}"<#t>
+        </#if>
+      <#else>
+        "${keyword.value}"<#t>
+      </#if><#sep>,</#sep><#t>
+    </#list>
+  ;
+  </#if>
+
+  <#if type=='dataset' || type=='nonGeographicDataset' || type=='signpost'>
+    <#include "turtle/_dataset.ftlh">
+  <#elseif type=='aggregate'|| type=='collection'|| type=='series'>
+    <#include "turtle/_aggregation.ftlh">
+  <#elseif type=='service'>
+    <#include "turtle/_service.ftlh">
+  <#elseif type=='application'>
+    <#include "turtle/_application.ftlh">
+  <#else>
+  </#if>
+.
+
+
+<#if pointsOfContact?has_content>
+  <@contactDetail pointsOfContact "c" />
+</#if>
+<#if publishers?has_content>
+  <@contactDetail publishers "pub" />
+</#if>
+<#if authors?has_content>
+  <@contactDetail authors "a" />
+</#if>