diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6be2ee8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ +# Guidelines for Pull Requests + +If you haven't yet read our code review guidelines, please do so, You can find them [here](https://diging.atlassian.net/wiki/spaces/DIGING/pages/2256076801/Code+Review+Guidelines). + +Please confirm the following by adding an x for each item (turn `[ ]` into `[x]`). + +- [ ] I have removed all code style changes that are not necessary (e.g. changing blanks across the whole file that don’t need to be changed, adding empty lines in parts other than your own code) +- [ ] I am not making any changes to files that don’t have any effect (e.g. imports added that don’t need to be added) +- [ ] I do not have any sysout statements in my code or commented out code that isn’t needed anymore +- [ ] I am not reformatting any files in the wrong format or without cause. +- [ ] I am not changing file encoding or line endings to something else than UTF-8, LF +- [ ] My pull request does not show an insane amount of files being changed although my ticket only requires a few files being changed +- [ ] I have added Javadoc/documentation where appropriate +- [ ] I have added test cases where appropriate +- [ ] I have explained any part of my code/implementation decisions that is not be self-explanatory + +## Please provide a brief description of your ticket +(you can copy the ticket if it hasn't changed) + + + +## Anything else the reviewer needs to know? + + diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..1060384 --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/nepomuk/pom.xml b/nepomuk/pom.xml index ae1e815..cd2f75f 100644 --- a/nepomuk/pom.xml +++ b/nepomuk/pom.xml @@ -13,10 +13,10 @@ 1.7.5 4.3.2.RELEASE 4.1.3.RELEASE - 0.10 - 0.4.3 + 0.16 + 0.6 0.3 - 0.1 + 0.2 $2a$04$NXKVTx9XrpEST6CLn/ps7eP8YsK3t.5A/mvAoYpg.X8QqgOKtc1za @@ -60,7 +60,6 @@ - edu.asu.diging @@ -68,8 +67,8 @@ ${geco.requests.version} - edu.asu.diging.giles-eco - util + edu.asu.diging + giles-eco-util ${geco.util.version} @@ -78,8 +77,8 @@ ${geco.september.util.version} - edu.asu.diging.giles-eco - kafka-util + edu.asu.diging + giles-eco-kafka-util ${geco.kafka-util.version} @@ -408,7 +407,26 @@ curator-framework 2.12.0 + + + + jakarta.xml.bind + jakarta.xml.bind-api + 2.3.2 + + + + + org.glassfish.jaxb + jaxb-runtime + 2.3.2 + + + javax.xml.bind + jaxb-api + 2.4.0-b180830.0359 + @@ -429,7 +447,7 @@ org.apache.maven.plugins maven-war-plugin - 2.6 + 3.3.2 @@ -462,8 +480,8 @@ maven-compiler-plugin - 1.8 - 1.8 + 11 + 11 @@ -531,7 +549,7 @@ - scm:svn:http://none - scm:svn:https://none + scm:git:http://none + scm:git:https://none diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/config/NepomukKafkaConfig.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/config/NepomukKafkaConfig.java index 6a38d1b..abe844a 100644 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/config/NepomukKafkaConfig.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/config/NepomukKafkaConfig.java @@ -15,6 +15,7 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import edu.asu.diging.gilesecosystem.nepomuk.core.service.properties.Properties; +import edu.asu.diging.gilesecosystem.nepomuk.core.kafka.impl.StorageDeletionRequestReceiver; import edu.asu.diging.gilesecosystem.nepomuk.core.kafka.impl.StorageRequestReceiver; import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager; @@ -65,6 +66,11 @@ public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { public StorageRequestReceiver receiver() { return new StorageRequestReceiver(); } + + @Bean + public StorageDeletionRequestReceiver deletionReceiver() { + return new StorageDeletionRequestReceiver(); + } @Override public String getHosts() { diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFileStorageManager.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFileStorageManager.java index bc45131..32d9894 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFileStorageManager.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFileStorageManager.java @@ -35,5 +35,31 @@ public abstract String getAndCreateStoragePath(String username, public abstract String getFileFolderPathInBaseFolder(String username, String uploadId, String fileId); + + /** + * Method to delete a file from storage + * + * @param username + * username of user who uploaded an image + * @param uploadId + * id of upload a file was part of + * @param documentId + * id of document + * @param fileId + * id of file + * @param deleteEmptyFolders + * if empty folders need to be deleted + * @throws NepomukFileStorageException + */ + public abstract void deleteFile(String username, String uploadId, String documentId, String fileId) throws NepomukFileStorageException; -} \ No newline at end of file + /** + * Checks if a file exists at the specified location. + * @param username The username associated with the file. + * @param uploadId The unique identifier for the upload. + * @param documentId The identifier for the document. + * @param fileName The name of the file to check. + * @return {@code true} if the file exists at the specified location, {@code false} otherwise. + */ + public abstract boolean checkIfFileExists(String username, String uploadId, String documentId, String fileName); +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesDatabaseClient.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesDatabaseClient.java index bfa08f2..b30477e 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesDatabaseClient.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesDatabaseClient.java @@ -29,4 +29,12 @@ public abstract IFile getFile(String uploadId, String documentId, String filenam List getUsernames(); -} \ No newline at end of file + /** + * Delete a file given the file ID. + * @param fileId + * ID of the file to be deleted + */ + public abstract void deleteFile(String fileId); + + public abstract List getFilesByDocumentId(String documentId); +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesManager.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesManager.java index eefb4c5..40d1ac0 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesManager.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesManager.java @@ -35,4 +35,14 @@ public abstract IFile getFile(String uploadId, String documentId, String filenam throws NoUniqueResultException; List getKnownUsernames(); -} \ No newline at end of file + + /** + * Delete a file given the file ID. + * @param fileId + * ID of the file to be deleted + */ + + public abstract void deleteFile(String fileId); + + public abstract List getFilesByDocumentId(String documentId); +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FileStorageManager.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FileStorageManager.java index 30a7bd1..c143692 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FileStorageManager.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FileStorageManager.java @@ -5,6 +5,9 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import org.springframework.stereotype.Service; @@ -49,8 +52,7 @@ public void saveFile(String username, String uploadId, String documentId, @Override public String getAndCreateStoragePath(String username, String uploadId, String documentId) { - String path = baseDirectory + File.separator - + getFileFolderPathInBaseFolder(username, uploadId, documentId); + String path = getStoragePath(username, uploadId, documentId); createDirectory(path); return path; } @@ -119,5 +121,42 @@ private boolean createDirectory(String dirPath) { public void setFileTypeFolder(String fileTypeFolder) { this.fileTypeFolder = fileTypeFolder; } + + private String getStoragePath(String username, String uploadId, + String documentId) { + return baseDirectory + File.separator + + getFileFolderPathInBaseFolder(username, uploadId, documentId); + } + + public void deleteFile(String username, String uploadId, String documentId, String fileName) throws NepomukFileStorageException { + String originalFilePath = getStoragePath(username, uploadId, documentId); + Path path = Paths.get(originalFilePath + File.separator + fileName); + try { + Files.delete(path); + } catch (IOException ex) { + throw new NepomukFileStorageException("Could not delete file.", ex); + } + String documentFolder = getStoragePath(username, uploadId, documentId); + File docFolder = new File(documentFolder); + if (docFolder.isDirectory() && docFolder.list().length == 0) { + // If another process adds a file to the folder just before deletion, we do not want to delete the folder. + // The goal is to ensure that the folder remains intact even if new files are added during the deletion process. + // Using the File.delete() instead of Files.delete(path) as an exception is not required to be thrown in this scenario. + boolean deleted = docFolder.delete(); + if (deleted) { + Path documentFolderPath = Paths.get(documentFolder); + Path uploadFolderDirectory = documentFolderPath.getParent(); + File uploadFolder = new File(uploadFolderDirectory.toString()); + if (uploadFolder.isDirectory() && uploadFolder.list().length == 0) { + uploadFolder.delete(); + } + } + } + } + @Override + public boolean checkIfFileExists(String username, String uploadId, String documentId, String fileName) { + Path path = Paths.get(getStoragePath(username, uploadId, documentId) + File.separator + fileName); + return Files.exists(path); + } } diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesDatabaseClient.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesDatabaseClient.java index 8e6e350..b7633da 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesDatabaseClient.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesDatabaseClient.java @@ -56,6 +56,13 @@ public List getFilesByUploadId(String uploadId) { return results; } + @Override + public List getFilesByDocumentId(String documentId) { + List results = new ArrayList(); + searchByProperty("documentId", documentId, File.class).forEach(f -> results.add((IFile)f)); + return results; + } + @Override public List getFilesByUsername(String username) { List results = new ArrayList(); @@ -120,4 +127,9 @@ public List getUsernames() { TypedQuery docs = em.createQuery(query, String.class); return docs.getResultList(); } + + @Override + public void deleteFile(String fileId) { + em.remove(getById(fileId)); + } } diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesManager.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesManager.java index 1751026..c973de9 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesManager.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesManager.java @@ -35,6 +35,11 @@ public class FilesManager implements IFilesManager { public List getFilesByUploadId(String uploadId) { return databaseClient.getFilesByUploadId(uploadId); } + + @Override + public List getFilesByDocumentId(String documentId) { + return databaseClient.getFilesByDocumentId(documentId); + } @Override public IFile getFile(String id) { @@ -86,5 +91,9 @@ public String getRelativePathOfFile(IFile file) { public List getKnownUsernames() { return databaseClient.getUsernames(); } - + + @Override + public void deleteFile(String fileId) { + databaseClient.deleteFile(fileId); + } } diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageDeletionRequestReceiver.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageDeletionRequestReceiver.java new file mode 100644 index 0000000..ca7ed8a --- /dev/null +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageDeletionRequestReceiver.java @@ -0,0 +1,41 @@ +package edu.asu.diging.gilesecosystem.nepomuk.core.kafka.impl; + +import java.io.IOException; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.kafka.annotation.KafkaListener; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IDeletionRequestProcessor; +import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.impl.StorageDeletionRequest; +import edu.asu.diging.gilesecosystem.septemberutil.properties.MessageType; +import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler; + +@PropertySource("classpath:/config.properties") +public class StorageDeletionRequestReceiver { + @Autowired + private IDeletionRequestProcessor deletionRequestProcessor; + + @Autowired + private ISystemMessageHandler messageHandler; + + /** + * Kafka listener method for receiving and processing a delete storage request message. + * @param message The message containing the delete storage request. + */ + @KafkaListener(topics = "${topic_delete_storage_request}") + public void receiveDeleteMessage(String message) { + ObjectMapper mapper = new ObjectMapper(); + IStorageDeletionRequest request = null; + try { + request = mapper.readValue(message, StorageDeletionRequest.class); + } catch (IOException e) { + messageHandler.handleMessage("Could not unmarshall request.", e, MessageType.ERROR); + return; + } + deletionRequestProcessor.processRequest(request); + } +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageRequestReceiver.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageRequestReceiver.java index 0585d9e..1ed8509 100644 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageRequestReceiver.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageRequestReceiver.java @@ -8,8 +8,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; +import edu.asu.diging.gilesecosystem.nepomuk.core.exception.NepomukFileStorageException; import edu.asu.diging.gilesecosystem.nepomuk.core.service.IRequestProcessor; +import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest; import edu.asu.diging.gilesecosystem.requests.IStorageRequest; +import edu.asu.diging.gilesecosystem.requests.impl.StorageDeletionRequest; import edu.asu.diging.gilesecosystem.requests.impl.StorageRequest; import edu.asu.diging.gilesecosystem.septemberutil.properties.MessageType; import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler; diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IDeletionRequestProcessor.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IDeletionRequestProcessor.java new file mode 100644 index 0000000..9ee5fad --- /dev/null +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IDeletionRequestProcessor.java @@ -0,0 +1,11 @@ +package edu.asu.diging.gilesecosystem.nepomuk.core.service; + +import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest; + +public interface IDeletionRequestProcessor { + /** + * Processes a storage deletion request by deleting associated files and generating a completed storage deletion request. + * @param request The storage deletion request to be processed. + */ + void processRequest(IStorageDeletionRequest request); +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IFileTypeHandler.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IFileTypeHandler.java index 3075fbe..23da00f 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IFileTypeHandler.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IFileTypeHandler.java @@ -23,4 +23,11 @@ public interface IFileTypeHandler { public abstract String getRelativePathInTypeFolder(IFile file); + /** + * This method deletes a file. + * @param file + * File to be deleted. + * @throws NepomukFileStorageException + */ + public abstract void deleteFile(IFile file) throws NepomukFileStorageException; } diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IRequestProcessor.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IRequestProcessor.java index 7459b34..bffffb5 100644 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IRequestProcessor.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IRequestProcessor.java @@ -6,4 +6,4 @@ public interface IRequestProcessor { public abstract void processRequest(IStorageRequest request); -} \ No newline at end of file +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandler.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandler.java index 693823f..9c910f6 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandler.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandler.java @@ -104,5 +104,13 @@ public String getRelativePathInTypeFolder(IFile file) { protected abstract IFileStorageManager getStorageManager(); + public void deleteFile(IFile file) throws NepomukFileStorageException { + if (getStorageManager().checkIfFileExists(file.getUsername(), file.getUploadId(), file.getDocumentId(), file.getFilename())) { + getStorageManager().deleteFile(file.getUsername(), file.getUploadId(), file.getDocumentId(), file.getFilename()); + } + // files manager will be called to delete the file from the database. + // Even if the file does not exist in storage we need to remove the file's database entry as it can be an older file version. + filesManager.deleteFile(file.getId()); + } } diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/impl/DeletionRequestProcessor.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/impl/DeletionRequestProcessor.java new file mode 100644 index 0000000..2a9b46d --- /dev/null +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/impl/DeletionRequestProcessor.java @@ -0,0 +1,81 @@ +package edu.asu.diging.gilesecosystem.nepomuk.core.service.impl; + +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import edu.asu.diging.gilesecosystem.nepomuk.core.exception.NepomukFileStorageException; +import edu.asu.diging.gilesecosystem.nepomuk.core.files.IFilesManager; +import edu.asu.diging.gilesecosystem.nepomuk.core.model.IFile; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IDeletionRequestProcessor; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IFileHandlerRegistry; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IFileTypeHandler; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.properties.Properties; +import edu.asu.diging.gilesecosystem.requests.ICompletedStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.IRequestFactory; +import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.RequestStatus; +import edu.asu.diging.gilesecosystem.requests.exceptions.MessageCreationException; +import edu.asu.diging.gilesecosystem.requests.impl.CompletedStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.kafka.IRequestProducer; +import edu.asu.diging.gilesecosystem.septemberutil.properties.MessageType; +import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler; +import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager; + +@Service +public class DeletionRequestProcessor implements IDeletionRequestProcessor { + @Autowired + private IFileHandlerRegistry fileHandlerRegistry; + + @Autowired + private IPropertiesManager propertiesManager; + + @Autowired + private IRequestProducer requestProducer; + + @Autowired + private ISystemMessageHandler messageHandler; + + @Autowired + private IFilesManager filesManager; + + @Autowired + private IRequestFactory deletionRequestFactory; + + @PostConstruct + public void init() { + deletionRequestFactory.config(CompletedStorageDeletionRequest.class); + } + + @Override + public void processRequest(IStorageDeletionRequest request) { + List files = filesManager.getFilesByDocumentId(request.getDocumentId()); + RequestStatus status = RequestStatus.COMPLETE; + for (IFile file : files) { + IFileTypeHandler handler = fileHandlerRegistry.getHandler(file.getFileType()); + try { + handler.deleteFile(file); + } catch(NepomukFileStorageException ex) { + messageHandler.handleMessage("Error deleting file with id " + file.getId(), ex, MessageType.ERROR); + status = RequestStatus.FAILED; + } + } + ICompletedStorageDeletionRequest completedRequest; + try { + completedRequest = deletionRequestFactory.createRequest(request.getRequestId(), request.getUploadId()); + } catch (InstantiationException | IllegalAccessException e) { + messageHandler.handleMessage("Request could not be created.", e, MessageType.ERROR); + return; + } + completedRequest.setStatus(status); + completedRequest.setDocumentId(request.getDocumentId()); + try { + requestProducer.sendRequest(completedRequest, propertiesManager.getProperty(Properties.KAFKA_TOPIC_STORAGE_DELETE_COMPLETE_REQUEST)); + } catch (MessageCreationException e) { + messageHandler.handleMessage("Request could not be send.", e, MessageType.ERROR); + } + } +} diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/properties/Properties.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/properties/Properties.java index afbe7fc..4f69a6c 100755 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/properties/Properties.java +++ b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/properties/Properties.java @@ -21,6 +21,8 @@ public interface Properties { public final static String KAFKA_TOPIC_IMAGE_EXTRACTION_REQUEST = "topic_image_extraction_request"; public final static String KAFKA_TOPIC_IMAGE_EXTRACTION_COMPLETE_REQUEST = "topic_image_extraction_request_complete"; public final static String KAFKA_TOPIC_SYSTEM_MESSAGES = "topic_system_messages"; + public final static String KAFKA_TOPIC_STORAGE_DELETE_REQUEST = "topic_delete_storage_request"; + public final static String KAFKA_TOPIC_STORAGE_DELETE_COMPLETE_REQUEST = "topic_delete_storage_request_complete"; public final static String GILES_TMP_FOLDER = "giles_files_tmp_dir"; public final static String APPLICATION_ID = "application_id"; diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/zookeeper/IZookeeperServiceRegistry.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/zookeeper/IZookeeperServiceRegistry.java deleted file mode 100644 index d863307..0000000 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/zookeeper/IZookeeperServiceRegistry.java +++ /dev/null @@ -1,5 +0,0 @@ -package edu.asu.diging.gilesecosystem.nepomuk.core.zookeeper; - -public interface IZookeeperServiceRegistry { - -} \ No newline at end of file diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/zookeeper/impl/ZookeeperServiceRegistry.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/zookeeper/impl/ZookeeperServiceRegistry.java deleted file mode 100644 index 8af56d4..0000000 --- a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/zookeeper/impl/ZookeeperServiceRegistry.java +++ /dev/null @@ -1,62 +0,0 @@ -package edu.asu.diging.gilesecosystem.nepomuk.core.zookeeper.impl; - -import java.util.concurrent.ConcurrentHashMap; - -import javax.annotation.PostConstruct; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryNTimes; -import org.apache.zookeeper.CreateMode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import edu.asu.diging.gilesecosystem.nepomuk.core.service.properties.Properties; -import edu.asu.diging.gilesecosystem.nepomuk.core.zookeeper.IZookeeperServiceRegistry; -import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager; - -/** - * This class registers Nepomuk with Zookeeper for discovery by Giles (or - * potentially other services). - * - * @author jdamerow - * - */ -@Service -public class ZookeeperServiceRegistry implements IZookeeperServiceRegistry { - - private String znode; - private String nepomukUrl; - - private CuratorFramework curatorFramework; - private ConcurrentHashMap uriToZnodePath; - - @Autowired - private IPropertiesManager propertiesManager; - - @PostConstruct - public void init() throws Exception { - curatorFramework = CuratorFrameworkFactory - .newClient(propertiesManager.getProperty(Properties.ZOOKEEPER_HOST) - + ":" - + propertiesManager.getProperty(Properties.ZOOKEEPER_PORT) , new RetryNTimes(5, 1000)); - curatorFramework.start(); - uriToZnodePath = new ConcurrentHashMap<>(); - - znode = propertiesManager.getProperty(Properties.ZOOKEEPER_SERVICE_ROOT) + propertiesManager.getProperty(Properties.ZOOKEEPER_NEPOMUK_SERVICE_NAME); - nepomukUrl = propertiesManager.getProperty(Properties.APP_BASE_URL); - registerNepomuk(); - } - - private void registerNepomuk() throws Exception { - // forPath actually throws Exception - if (curatorFramework.checkExists().forPath(znode) == null) { - curatorFramework.create().creatingParentsIfNeeded().forPath(znode); - } - String znodePath = curatorFramework - .create() - .withMode(CreateMode.EPHEMERAL_SEQUENTIAL) - .forPath(znode+"/_", nepomukUrl.getBytes()); - uriToZnodePath.put(nepomukUrl, znodePath); - } -} diff --git a/nepomuk/src/main/resources/config.properties b/nepomuk/src/main/resources/config.properties index e925611..2360f08 100755 --- a/nepomuk/src/main/resources/config.properties +++ b/nepomuk/src/main/resources/config.properties @@ -32,11 +32,13 @@ kafka_hosts=${nepomuk.kafka.hosts} topic_storage_request=geco.requests.storage topic_storage_request_complete=geco.requests.storage.complete topic_system_messages=geco.requests.system.messages +topic_delete_storage_request=geco.requests.delete.storage +topic_delete_storage_request_complete=geco.requests.delete.storage.complete # Zookeeper zookeeper_host=${zookeeper.host} zookeeper_port=${zookeeper.port} -zookeeepr_service_nepomuk_name=nepomuk +zookeeper_service_name=nepomuk zookeeper_service_root=/services/ # connection to Giles diff --git a/nepomuk/src/test/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandlerTest.java b/nepomuk/src/test/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandlerTest.java new file mode 100644 index 0000000..edd5e6b --- /dev/null +++ b/nepomuk/src/test/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandlerTest.java @@ -0,0 +1,96 @@ +package edu.asu.diging.gilesecosystem.nepomuk.core.service.handlers; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; + +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import edu.asu.diging.gilesecosystem.nepomuk.core.exception.NepomukFileStorageException; +import edu.asu.diging.gilesecosystem.nepomuk.core.files.IFileStorageManager; +import edu.asu.diging.gilesecosystem.nepomuk.core.files.IFilesManager; +import edu.asu.diging.gilesecosystem.nepomuk.core.model.IFile; +import edu.asu.diging.gilesecosystem.nepomuk.core.model.impl.File; +import edu.asu.diging.gilesecosystem.septemberutil.properties.MessageType; +import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler; + +public class AbstractFileHandlerTest { + @Mock + private IFilesManager filesManager; + + @Mock + private ISystemMessageHandler messageHandler; + + @Mock + private IFileStorageManager fileStorageManager; + + @InjectMocks + private AbstractFileHandler abstractFileHandler; + + IFile file; + + private String FILE_ID = "fileId1"; + private String GILES_ID = "fileIdGiles1"; + private String CONTENT_TYPE = "contentType"; + private String DOCUMENT_ID = "documentId"; + private String FILENAME = "filename"; + private long SIZE = 1000; + private String UPLOAD_DATE = "date"; + private String UPLOAD_ID = "uploadId"; + + @Before + public void setUp() { + abstractFileHandler = mock(AbstractFileHandler.class); + MockitoAnnotations.initMocks(this); + Mockito.when(abstractFileHandler.getStorageManager()).thenReturn(fileStorageManager); + file = createFile(FILE_ID, GILES_ID); + } + + @Test + public void test_deleteFile_success() throws NepomukFileStorageException { + Mockito.when(abstractFileHandler.getStorageManager().checkIfFileExists("github_3123", UPLOAD_ID, DOCUMENT_ID, FILENAME)).thenReturn(true); + Mockito.doCallRealMethod().when(abstractFileHandler).deleteFile(file); + abstractFileHandler.deleteFile(file); + Mockito.verify(fileStorageManager, times(1)).deleteFile("github_3123", UPLOAD_ID, DOCUMENT_ID, FILENAME); + Mockito.verify(filesManager, times(1)).deleteFile(file.getId()); + } + + @Test + public void test_deleteFile_whenFileNotInStorage_success() throws NepomukFileStorageException { + Mockito.when(abstractFileHandler.getStorageManager().checkIfFileExists("github_3123", UPLOAD_ID, DOCUMENT_ID, FILENAME)).thenReturn(false); + Mockito.doCallRealMethod().when(abstractFileHandler).deleteFile(file); + abstractFileHandler.deleteFile(file); + Mockito.verify(fileStorageManager, times(0)).deleteFile("github_3123", UPLOAD_ID, DOCUMENT_ID, FILENAME); + Mockito.verify(filesManager, times(1)).deleteFile(file.getId()); + } + + @Test(expected=NepomukFileStorageException.class) + public void test_deleteFile_throwsNepomukFileStorageException() throws NepomukFileStorageException { + Mockito.when(abstractFileHandler.getStorageManager().checkIfFileExists("github_3123", UPLOAD_ID, DOCUMENT_ID, FILENAME)).thenReturn(true); + Mockito.doCallRealMethod().when(abstractFileHandler).deleteFile(file); + Mockito.doThrow(new NepomukFileStorageException()).when(fileStorageManager).deleteFile("github_3123", UPLOAD_ID, DOCUMENT_ID, FILENAME); + abstractFileHandler.deleteFile(file); + Mockito.verify(filesManager, times(0)).deleteFile(file.getId()); + } + + private File createFile(String fileId, String gilesId) { + File file = new File(); + file.setId(fileId); + file.setContentType(CONTENT_TYPE); + file.setDocumentId(DOCUMENT_ID); + file.setFilename(FILENAME); + file.setSize(SIZE); + file.setUploadDate(UPLOAD_DATE); + file.setUploadId(UPLOAD_ID); + file.setUsername("github_3123"); + file.setGilesFileId(gilesId); + return file; + } + +} diff --git a/nepomuk/src/test/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/impl/DeletionRequestProcessorTest.java b/nepomuk/src/test/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/impl/DeletionRequestProcessorTest.java new file mode 100644 index 0000000..7a77dcc --- /dev/null +++ b/nepomuk/src/test/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/impl/DeletionRequestProcessorTest.java @@ -0,0 +1,149 @@ +package edu.asu.diging.gilesecosystem.nepomuk.core.service.impl; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; + +import java.util.ArrayList; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import edu.asu.diging.gilesecosystem.nepomuk.core.exception.NepomukFileStorageException; +import edu.asu.diging.gilesecosystem.nepomuk.core.files.IFilesManager; +import edu.asu.diging.gilesecosystem.nepomuk.core.model.IFile; +import edu.asu.diging.gilesecosystem.nepomuk.core.model.impl.File; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IDeletionRequestProcessor; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IFileHandlerRegistry; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.IFileTypeHandler; +import edu.asu.diging.gilesecosystem.nepomuk.core.service.properties.Properties; +import edu.asu.diging.gilesecosystem.requests.ICompletedStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.IRequestFactory; +import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.exceptions.MessageCreationException; +import edu.asu.diging.gilesecosystem.requests.impl.CompletedStorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.impl.StorageDeletionRequest; +import edu.asu.diging.gilesecosystem.requests.kafka.IRequestProducer; +import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler; +import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager; + +public class DeletionRequestProcessorTest { + @Mock + private IFileHandlerRegistry fileHandlerRegistry; + + @Mock + private IPropertiesManager propertiesManager; + + @Mock + private ISystemMessageHandler messageHandler; + + @Mock + private IRequestProducer requestProducer; + + @Mock + private IRequestFactory deletionRequestFactory; + + @Mock + private IFilesManager filesManager; + + @Mock + private IFileTypeHandler fileTypeHandler; + + @InjectMocks + private IDeletionRequestProcessor deletionRequestProcessor; + + IFile file1, file2; + IStorageDeletionRequest storageDeletionRequest; + ICompletedStorageDeletionRequest completedStorageDeletionRequest; + IFileTypeHandler handler; + + private String FILE_ID_1 = "fileId1"; + private String FILE_ID_2= "fileId2"; + private String GILES_ID_1 = "fileIdGiles1"; + private String GILES_ID_2= "fileIdGiles2"; + private String CONTENT_TYPE = "contentType"; + private String DOCUMENT_ID = "documentId"; + private String FILENAME = "filename"; + private long SIZE = 1000; + private String UPLOAD_DATE = "date"; + private String UPLOAD_ID = "uploadId"; + + @Before + public void setUp() throws NepomukFileStorageException, InstantiationException, IllegalAccessException { + deletionRequestProcessor = new DeletionRequestProcessor(); + MockitoAnnotations.initMocks(this); + file1 = createFile(FILE_ID_1, GILES_ID_1); + file2 = createFile(FILE_ID_2, GILES_ID_2); + ArrayList files = new ArrayList(); + files.add(file1); + storageDeletionRequest = createStorageDeletionRequest(); + completedStorageDeletionRequest = createCompletedStorageDeletionRequest(); + handler = mock(IFileTypeHandler.class); + Mockito.when(filesManager.getFilesByDocumentId(file1.getDocumentId())).thenReturn(files); + Mockito.when(propertiesManager.getProperty(Properties.KAFKA_TOPIC_STORAGE_DELETE_COMPLETE_REQUEST)).thenReturn("topic_delete_storage_request_complete"); + Mockito.when(fileHandlerRegistry.getHandler(file1.getFileType())).thenReturn(handler); + Mockito.when(deletionRequestFactory.createRequest("REQ123", UPLOAD_ID)).thenReturn(completedStorageDeletionRequest); + } + + @Test + public void test_processRequest_allFilesDeleted_success() throws NepomukFileStorageException, MessageCreationException { + Mockito.when(filesManager.getFilesByDocumentId(file1.getDocumentId())).thenReturn(new ArrayList()); + doNothing().when(handler).deleteFile(file1); + deletionRequestProcessor.processRequest(storageDeletionRequest); + Mockito.verify(requestProducer, times(1)).sendRequest(completedStorageDeletionRequest, "topic_delete_storage_request_complete"); + } + + @Test() + public void test_processRequest_deletionFailure_throwsNepomukFileStorageException() throws MessageCreationException, NepomukFileStorageException { + Mockito.doThrow(new NepomukFileStorageException()).when(handler).deleteFile(Mockito.any()); + deletionRequestProcessor.processRequest(storageDeletionRequest); + Mockito.verify(requestProducer, times(1)).sendRequest(completedStorageDeletionRequest, "topic_delete_storage_request_complete"); + } + + @Test + public void test_processRequest_multipleFilesToDelete_success() throws MessageCreationException, NepomukFileStorageException { + doNothing().when(handler).deleteFile(file1); + doNothing().when(handler).deleteFile(file2); + ArrayList files = new ArrayList(); + files.add(file2); + files.add(file1); + Mockito.when(filesManager.getFilesByDocumentId(storageDeletionRequest.getDocumentId())).thenReturn(files); + deletionRequestProcessor.processRequest(storageDeletionRequest); + Mockito.verify(requestProducer, times(1)).sendRequest(completedStorageDeletionRequest, "topic_delete_storage_request_complete"); + } + + private File createFile(String fileId, String gilesId) { + File file = new File(); + file.setId(fileId); + file.setContentType(CONTENT_TYPE); + file.setDocumentId(DOCUMENT_ID); + file.setFilename(FILENAME); + file.setSize(SIZE); + file.setUploadDate(UPLOAD_DATE); + file.setUploadId(UPLOAD_ID); + file.setUsername("github_3123"); + file.setGilesFileId(gilesId); + return file; + } + + private StorageDeletionRequest createStorageDeletionRequest() { + StorageDeletionRequest storageDeletionRequest = new StorageDeletionRequest(); + storageDeletionRequest.setRequestId("REQ123"); + storageDeletionRequest.setUploadId(UPLOAD_ID); + storageDeletionRequest.setDocumentId(DOCUMENT_ID); + + return storageDeletionRequest; + } + + private CompletedStorageDeletionRequest createCompletedStorageDeletionRequest() { + CompletedStorageDeletionRequest completedStorageDeletionRequest = new CompletedStorageDeletionRequest(); + completedStorageDeletionRequest.setRequestId("REQ123"); + completedStorageDeletionRequest.setUploadId(UPLOAD_ID); + + return completedStorageDeletionRequest; + } +}