[Story/GECO-139] Adding deletion of files to nepomuk (#30)

* [GECO-139] Add new topic for listening to delete requests * [GECO-139] Initial changes to nepomuk to support deletion of files * [GECO-139] Using storage file id * [GECO-139] Changes to nepomuk message handler addition * Revert "[GECO-139] Changes to nepomuk message handler addition" This reverts commit dc19c70. * [GECO-139] Fixing bug for null in filepath for baseDirectory and adding producer part * [GECO-139] Implementing f no files are remaining for the document the request to delete the document and upload in Giles is pushed * [GECO-139] Implementing checks for storage deletion in case of old file deletion * [GECO-139] Adding tests and refactoring * [GECO-139] Adding test cases * [GECO-139] Clearing code factor issues * [GECO-139] Addressing review comments * [GECO-139] Grouping similar methods together * [GECO-139] Making changes to process documents * [GECO-139] Making testcase changes * [GECO-139] : Update to pom * [GECO-139] Addressing code factor issues * [GECO-139] Review comments * [GECO-139] Updating pom with new published version * [GECO-139] Updating pom * [GECO-139] Addressing review comments * [GECO-139] Code changes and refactoring * [GECO-139] Addressing review comments * [GECO-139] Addressing review comments * [GECO-139] Changing tests * [GECO-139] Adding service * [GECO-139] Changing request version * [GECO-139] Comment out svn to see if that makes Jenkins build Nepomuk * [GECO-139] try to use dummy git connection --------- Co-authored-by: Julia Damerow <[email protected]>
diging · Apr 2, 2024 · f4ff36b · f4ff36b
1 parent ab1f93d
commit f4ff36b
Show file tree

Hide file tree

Showing 19 changed files with 537 additions and 14 deletions.
diff --git a/nepomuk/pom.xml b/nepomuk/pom.xml
@@ -13,7 +13,7 @@
         <org.slf4j-version>1.7.5</org.slf4j-version>
         <org.springframework-version>4.3.2.RELEASE</org.springframework-version>
         <spring-security-version>4.1.3.RELEASE</spring-security-version>
-        <geco.requests.version>0.13</geco.requests.version>
+        <geco.requests.version>0.16</geco.requests.version>
         <geco.util.version>0.6</geco.util.version>
         <geco.september.util.version>0.3</geco.september.util.version>
         <geco.kafka-util.version>0.2</geco.kafka-util.version>
@@ -60,7 +60,6 @@
     </repositories>
 
     <dependencies>
-
         <!-- Giles Ecosystem -->
         <dependency>
             <groupId>edu.asu.diging</groupId>
@@ -408,6 +407,20 @@
             <artifactId>curator-framework</artifactId>
             <version>2.12.0</version>
         </dependency>
+
+        <!-- API, java.xml.bind module -->
+        <dependency>
+            <groupId>jakarta.xml.bind</groupId>
+            <artifactId>jakarta.xml.bind-api</artifactId>
+            <version>2.3.2</version>
+        </dependency>
+
+        <!-- Runtime, com.sun.xml.bind module -->
+        <dependency>
+            <groupId>org.glassfish.jaxb</groupId>
+            <artifactId>jaxb-runtime</artifactId>
+            <version>2.3.2</version>
+        </dependency>
 
         <dependency>
             <groupId>javax.xml.bind</groupId>
@@ -434,7 +447,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-war-plugin</artifactId>
-                <version>2.6</version>
+                <version>3.3.2</version>
                 <configuration>
                     <webResources>
                         <resource>
@@ -467,8 +480,8 @@
             <plugin>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>11</source>
+                    <target>11</target>
                 </configuration>
             </plugin>
             <plugin>
@@ -536,7 +549,7 @@
     </build>
 
     <scm>
-        <connection>scm:svn:http://none</connection>
-        <url>scm:svn:https://none</url>
+        <connection>scm:git:http://none</connection>
+        <url>scm:git:https://none</url>
     </scm>
 </project>
diff --git a/...k/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/config/NepomukKafkaConfig.java b/...k/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/config/NepomukKafkaConfig.java
@@ -15,6 +15,7 @@
 import org.springframework.kafka.core.ConsumerFactory;
 import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
 import edu.asu.diging.gilesecosystem.nepomuk.core.service.properties.Properties;
+import edu.asu.diging.gilesecosystem.nepomuk.core.kafka.impl.StorageDeletionRequestReceiver;
 import edu.asu.diging.gilesecosystem.nepomuk.core.kafka.impl.StorageRequestReceiver;
 import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;
 
@@ -65,6 +66,11 @@ public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() {
     public StorageRequestReceiver receiver() {
         return new StorageRequestReceiver();
     }
+
+    @Bean
+    public StorageDeletionRequestReceiver deletionReceiver() {
+        return new StorageDeletionRequestReceiver();
+    }
 
     @Override
     public String getHosts() {

diff --git a/...k/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFileStorageManager.java b/...k/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFileStorageManager.java
@@ -35,5 +35,31 @@ public abstract String getAndCreateStoragePath(String username,
 
     public abstract String getFileFolderPathInBaseFolder(String username, String uploadId,
             String fileId);
+
+    /**
+     * Method to delete a file from storage
+     * 
+     * @param username
+     *            username of user who uploaded an image
+     * @param uploadId
+     *            id of upload a file was part of
+     * @param documentId
+     *            id of document
+     * @param fileId
+     *            id of file
+     * @param deleteEmptyFolders
+     *            if empty folders need to be deleted 
+     * @throws NepomukFileStorageException 
+     */
+    public abstract void deleteFile(String username, String uploadId, String documentId, String fileId) throws NepomukFileStorageException;
 
-}
+    /**
+     * Checks if a file exists at the specified location.
+     * @param username   The username associated with the file.
+     * @param uploadId   The unique identifier for the upload.
+     * @param documentId The identifier for the document.
+     * @param fileName   The name of the file to check.
+     * @return {@code true} if the file exists at the specified location, {@code false} otherwise.
+     */
+    public abstract boolean checkIfFileExists(String username, String uploadId, String documentId, String fileName);
+}
diff --git a/.../src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesDatabaseClient.java b/.../src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesDatabaseClient.java
@@ -29,4 +29,12 @@ public abstract IFile getFile(String uploadId, String documentId, String filenam
 
     List<String> getUsernames();
 
-}
+    /**
+     * Delete a file given the file ID.
+     * @param fileId 
+     *         ID of the file to be deleted
+     */
+    public abstract void deleteFile(String fileId);
+
+    public abstract List<IFile> getFilesByDocumentId(String documentId);
+}
diff --git a/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesManager.java b/nepomuk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/IFilesManager.java
@@ -35,4 +35,14 @@ public abstract IFile getFile(String uploadId, String documentId, String filenam
             throws NoUniqueResultException;
 
     List<String> getKnownUsernames();
-}
+
+    /**
+     * Delete a file given the file ID.
+     * @param fileId 
+     *         ID of the file to be deleted
+     */
+
+    public abstract void deleteFile(String fileId);
+
+    public abstract List<IFile> getFilesByDocumentId(String documentId);
+}
diff --git a/...c/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FileStorageManager.java b/...c/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FileStorageManager.java
@@ -5,6 +5,9 @@
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 import org.springframework.stereotype.Service;
 
@@ -49,8 +52,7 @@ public void saveFile(String username, String uploadId, String documentId,
     @Override
     public String getAndCreateStoragePath(String username, String uploadId,
             String documentId) {
-        String path = baseDirectory + File.separator 
-                + getFileFolderPathInBaseFolder(username, uploadId, documentId);
+        String path = getStoragePath(username, uploadId, documentId);
         createDirectory(path);
         return path;
     }
@@ -119,5 +121,42 @@ private boolean createDirectory(String dirPath) {
     public void setFileTypeFolder(String fileTypeFolder) {
         this.fileTypeFolder = fileTypeFolder;
     }
+
+    private String getStoragePath(String username, String uploadId,
+            String documentId) {
+        return baseDirectory + File.separator 
+                + getFileFolderPathInBaseFolder(username, uploadId, documentId);
+    }
+
+    public void deleteFile(String username, String uploadId, String documentId, String fileName) throws NepomukFileStorageException {
+        String originalFilePath = getStoragePath(username, uploadId, documentId);
+        Path path = Paths.get(originalFilePath + File.separator + fileName);
+        try {
+            Files.delete(path);
+        } catch (IOException ex) {
+            throw new NepomukFileStorageException("Could not delete file.", ex);
+        }
+        String documentFolder = getStoragePath(username, uploadId, documentId);
+        File docFolder = new File(documentFolder);
+        if (docFolder.isDirectory() && docFolder.list().length == 0) {
+            // If another process adds a file to the folder just before deletion, we do not want to delete the folder.
+            // The goal is to ensure that the folder remains intact even if new files are added during the deletion process.
+            // Using the File.delete() instead of Files.delete(path) as an exception is not required to be thrown in this scenario.
+            boolean deleted = docFolder.delete();
+            if (deleted) {
+                Path documentFolderPath = Paths.get(documentFolder);
+                Path uploadFolderDirectory = documentFolderPath.getParent();
+                File uploadFolder = new File(uploadFolderDirectory.toString());
+                if (uploadFolder.isDirectory() && uploadFolder.list().length == 0) {
+                    uploadFolder.delete();
+                }
+            }
+        }
+    }
 
+    @Override
+    public boolean checkIfFileExists(String username, String uploadId, String documentId, String fileName) {
+        Path path = Paths.get(getStoragePath(username, uploadId, documentId) + File.separator + fileName);
+        return Files.exists(path);
+    }
 }
diff --git a/.../main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesDatabaseClient.java b/.../main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesDatabaseClient.java
@@ -56,6 +56,13 @@ public List<IFile> getFilesByUploadId(String uploadId) {
         return results;
     }
 
+    @Override
+    public List<IFile> getFilesByDocumentId(String documentId) {
+        List<IFile> results = new ArrayList<IFile>();
+        searchByProperty("documentId", documentId, File.class).forEach(f -> results.add((IFile)f));
+        return results;
+    }
+
     @Override
     public List<IFile> getFilesByUsername(String username) {
         List<IFile> results = new ArrayList<IFile>();
@@ -120,4 +127,9 @@ public List<String> getUsernames() {
         TypedQuery<String> docs = em.createQuery(query, String.class);
         return docs.getResultList();       
     }
+
+    @Override
+    public void deleteFile(String fileId) {
+        em.remove(getById(fileId));
+    }
 }
diff --git a/...muk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesManager.java b/...muk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/files/impl/FilesManager.java
@@ -35,6 +35,11 @@ public class FilesManager implements IFilesManager {
     public List<IFile> getFilesByUploadId(String uploadId) {
         return databaseClient.getFilesByUploadId(uploadId);
     }
+
+    @Override
+    public List<IFile> getFilesByDocumentId(String documentId) {
+        return databaseClient.getFilesByDocumentId(documentId);
+    }
 
     @Override
     public IFile getFile(String id) {
@@ -86,5 +91,9 @@ public String getRelativePathOfFile(IFile file) {
     public List<String> getKnownUsernames() {
         return databaseClient.getUsernames();
     }
-
+
+    @Override
+    public void deleteFile(String fileId) {
+        databaseClient.deleteFile(fileId);
+    }
 }
diff --git a/...edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageDeletionRequestReceiver.java b/...edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageDeletionRequestReceiver.java
@@ -0,0 +1,41 @@
+package edu.asu.diging.gilesecosystem.nepomuk.core.kafka.impl;
+
+import java.io.IOException;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.kafka.annotation.KafkaListener;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import edu.asu.diging.gilesecosystem.nepomuk.core.service.IDeletionRequestProcessor;
+import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest;
+import edu.asu.diging.gilesecosystem.requests.impl.StorageDeletionRequest;
+import edu.asu.diging.gilesecosystem.septemberutil.properties.MessageType;
+import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler;
+
+@PropertySource("classpath:/config.properties")
+public class StorageDeletionRequestReceiver {
+    @Autowired
+    private IDeletionRequestProcessor deletionRequestProcessor;
+
+    @Autowired
+    private ISystemMessageHandler messageHandler;
+
+    /**
+     * Kafka listener method for receiving and processing a delete storage request message.
+     * @param message The message containing the delete storage request.
+     */
+    @KafkaListener(topics = "${topic_delete_storage_request}")
+    public void receiveDeleteMessage(String message) {
+        ObjectMapper mapper = new ObjectMapper();
+        IStorageDeletionRequest request = null;
+        try {
+            request = mapper.readValue(message, StorageDeletionRequest.class);
+        } catch (IOException e) {
+            messageHandler.handleMessage("Could not unmarshall request.", e, MessageType.ERROR);
+            return;
+        }
+        deletionRequestProcessor.processRequest(request);
+    }
+}
diff --git a/...in/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageRequestReceiver.java b/...in/java/edu/asu/diging/gilesecosystem/nepomuk/core/kafka/impl/StorageRequestReceiver.java
@@ -8,8 +8,11 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import edu.asu.diging.gilesecosystem.nepomuk.core.exception.NepomukFileStorageException;
 import edu.asu.diging.gilesecosystem.nepomuk.core.service.IRequestProcessor;
+import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest;
 import edu.asu.diging.gilesecosystem.requests.IStorageRequest;
+import edu.asu.diging.gilesecosystem.requests.impl.StorageDeletionRequest;
 import edu.asu.diging.gilesecosystem.requests.impl.StorageRequest;
 import edu.asu.diging.gilesecosystem.septemberutil.properties.MessageType;
 import edu.asu.diging.gilesecosystem.septemberutil.service.ISystemMessageHandler;

diff --git a/...in/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IDeletionRequestProcessor.java b/...in/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IDeletionRequestProcessor.java
@@ -0,0 +1,11 @@
+package edu.asu.diging.gilesecosystem.nepomuk.core.service;
+
+import edu.asu.diging.gilesecosystem.requests.IStorageDeletionRequest;
+
+public interface IDeletionRequestProcessor {
+    /**
+     * Processes a storage deletion request by deleting associated files and generating a completed storage deletion request.
+     * @param request The storage deletion request to be processed.
+     */
+    void processRequest(IStorageDeletionRequest request);
+}
diff --git a/...uk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IFileTypeHandler.java b/...uk/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IFileTypeHandler.java
@@ -23,4 +23,11 @@ public interface IFileTypeHandler {
 
     public abstract String getRelativePathInTypeFolder(IFile file);
 
+    /**
+     * This method deletes a file.
+     * @param file 
+     *        File to be deleted.
+     * @throws NepomukFileStorageException 
+     */
+    public abstract void deleteFile(IFile file) throws NepomukFileStorageException; 
 }
diff --git a/...k/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IRequestProcessor.java b/...k/src/main/java/edu/asu/diging/gilesecosystem/nepomuk/core/service/IRequestProcessor.java
@@ -6,4 +6,4 @@ public interface IRequestProcessor {
 
     public abstract void processRequest(IStorageRequest request);
 
-}
+}
diff --git a/...java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandler.java b/...java/edu/asu/diging/gilesecosystem/nepomuk/core/service/handlers/AbstractFileHandler.java
@@ -104,5 +104,13 @@ public String getRelativePathInTypeFolder(IFile file) {
 
     protected abstract IFileStorageManager getStorageManager();
 
+    public void deleteFile(IFile file) throws NepomukFileStorageException {
+        if (getStorageManager().checkIfFileExists(file.getUsername(), file.getUploadId(), file.getDocumentId(), file.getFilename())) {
+            getStorageManager().deleteFile(file.getUsername(), file.getUploadId(), file.getDocumentId(), file.getFilename());
+        }
+        // files manager will be called to delete the file from the database. 
+        // Even if the file does not exist in storage we need to remove the file's database entry as it can be an older file version.
+        filesManager.deleteFile(file.getId());
+    }
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,4 +6,4 @@ public interface IRequestProcessor {

		public abstract void processRequest(IStorageRequest request);

		}
		}