Add REST api getFilePathsByChecksum (#95)

Commonjava · Sep 20, 2024 · 40fcb8a · 40fcb8a
1 parent b3162d0
commit 40fcb8a
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 2 deletions.
diff --git a/pom.xml b/pom.xml
@@ -42,7 +42,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <path-mapped-storage.version>2.8</path-mapped-storage.version>
+        <path-mapped-storage.version>2.9-SNAPSHOT</path-mapped-storage.version>
         <toolchains-plugin.version>3.0.0</toolchains-plugin.version>
         <cassandra-maven-plugin.version>3.8</cassandra-maven-plugin.version>
         <quarkus.package.type>uber-jar</quarkus.package.type>

diff --git a/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java b/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java
@@ -41,6 +41,9 @@ public interface StorageServiceConfig
     @WithName( "removableFilesystemPattern" )
     String removableFilesystemPattern();
 
+    @WithName( "deduplicatePattern" )
+    String deduplicatePattern();
+
     @WithName( "physicalFileExistenceCheck" )
     @WithDefault("false")
     boolean physicalFileExistenceCheck();

diff --git a/src/main/java/org/commonjava/service/storage/controller/StorageController.java b/src/main/java/org/commonjava/service/storage/controller/StorageController.java
@@ -20,6 +20,7 @@
 import org.apache.commons.io.IOUtils;
 import org.commonjava.service.storage.dto.*;
 import org.commonjava.storage.pathmapped.core.PathMappedFileManager;
+import org.commonjava.storage.pathmapped.model.FileChecksum;
 import org.commonjava.storage.pathmapped.model.Filesystem;
 import org.commonjava.storage.pathmapped.model.PathMap;
 import org.commonjava.storage.pathmapped.spi.PathDB;
@@ -40,6 +41,7 @@
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.emptySet;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import static org.commonjava.service.storage.util.Utils.getDuration;
@@ -166,6 +168,22 @@ public FileInfoObj getFileInfo(String filesystem, String path )
         return null;
     }
 
+    /**
+     * Get paths (each includes the filesystem+path) by file's checksum.
+     * @return The paths set (because each physical file may link to multiple logical files).
+     */
+    public Set<String> getFilePathsByChecksum( String checksum )
+    {
+        final PathDB pathDB = fileManager.getPathDB();
+        FileChecksum c = pathDB.getFileChecksum( checksum );
+        if ( c != null )
+        {
+            String fileId = c.getFileId();
+            return pathDB.getPathsByFileId( fileId );
+        }
+        return emptySet();
+    }
+
     public BatchCleanupResult cleanup(Set<String> paths, Set<String> filesystems )
     {
         Set<String> success = new HashSet<>();
@@ -333,5 +351,4 @@ public FileCopyResult copy(FileCopyRequest request)
 
         return new FileCopyResult( true, completed, skipped );
     }
-
 }
diff --git a/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java b/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java
@@ -60,6 +60,7 @@ public PathMappedFileManager getFileManager()
         config.setPhysicalFileExistenceCheckEnabled( storageConfig.physicalFileExistenceCheck() );
         config.setGcBatchSize( storageConfig.gcBatchSize() );
         config.setGcIntervalInMinutes( storageConfig.gcIntervalInMinutes() );
+        config.setDeduplicatePattern( storageConfig.deduplicatePattern() );
 
         PathDB pathDB = new CassandraPathDB( config );
         PhysicalStore physicalStore;

diff --git a/src/main/java/org/commonjava/service/storage/jaxrs/StorageResource.java b/src/main/java/org/commonjava/service/storage/jaxrs/StorageResource.java
@@ -45,6 +45,7 @@
 import java.io.InputStream;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON;
@@ -214,6 +215,20 @@ public Response getFileInfo(
         return responseHelper.formatOkResponseWithJsonEntity( result );
     }
 
+    @Operation( summary = "Get paths (each includes 'filesystem + path') by file's checksum." )
+    @APIResponses( { @APIResponse( responseCode = "200",
+                    description = "The paths or 'empty set' if not found." ) } )
+    @Produces( APPLICATION_JSON )
+    @GET
+    @Path( "checksum/{checksum}" )
+    public Response getFilePathsByChecksum(
+                    final @Parameter( in = PATH, required = true ) @PathParam( "checksum" ) String checksum )
+    {
+        final Set<String> paths = controller.getFilePathsByChecksum( checksum );
+        logger.info( "File paths: {}", paths );
+        return responseHelper.formatOkResponseWithJsonEntity( paths );
+    }
+
     @Operation( summary = "Copy files." )
     @RequestBody( description = "The file copy request", name = "body", required = true,
             content = @Content( schema = @Schema( implementation = FileCopyRequest.class ) ) )

diff --git a/src/test/java/org/commonjava/service/storage/StorageResourceIT.java b/src/test/java/org/commonjava/service/storage/StorageResourceIT.java
@@ -28,6 +28,8 @@
 import org.junit.jupiter.api.Test;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -113,6 +115,44 @@ public void testGetFileInfo()
         assertNotEquals( -1, response.jsonPath().getLong( "fileLength" ) );
     }
 
+    @Test
+    public void testGetPathsByChecksum() throws Exception
+    {
+        // filesystem must match the 'deduplicatePattern'
+        final String g_filesystem = "generic-http:remote:g_filesystem";
+        final String checksum = "209094962790ca300e4e387f4b3dd130ff06ef9a9c3e08c2f96889334db4cf4c";
+
+        // upload the file to g_filesystem
+        try (InputStream in = url.openStream())
+        {
+            prepareFile( in, g_filesystem, PATH );
+        }
+
+        // test with right checksum
+        Response response = given().pathParam( "checksum", checksum )
+                                   .when()
+                                   .get( API_BASE + "/checksum/{checksum}" )
+                                   .then()
+                                   .extract()
+                                   .response();
+
+        assertEquals( 200, response.statusCode() );
+        //System.out.println(">>>" + response.jsonPath().getList( "" ));
+        assertTrue( response.jsonPath().getList( "" ).contains( g_filesystem + ":/" + PATH ) );
+
+        // test with wrong checksum
+        response = given().pathParam( "checksum", "wrong-094962790ca30" )
+                          .when()
+                          .get( API_BASE + "/checksum/{checksum}" )
+                          .then()
+                          .extract()
+                          .response();
+
+        assertEquals( 200, response.statusCode() );
+        //System.out.println(">>>" + response.getBody().asString() );
+        assertTrue( response.jsonPath().getList( "" ).isEmpty() );
+    }
+
     @Test
     public void testList()
     {

diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml
@@ -42,4 +42,5 @@ cassandra:
 storage:
     baseDir: "/tmp"
     readonly: false
+    deduplicatePattern: "(generic-http|npm).+"
     removableFilesystemPattern: ".+:(remote|group):.+"