From 40fcb8aad29ad37a816b912262c592931cd834bf Mon Sep 17 00:00:00 2001 From: Rui Han Date: Fri, 20 Sep 2024 10:37:13 +0700 Subject: [PATCH] Add REST api getFilePathsByChecksum (#95) --- pom.xml | 2 +- .../storage/config/StorageServiceConfig.java | 3 ++ .../storage/controller/StorageController.java | 19 ++++++++- .../storage/core/FileManagerProducer.java | 1 + .../storage/jaxrs/StorageResource.java | 15 +++++++ .../service/storage/StorageResourceIT.java | 40 +++++++++++++++++++ src/test/resources/application.yaml | 1 + 7 files changed, 79 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4f6e3a4..9a07852 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ UTF-8 UTF-8 - 2.8 + 2.9-SNAPSHOT 3.0.0 3.8 uber-jar diff --git a/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java b/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java index ef37a0b..6e17dd5 100644 --- a/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java +++ b/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java @@ -41,6 +41,9 @@ public interface StorageServiceConfig @WithName( "removableFilesystemPattern" ) String removableFilesystemPattern(); + @WithName( "deduplicatePattern" ) + String deduplicatePattern(); + @WithName( "physicalFileExistenceCheck" ) @WithDefault("false") boolean physicalFileExistenceCheck(); diff --git a/src/main/java/org/commonjava/service/storage/controller/StorageController.java b/src/main/java/org/commonjava/service/storage/controller/StorageController.java index 7f7528e..d51ada1 100644 --- a/src/main/java/org/commonjava/service/storage/controller/StorageController.java +++ b/src/main/java/org/commonjava/service/storage/controller/StorageController.java @@ -20,6 +20,7 @@ import org.apache.commons.io.IOUtils; import org.commonjava.service.storage.dto.*; import org.commonjava.storage.pathmapped.core.PathMappedFileManager; +import org.commonjava.storage.pathmapped.model.FileChecksum; import org.commonjava.storage.pathmapped.model.Filesystem; import org.commonjava.storage.pathmapped.model.PathMap; import org.commonjava.storage.pathmapped.spi.PathDB; @@ -40,6 +41,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.commonjava.service.storage.util.Utils.getDuration; @@ -166,6 +168,22 @@ public FileInfoObj getFileInfo(String filesystem, String path ) return null; } + /** + * Get paths (each includes the filesystem+path) by file's checksum. + * @return The paths set (because each physical file may link to multiple logical files). + */ + public Set getFilePathsByChecksum( String checksum ) + { + final PathDB pathDB = fileManager.getPathDB(); + FileChecksum c = pathDB.getFileChecksum( checksum ); + if ( c != null ) + { + String fileId = c.getFileId(); + return pathDB.getPathsByFileId( fileId ); + } + return emptySet(); + } + public BatchCleanupResult cleanup(Set paths, Set filesystems ) { Set success = new HashSet<>(); @@ -333,5 +351,4 @@ public FileCopyResult copy(FileCopyRequest request) return new FileCopyResult( true, completed, skipped ); } - } diff --git a/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java b/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java index c992daf..5c757cd 100644 --- a/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java +++ b/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java @@ -60,6 +60,7 @@ public PathMappedFileManager getFileManager() config.setPhysicalFileExistenceCheckEnabled( storageConfig.physicalFileExistenceCheck() ); config.setGcBatchSize( storageConfig.gcBatchSize() ); config.setGcIntervalInMinutes( storageConfig.gcIntervalInMinutes() ); + config.setDeduplicatePattern( storageConfig.deduplicatePattern() ); PathDB pathDB = new CassandraPathDB( config ); PhysicalStore physicalStore; diff --git a/src/main/java/org/commonjava/service/storage/jaxrs/StorageResource.java b/src/main/java/org/commonjava/service/storage/jaxrs/StorageResource.java index 646e72a..2650a36 100644 --- a/src/main/java/org/commonjava/service/storage/jaxrs/StorageResource.java +++ b/src/main/java/org/commonjava/service/storage/jaxrs/StorageResource.java @@ -45,6 +45,7 @@ import java.io.InputStream; import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; @@ -214,6 +215,20 @@ public Response getFileInfo( return responseHelper.formatOkResponseWithJsonEntity( result ); } + @Operation( summary = "Get paths (each includes 'filesystem + path') by file's checksum." ) + @APIResponses( { @APIResponse( responseCode = "200", + description = "The paths or 'empty set' if not found." ) } ) + @Produces( APPLICATION_JSON ) + @GET + @Path( "checksum/{checksum}" ) + public Response getFilePathsByChecksum( + final @Parameter( in = PATH, required = true ) @PathParam( "checksum" ) String checksum ) + { + final Set paths = controller.getFilePathsByChecksum( checksum ); + logger.info( "File paths: {}", paths ); + return responseHelper.formatOkResponseWithJsonEntity( paths ); + } + @Operation( summary = "Copy files." ) @RequestBody( description = "The file copy request", name = "body", required = true, content = @Content( schema = @Schema( implementation = FileCopyRequest.class ) ) ) diff --git a/src/test/java/org/commonjava/service/storage/StorageResourceIT.java b/src/test/java/org/commonjava/service/storage/StorageResourceIT.java index 2676b96..81b6ac1 100644 --- a/src/test/java/org/commonjava/service/storage/StorageResourceIT.java +++ b/src/test/java/org/commonjava/service/storage/StorageResourceIT.java @@ -28,6 +28,8 @@ import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -113,6 +115,44 @@ public void testGetFileInfo() assertNotEquals( -1, response.jsonPath().getLong( "fileLength" ) ); } + @Test + public void testGetPathsByChecksum() throws Exception + { + // filesystem must match the 'deduplicatePattern' + final String g_filesystem = "generic-http:remote:g_filesystem"; + final String checksum = "209094962790ca300e4e387f4b3dd130ff06ef9a9c3e08c2f96889334db4cf4c"; + + // upload the file to g_filesystem + try (InputStream in = url.openStream()) + { + prepareFile( in, g_filesystem, PATH ); + } + + // test with right checksum + Response response = given().pathParam( "checksum", checksum ) + .when() + .get( API_BASE + "/checksum/{checksum}" ) + .then() + .extract() + .response(); + + assertEquals( 200, response.statusCode() ); + //System.out.println(">>>" + response.jsonPath().getList( "" )); + assertTrue( response.jsonPath().getList( "" ).contains( g_filesystem + ":/" + PATH ) ); + + // test with wrong checksum + response = given().pathParam( "checksum", "wrong-094962790ca30" ) + .when() + .get( API_BASE + "/checksum/{checksum}" ) + .then() + .extract() + .response(); + + assertEquals( 200, response.statusCode() ); + //System.out.println(">>>" + response.getBody().asString() ); + assertTrue( response.jsonPath().getList( "" ).isEmpty() ); + } + @Test public void testList() { diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index 7fde6f9..b9141cc 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -42,4 +42,5 @@ cassandra: storage: baseDir: "/tmp" readonly: false + deduplicatePattern: "(generic-http|npm).+" removableFilesystemPattern: ".+:(remote|group):.+"