From 9c65d7c47871a6796a6386ef51e1a7cf14f60e26 Mon Sep 17 00:00:00 2001 From: Rui Han Date: Fri, 12 Apr 2024 18:49:20 +0800 Subject: [PATCH] Add S3StorageIT and initiate StorageController @Startup (#86) * Add S3StorageIT and initiate StorageController @Startup * Add testFileExist by setting storage.physicalFileExistenceCheck to true * Add config for gcBatchSize --- pom.xml | 23 +++++ .../storage/config/StorageServiceConfig.java | 8 ++ .../storage/controller/StorageController.java | 2 + .../storage/core/FileManagerProducer.java | 4 +- src/main/resources/application.yaml | 3 - .../service/storage/S3StorageIT.java | 94 +++++++++++++++++++ .../storage/util/LocalStackTestResource.java | 76 +++++++++++++++ 7 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/commonjava/service/storage/S3StorageIT.java create mode 100644 src/test/java/org/commonjava/service/storage/util/LocalStackTestResource.java diff --git a/pom.xml b/pom.xml index 7ea8f99..7ffa9d1 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ 3.0.0 3.8 uber-jar + 1.19.7 false @@ -58,6 +59,13 @@ pom import + + org.testcontainers + testcontainers-bom + ${testcontainers.version} + pom + import + @@ -173,6 +181,21 @@ quarkus-junit5-mockito test + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + org.testcontainers + localstack + test + diff --git a/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java b/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java index 9c8b746..b97bc14 100644 --- a/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java +++ b/src/main/java/org/commonjava/service/storage/config/StorageServiceConfig.java @@ -41,6 +41,14 @@ public interface StorageServiceConfig @WithName( "removableFilesystemPattern" ) String removableFilesystemPattern(); + @WithName( "physicalFileExistenceCheck" ) + @WithDefault("false") + boolean physicalFileExistenceCheck(); + + @WithName( "gcBatchSize" ) + @WithDefault("100") + int gcBatchSize(); + @WithName( "type" ) @WithDefault( STORAGE_NFS ) String type(); diff --git a/src/main/java/org/commonjava/service/storage/controller/StorageController.java b/src/main/java/org/commonjava/service/storage/controller/StorageController.java index 96ae951..7f7528e 100644 --- a/src/main/java/org/commonjava/service/storage/controller/StorageController.java +++ b/src/main/java/org/commonjava/service/storage/controller/StorageController.java @@ -16,6 +16,7 @@ package org.commonjava.service.storage.controller; import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.runtime.Startup; import org.apache.commons.io.IOUtils; import org.commonjava.service.storage.dto.*; import org.commonjava.storage.pathmapped.core.PathMappedFileManager; @@ -45,6 +46,7 @@ import static org.commonjava.service.storage.util.Utils.sort; import static org.commonjava.storage.pathmapped.util.PathMapUtils.ROOT_DIR; +@Startup @ApplicationScoped public class StorageController { diff --git a/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java b/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java index 3a18955..1f30c98 100644 --- a/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java +++ b/src/main/java/org/commonjava/service/storage/core/FileManagerProducer.java @@ -60,7 +60,9 @@ public PathMappedFileManager getFileManager() DefaultPathMappedStorageConfig config = new DefaultPathMappedStorageConfig( props ); config.setGcBatchSize( STORAGE_GC_BATCH_SIZE ); - + config.setPhysicalFileExistenceCheckEnabled( storageConfig.physicalFileExistenceCheck() ); + config.setGcBatchSize( storageConfig.gcBatchSize() ); + PathDB pathDB = new CassandraPathDB( config ); PhysicalStore physicalStore; String storageType = storageConfig.type(); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1100f70..f18a984 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -53,9 +53,6 @@ quarkus: max-backup-index: 5 max-file-size: 10M - devservices: - enabled: false - swagger-ui: always-include: true diff --git a/src/test/java/org/commonjava/service/storage/S3StorageIT.java b/src/test/java/org/commonjava/service/storage/S3StorageIT.java new file mode 100644 index 0000000..127cb1d --- /dev/null +++ b/src/test/java/org/commonjava/service/storage/S3StorageIT.java @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/service-parent) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.service.storage; + +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import org.commonjava.service.storage.util.LocalStackTestResource; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.util.List; + +import static io.restassured.RestAssured.given; +import static org.commonjava.service.storage.jaxrs.StorageResource.API_BASE; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@QuarkusTest +@QuarkusTestResource(LocalStackTestResource.class) +public class S3StorageIT + extends StorageIT +{ + @Inject + S3Client s3Client; + + @Test + public void testGetFile() + { + given().pathParam( "filesystem", filesystem ) + .pathParam( "path", PATH) + .when() + .get( API_BASE + "/content/{filesystem}/{path}" ) + .then() + .statusCode( 200 ); + } + + /** + * We set "storage.physicalFileExistenceCheck" to "true" in LocalStackTestResource so than it checks + * both pathDB and physical store. + */ + @Test + public void testFileExist() + { + given().pathParam( "filesystem", filesystem ) + .pathParam( "path", PATH ) + .when() + .head( API_BASE + "/content/{filesystem}/{path}" ) + .then() + .statusCode( 200 ); + } + + @Test + public void testPutFile() + { + given().pathParam( "filesystem", filesystem ) + .pathParam( "path", PATH) + .when() + .put( API_BASE + "/content/{filesystem}/{path}" ) + .then() + .statusCode( 200 ); + + verifyBucket(); + } + + private void verifyBucket() + { + ListObjectsRequest lor = ListObjectsRequest.builder().bucket("test").build(); + ListObjectsResponse response = s3Client.listObjects(lor); + List contents = response.contents(); + + assertTrue(contents.size() > 0 ); + for (S3Object et: contents) + { + //System.out.println( ">>> " + et.key()); + assertTrue(et.key().contains(PATH)); + } + } +} diff --git a/src/test/java/org/commonjava/service/storage/util/LocalStackTestResource.java b/src/test/java/org/commonjava/service/storage/util/LocalStackTestResource.java new file mode 100644 index 0000000..4a2c28e --- /dev/null +++ b/src/test/java/org/commonjava/service/storage/util/LocalStackTestResource.java @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2021 Red Hat, Inc. (https://github.com/Commonjava/service-parent) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.service.storage.util; + +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; + +import java.util.HashMap; +import java.util.Map; + +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; + +public class LocalStackTestResource implements QuarkusTestResourceLifecycleManager { + + static DockerImageName dockerImageName = DockerImageName.parse("localstack/localstack:3.2.0"); + static LocalStackContainer localStackContainer = new LocalStackContainer(dockerImageName) + .withServices(S3); + + @Override + public Map start() { + localStackContainer.start(); + prepareBucket(); + + HashMap map = new HashMap<>(); + map.put("storage.type", "s3"); + map.put("storage.physicalFileExistenceCheck", "true"); + map.put("storage.bucket.name", "test"); + map.put("quarkus.s3.endpoint-override", localStackContainer.getEndpointOverride(S3).toString()); + map.put("quarkus.s3.aws.region", localStackContainer.getRegion()); + map.put("quarkus.s3.aws.credentials.type", "static"); + map.put("quarkus.s3.aws.credentials.static-provider.access-key-id", localStackContainer.getAccessKey()); + map.put("quarkus.s3.aws.credentials.static-provider.secret-access-key", localStackContainer.getSecretKey()); + return map; + } + + private void prepareBucket() + { + S3Client s3 = S3Client + .builder() + .endpointOverride(localStackContainer.getEndpoint()) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(localStackContainer.getAccessKey(), localStackContainer.getSecretKey()) + ) + ) + .region(Region.of(localStackContainer.getRegion())) + .build(); + + s3.createBucket(CreateBucketRequest.builder().bucket("test").build()); + } + + @Override + public void stop() { + localStackContainer.stop(); + } + +} \ No newline at end of file