From 8eef2727cc694c79bfb1a1a94f6fc7221ee88fc0 Mon Sep 17 00:00:00 2001 From: ruhan Date: Wed, 17 Apr 2024 17:54:25 +0800 Subject: [PATCH] Add getAllRemoteRepositoryHosts api for setting up egress network policy --- .../controller/QueryController.java | 25 +++++++- .../jaxrs/RepositoryQueryResources.java | 14 +++++ .../service/repository/util/EgressUtils.java | 63 +++++++++++++++++++ src/main/resources/egress-template.yaml | 16 +++++ .../jaxrs/RepositoryQueryResourcesTest.java | 21 +++++++ 5 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/commonjava/indy/service/repository/util/EgressUtils.java create mode 100644 src/main/resources/egress-template.yaml diff --git a/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java b/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java index 90b7999..fbb5ab6 100644 --- a/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java +++ b/src/main/java/org/commonjava/indy/service/repository/controller/QueryController.java @@ -50,7 +50,8 @@ import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static org.commonjava.indy.service.repository.model.StoreKey.fromString; import static org.commonjava.indy.service.repository.model.pkg.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; -import static org.commonjava.indy.service.repository.model.pkg.PackageTypeConstants.isValidPackageType; +import static org.commonjava.indy.service.repository.model.pkg.PackageTypeConstants.*; +import static org.commonjava.indy.service.repository.util.EgressUtils.*; @ApplicationScoped public class QueryController @@ -204,6 +205,28 @@ public List getAllRemoteRepositories( final String packageType "Failed to get all remote repos for package type {}", packageType ); } + /** + * Get all remote hostnames in known package types [maven, npm, generic-http]. + * @param format string or egress, default is comma separated string + */ + public String getAllRemoteRepositoryHosts( final String format ) + throws IndyDataException + { + final Set ret = new HashSet<>(); + storeManager.query().getAllRemoteRepositories(PKG_TYPE_MAVEN).forEach( r -> ret.add( r.getHost() ) ); + storeManager.query().getAllRemoteRepositories(PKG_TYPE_NPM).forEach( r -> ret.add( r.getHost() ) ); + storeManager.query().getAllRemoteRepositories(PKG_TYPE_GENERIC_HTTP).forEach( r -> ret.add( r.getHost() ) ); + + if ( EGRESS.equals(format) ) + { + return generateEgressString( ret ); + } + else + { + return generateCommaSeparatedString( ret ); + } + } + public List getAllHostedRepositories( final String packageType, final String enabled ) throws IndyWorkflowException { diff --git a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java index 2a94284..d5721a1 100644 --- a/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java +++ b/src/main/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResources.java @@ -50,6 +50,7 @@ import java.util.Map; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static jakarta.ws.rs.core.Response.ok; @@ -118,6 +119,19 @@ public Response getAllRemoteRepositories( return generateStoreListingResponse( () -> queryController.getAllRemoteRepositories( packageType, enabled ) ); } + @Operation( description = "Retrieve all remote repository hostname for setting up egress network policy" ) + @GET + @Path( "/remotes/hosts" ) + @Produces( TEXT_PLAIN ) + public Response getAllRemoteRepositoryHosts( + @Parameter( description = "return format in (string|egress), default is 'string'" ) + @QueryParam( "format" ) final String format) + throws Exception + { + return responseHelper.formatOkResponseWithEntity( + queryController.getAllRemoteRepositoryHosts(format), TEXT_PLAIN, null); + } + @Operation( description = "Retrieve all hosted repository definitions by specified package type" ) @APIResponse( responseCode = "200", content = @Content( schema = @Schema( implementation = StoreListingDTO.class ) ), diff --git a/src/main/java/org/commonjava/indy/service/repository/util/EgressUtils.java b/src/main/java/org/commonjava/indy/service/repository/util/EgressUtils.java new file mode 100644 index 0000000..dc96bdd --- /dev/null +++ b/src/main/java/org/commonjava/indy/service/repository/util/EgressUtils.java @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2022-2023 Red Hat, Inc. (https://github.com/Commonjava/indy-repository-service) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.commonjava.indy.service.repository.util; + +import org.apache.commons.io.IOUtils; +import org.commonjava.indy.service.repository.exception.IndyDataException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Set; + +import static java.lang.System.lineSeparator; + +public class EgressUtils +{ + public static final String EGRESS = "egress"; + + public static String generateEgressString(Set ret) + throws IndyDataException + { + final String template; + try (InputStream in = Thread.currentThread().getContextClassLoader() + .getResourceAsStream( "egress-template.yaml" )) + { + template = IOUtils.toString(in, Charset.defaultCharset()); + } + catch (IOException e) + { + throw new IndyDataException("Read egress template failed", e); + } + + final StringBuilder customerRules = new StringBuilder(); + ret.stream().sorted().forEach( s -> { + customerRules.append(" - to:" + lineSeparator()); + customerRules.append(" dnsName: " + s + lineSeparator()); + customerRules.append(" type: Allow" + lineSeparator()); + }); + + return template.replace("$customer_rules", customerRules ); + } + + public static String generateCommaSeparatedString(Set ret) + { + final StringBuilder sb = new StringBuilder(); + ret.stream().sorted().forEach( s -> sb.append(s).append(",")); + return sb.toString(); + } + +} diff --git a/src/main/resources/egress-template.yaml b/src/main/resources/egress-template.yaml new file mode 100644 index 0000000..6ee7d03 --- /dev/null +++ b/src/main/resources/egress-template.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: EgressNetworkPolicy +metadata: + name: itpaas-egress-firewall +spec: + egress: + - to: + cidrSelector: 172.0.0.0/8 + type: Allow + - to: + cidrSelector: 10.0.0.0/9 + type: Allow +$customer_rules + - to: + cidrSelector: 0.0.0.0/0 + type: Deny \ No newline at end of file diff --git a/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java b/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java index f00e607..7476dff 100644 --- a/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java +++ b/src/test/java/org/commonjava/indy/service/repository/jaxrs/RepositoryQueryResourcesTest.java @@ -17,14 +17,21 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import io.restassured.response.Response; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; + import static io.restassured.RestAssured.given; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; import static jakarta.ws.rs.core.Response.Status.OK; import static org.commonjava.indy.service.repository.util.PathUtils.normalize; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.greaterThan; +import static org.junit.jupiter.api.Assertions.assertTrue; @QuarkusTest @TestProfile( MockTestProfile.class ) @@ -44,6 +51,20 @@ public void testGetAll() .body( "items.size()", greaterThan( 1 ) ); } + @Test + public void testGetAllRemoteHosts() throws IOException + { + Response response = given().when() + .get(normalize(BASE_QUERY_PATH, "remotes/hosts?format=egress")); + final String ret; + try (InputStream in = response.getBody().asInputStream()) + { + ret = IOUtils.toString( in, Charset.defaultCharset() ); + } + //System.out.println(">>>\n" + ret); + assertTrue( ret.contains("dnsName: repo.maven.apache.org") ); + } + @Test public void testGetAllByDefaultPackageTypes() {