diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 149cfe427..27fe648c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ env: GITHUB_ACTIONS_DEPLOY_TIMEOUT: 60 jobs: - veracode: + veracode_sca: name: "Veracode - Source Clear Scan (SCA)" runs-on: ubuntu-latest if: > @@ -52,6 +52,62 @@ jobs: with: srcclr-api-token: ${{ secrets.SRCCLR_API_TOKEN }} + veracode_sast: + name: "Pipeline SAST Scan" + runs-on: ubuntu-latest + if: > + (github.ref_name == 'master' || startsWith(github.ref_name, 'SP/') || startsWith(github.ref_name, 'HF/') || github.event_name == 'pull_request') && + github.actor != 'dependabot[bot]' && + !contains(github.event.head_commit.message, '[skip tests]') + steps: + - uses: actions/checkout@v3 + - uses: Alfresco/alfresco-build-tools/.github/actions/get-build-info@v1.34.0 + - uses: Alfresco/alfresco-build-tools/.github/actions/setup-java-build@v1.34.0 + - name: "Login to Docker Hub" + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: "Login to Quay.io" + uses: docker/login-action@v2.1.0 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + - uses: Alfresco/alfresco-build-tools/.github/actions/github-download-file@v5.6.0 + with: + token: ${{ secrets.BOT_GITHUB_TOKEN }} + repository: "Alfresco/veracode-baseline-archive" + file-path: "alfresco-transform-core/alfresco-transform-core-baseline.json" + target: "baseline.json" + - name: "Build" + run: mvn -B -U install -DskipTests + - name: "Create zip" + run: zip -r to-scan.zip engines/aio/target/alfresco-transform-core-aio-*.jar engines/base/target/alfresco-base-t-engine-*.jar model/target/alfresco-transform-model-*.jar + - name: "Run SAST Scan" + uses: veracode/Veracode-pipeline-scan-action@v1.0.10 + with: + vid: ${{ secrets.VERACODE_API_ID }} + vkey: ${{ secrets.VERACODE_API_KEY }} + file: "to-scan.zip" + fail_build: true + project_name: alfresco-transform-core + issue_details: true + veracode_policy_name: Alfresco Default + summary_output: true + summary_output_file: results.json + summary_display: true + baseline_file: baseline.json + - name: Upload scan result + if: success() || failure() + run: zip readable_output.zip results.json + - name: Upload Artifact + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: Veracode Pipeline-Scan Results (Human Readable) + path: readable_output.zip + build_and_test: name: "Core & Base Snapshot deployment" runs-on: ubuntu-latest @@ -146,7 +202,7 @@ jobs: release: name: "Release" runs-on: ubuntu-latest - needs: [veracode, build_and_test, all_tests_matrix] + needs: [veracode_sca, build_and_test, all_tests_matrix] if: > !(failure() || cancelled()) && contains(github.event.head_commit.message, '[release]') && diff --git a/deprecated/alfresco-transformer-base/pom.xml b/deprecated/alfresco-transformer-base/pom.xml index 13697ea84..dbe974aa3 100644 --- a/deprecated/alfresco-transformer-base/pom.xml +++ b/deprecated/alfresco-transformer-base/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml diff --git a/engines/aio/Dockerfile b/engines/aio/Dockerfile index bbb3e27a3..df305972d 100644 --- a/engines/aio/Dockerfile +++ b/engines/aio/Dockerfile @@ -17,7 +17,7 @@ ENV IMAGEMAGICK_DEP_RPM_URL=https://dl.fedoraproject.org/pub/epel/epel-release-l ARG LIBREOFFICE_VERSION=7.2.5 ENV LIBREOFFICE_RPM_URL=https://nexus.alfresco.com/nexus/service/local/repositories/thirdparty/content/org/libreoffice/libreoffice-dist/${LIBREOFFICE_VERSION}/libreoffice-dist-${LIBREOFFICE_VERSION}-linux.gz -ENV LIBREOFFICE_ARM64_RPM_URL=https://dl.rockylinux.org/pub/rocky/8/Devel/aarch64/os/Packages/l/libreoffice-6.4.7.2-13.el8.aarch64.rpm +ENV LIBREOFFICE_ARM64_RPM_URL=https://dl.rockylinux.org/pub/rocky/8/Devel/aarch64/os/Packages/l/libreoffice-6.4.7.2-15.el8.aarch64.rpm ARG PDF_RENDERER_VERSION=1.2 ENV ALFRESCO_PDF_RENDERER_LIB_RPM_URL=https://nexus.alfresco.com/nexus/service/local/repositories/releases/content/org/alfresco/alfresco-pdf-renderer/${PDF_RENDERER_VERSION}/alfresco-pdf-renderer-${PDF_RENDERER_VERSION}-linux.tgz diff --git a/engines/aio/pom.xml b/engines/aio/pom.xml index 4b0efd48f..9859cea60 100644 --- a/engines/aio/pom.xml +++ b/engines/aio/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -132,6 +132,11 @@ com.google.guava guava + + org.testcontainers + testcontainers + test + @@ -140,7 +145,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.6.0 + 3.6.1 ${project.groupId} alfresco-transform-imagemagick,alfresco-transform-libreoffice,alfresco-transform-misc,alfresco-transform-pdf-renderer,alfresco-transform-tika diff --git a/engines/aio/src/test/java/org/alfresco/transform/aio/AIOLivenessReadinessProbeIT.java b/engines/aio/src/test/java/org/alfresco/transform/aio/AIOLivenessReadinessProbeIT.java new file mode 100644 index 000000000..5f18b3081 --- /dev/null +++ b/engines/aio/src/test/java/org/alfresco/transform/aio/AIOLivenessReadinessProbeIT.java @@ -0,0 +1,37 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2023 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.aio; + +import org.alfresco.transform.base.LivenessReadinessProbeTest; + +public class AIOLivenessReadinessProbeIT extends LivenessReadinessProbeTest { + @Override + protected LivenessReadinessProbeTest.ImagesForTests getImageForTest() { + return new ImagesForTests("alfresco-transform-core-aio", "text/plain", "text/plain", "original.txt"); + } +} diff --git a/engines/base/pom.xml b/engines/base/pom.xml index 352a9005b..131f15a57 100644 --- a/engines/base/pom.xml +++ b/engines/base/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -35,6 +35,16 @@ org.springframework.boot spring-boot-starter-webflux + + + org.springframework.boot + spring-boot-starter-reactor-netty + + + + + org.eclipse.jetty + jetty-reactive-httpclient org.springframework.retry @@ -97,6 +107,11 @@ 4.2.0 test + + org.testcontainers + testcontainers + test + diff --git a/engines/base/src/main/java/org/alfresco/transform/base/config/MTLSConfig.java b/engines/base/src/main/java/org/alfresco/transform/base/config/MTLSConfig.java index 4b6a12e3e..dd305d106 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/config/MTLSConfig.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/config/MTLSConfig.java @@ -26,8 +26,6 @@ */ package org.alfresco.transform.base.config; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; import org.alfresco.transform.base.WebClientBuilderAdjuster; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; @@ -41,21 +39,19 @@ import org.apache.hc.core5.http.config.RegistryBuilder; import org.apache.hc.core5.http.ssl.TLS; import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.transport.HttpClientTransportDynamic; +import org.eclipse.jetty.io.ClientConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.Resource; import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.http.client.reactive.JettyClientHttpConnector; import org.springframework.web.client.RestTemplate; -import reactor.netty.http.client.HttpClient; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLEngine; -import javax.net.ssl.SSLException; -import javax.net.ssl.SSLParameters; -import javax.net.ssl.TrustManagerFactory; import java.io.IOException; import java.io.InputStream; import java.security.KeyManagementException; @@ -90,35 +86,33 @@ public class MTLSConfig { private boolean hostNameVerificationDisabled; @Bean - public WebClientBuilderAdjuster webClientBuilderAdjuster(SslContextBuilder nettySslContextBuilder) + public WebClientBuilderAdjuster webClientBuilderAdjuster(SslContextFactory.Client sslContextFactory) { return builder -> { if(isTlsOrMtlsConfigured()) { - HttpClient httpClientWithSslContext = null; - try { - httpClientWithSslContext = createHttpClientWithSslContext(nettySslContextBuilder); - } catch (SSLException e) { - throw new RuntimeException(e); - } - builder.clientConnector(new ReactorClientHttpConnector(httpClientWithSslContext)); + ClientConnector clientConnector = new ClientConnector(); + clientConnector.setSslContextFactory(sslContextFactory); + HttpClient httpClientWithSslContext = new HttpClient(new HttpClientTransportDynamic(clientConnector)); + + builder.clientConnector(new JettyClientHttpConnector(httpClientWithSslContext)); } }; } @Bean - public RestTemplate restTemplate(SSLContextBuilder apacheSSLContextBuilder) throws IOException, CertificateException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException, UnrecoverableKeyException + public RestTemplate restTemplate(SSLContextBuilder sslContextBuilder) throws NoSuchAlgorithmException, KeyManagementException { if(isTlsOrMtlsConfigured()) { - return createRestTemplateWithSslContext(apacheSSLContextBuilder); + return createRestTemplateWithSslContext(sslContextBuilder); } else { return new RestTemplate(); } } @Bean - public SSLContextBuilder apacheSSLContextBuilder() throws CertificateException, KeyStoreException, IOException, NoSuchAlgorithmException, UnrecoverableKeyException { + public SSLContextBuilder sslContextBuilder() throws CertificateException, KeyStoreException, IOException, NoSuchAlgorithmException, UnrecoverableKeyException { SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); if(isKeystoreConfigured()) { @@ -136,21 +130,15 @@ public SSLContextBuilder apacheSSLContextBuilder() throws CertificateException, } @Bean - public SslContextBuilder nettySslContextBuilder() throws UnrecoverableKeyException, CertificateException, KeyStoreException, IOException, NoSuchAlgorithmException { - SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); - if(isKeystoreConfigured()) - { - KeyManagerFactory keyManagerFactory = initKeyManagerFactory(); - sslContextBuilder.keyManager(keyManagerFactory); - } - - if(isTruststoreConfigured()) - { - TrustManagerFactory trustManagerFactory = initTrustManagerFactory(); - sslContextBuilder.trustManager(trustManagerFactory); - } + public SslContextFactory.Client sslContextFactory(SSLContextBuilder sslContextBuilder) throws NoSuchAlgorithmException, KeyManagementException + { + SslContextFactory.Client sslContextFactory = new SslContextFactory.Client(); + sslContextFactory.setSslContext(sslContextBuilder.build()); + sslContextFactory.setTrustAll(false); + sslContextFactory.setEndpointIdentificationAlgorithm(hostNameVerificationDisabled ? "" : "HTTPS"); + sslContextFactory.setIncludeProtocols(TLS.V_1_2.getId(), TLS.V_1_3.getId()); - return sslContextBuilder; + return sslContextFactory; } private boolean isTlsOrMtlsConfigured() @@ -168,21 +156,6 @@ private boolean isKeystoreConfigured() return keyStoreResource != null; } - private HttpClient createHttpClientWithSslContext(SslContextBuilder sslContextBuilder) throws SSLException { - SslContext sslContext = sslContextBuilder.build(); - return HttpClient.create().secure(p -> p.sslContext(sslContext).handlerConfigurator(handler -> { - SSLEngine sslEngine = handler.engine(); - SSLParameters sslParameters = sslEngine.getSSLParameters(); - if(hostNameVerificationDisabled) - { - sslParameters.setEndpointIdentificationAlgorithm(""); - } else { - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - } - sslEngine.setSSLParameters(sslParameters); - })); - } - private RestTemplate createRestTemplateWithSslContext(SSLContextBuilder sslContextBuilder) throws NoSuchAlgorithmException, KeyManagementException { final SSLConnectionSocketFactoryBuilder sslConnectionSocketFactoryBuilder = SSLConnectionSocketFactoryBuilder.create() @@ -215,20 +188,4 @@ private KeyStore getKeyStore(String keyStoreType, Resource keyStoreResource, cha } return keyStore; } - - private TrustManagerFactory initTrustManagerFactory() throws NoSuchAlgorithmException, KeyStoreException, IOException, CertificateException - { - KeyStore trustStore = getKeyStore(trustStoreType, trustStoreResource, trustStorePassword); - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(trustStore); - return trustManagerFactory; - } - - private KeyManagerFactory initKeyManagerFactory() throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException, UnrecoverableKeyException - { - KeyStore clientKeyStore = getKeyStore(keyStoreType, keyStoreResource, keyStorePassword); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(clientKeyStore, keyStorePassword); - return keyManagerFactory; - } } diff --git a/engines/base/src/main/java/org/alfresco/transform/base/probes/ProbeTransform.java b/engines/base/src/main/java/org/alfresco/transform/base/probes/ProbeTransform.java index 254f4ba2c..28ca153a3 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/probes/ProbeTransform.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/probes/ProbeTransform.java @@ -169,6 +169,9 @@ private long getPositiveLongEnv(String name, long defaultValue) // We don't want to be doing test transforms every few seconds, but do want frequent live probes. public String doTransformOrNothing(boolean isLiveProbe, TransformHandler transformHandler) { + // If we already decided to die then don't bother with any other checks. + checkMaxTransformTimeAndCount(isLiveProbe); + // If not initialised OR it is a live probe and we are scheduled to to do a test transform. probeCount++; // TODO: update/fix/refactor liveness probes as part of ATS-138 @@ -182,7 +185,8 @@ public String doTransformOrNothing(boolean isLiveProbe, TransformHandler transfo || !initialised.get() ? doTransform(isLiveProbe, transformHandler) : doNothing(isLiveProbe); - + + // Check if the test transformation was too slow. checkMaxTransformTimeAndCount(isLiveProbe); return result; } diff --git a/engines/base/src/test/java/org/alfresco/transform/base/LivenessReadinessProbeTest.java b/engines/base/src/test/java/org/alfresco/transform/base/LivenessReadinessProbeTest.java new file mode 100644 index 000000000..b1bde0d1e --- /dev/null +++ b/engines/base/src/test/java/org/alfresco/transform/base/LivenessReadinessProbeTest.java @@ -0,0 +1,122 @@ +package org.alfresco.transform.base; + +import org.junit.jupiter.api.Test; + +import org.springframework.core.io.ClassPathResource; + +import org.springframework.http.client.MultipartBodyBuilder; +import org.springframework.test.web.reactive.server.WebTestClient; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.net.URISyntaxException; + +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import static org.springframework.http.HttpStatus.OK; +import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS; + +import org.springframework.web.reactive.function.client.WebClient; + +public abstract class LivenessReadinessProbeTest +{ + protected final Integer MAX_TRANSFORMS = 10; + + @Test + public void readinessShouldReturnAn429ErrorAfterReachingMaxTransforms() throws URISyntaxException + { + final ImagesForTests testData = getImageForTest(); + + try (final var env = createEnv(testData.image)) + { + env.start(); + var url = "http://localhost:" + env.getFirstMappedPort(); + + /* + Asserts that /ready probe hasn't died before sending a transformation request. + Each /ready request creates a valid transformation and increases the counter of + used transformations, hence the need to divide MAX_TRANSFORMS + */ + for (int i = 0; i createEnv(String image) throws URISyntaxException + { + System.out.println(image); + final GenericContainer transformCore = new GenericContainer<>("alfresco/"+image+":latest"); + + return transformCore.withEnv("livenessTransformEnabled", "true") + .withEnv("maxTransforms", MAX_TRANSFORMS.toString()) + .withNetworkAliases(image) + .withExposedPorts(8090) + .waitingFor(Wait.forListeningPort()); + } + + protected static class ImagesForTests + { + private final String image; + private final String sourceMimetype; + private final String targetMimetype; + private final String filename; + + public ImagesForTests(String image, String sourceMimetype, String targetMimetype, String filename) + { + this.image = Objects.requireNonNull(image); + this.sourceMimetype = Objects.requireNonNull(sourceMimetype); + this.targetMimetype = Objects.requireNonNull(targetMimetype); + this.filename = Objects.requireNonNull(filename); + } + } + + private void sendTransformRequest(String url, String sourceMimetype, String targetMimetype, String filename) + { + var builder = createRequestBuilder(sourceMimetype, targetMimetype, filename); + WebClient client = WebClient.create(); + WebClient.ResponseSpec responseSpec = client.post() + .uri(url + "/test") + .bodyValue(builder.build()) + .retrieve(); + + System.out.println(responseSpec.toBodilessEntity().block()); + assertEquals(OK, responseSpec.toBodilessEntity().block().getStatusCode()); + } + + private MultipartBodyBuilder createRequestBuilder(String sourceMimetype, String targetMimetype, String filename) + { + MultipartBodyBuilder builder = new MultipartBodyBuilder(); + builder.part("_sourceMimetype", sourceMimetype); + builder.part("_targetMimetype", targetMimetype); + builder.part("file", new ClassPathResource(filename)); + + return builder; + } + + private static void assertProbeDied(String url) + { + WebTestClient client = WebTestClient.bindToServer().baseUrl(url+"/ready").build(); + client.get() + .exchange() + .expectStatus().isEqualTo(TOO_MANY_REQUESTS); + } + + private static void assertProbeIsOk(String url) + { + WebTestClient client = WebTestClient.bindToServer().baseUrl(url+"/ready").build(); + client.get() + .exchange() + .expectStatus().isEqualTo(OK); + } +} \ No newline at end of file diff --git a/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java b/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java index bde11f48b..b1ae9e434 100644 --- a/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java +++ b/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java @@ -26,6 +26,51 @@ */ package org.alfresco.transform.base; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.http.HttpHeaders.ACCEPT; +import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION; +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; +import static org.springframework.http.HttpStatus.CREATED; +import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import static org.alfresco.transform.base.AbstractBaseTest.getTestFile; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_BMP; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_ERROR; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_LIVE; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_LOG; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_READY; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_ROOT; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TEST; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG_LATEST; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_VERSION; +import static org.alfresco.transform.common.RequestParamMap.PAGE_REQUEST_PARAM; +import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; +import static org.alfresco.transform.common.RequestParamMap.SOURCE_MIMETYPE; +import static org.alfresco.transform.common.RequestParamMap.TARGET_MIMETYPE; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.StringJoiner; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; @@ -33,16 +78,6 @@ import ch.qos.logback.core.AppenderBase; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import org.alfresco.transform.base.fakes.FakeTransformEngineWithTwoCustomTransformers; -import org.alfresco.transform.base.fakes.FakeTransformerPdf2Png; -import org.alfresco.transform.base.fakes.FakeTransformerTxT2Pdf; -import org.alfresco.transform.base.model.FileRefEntity; -import org.alfresco.transform.base.model.FileRefResponse; -import org.alfresco.transform.base.sfs.SharedFileStoreClient; -import org.alfresco.transform.base.transform.TransformHandler; -import org.alfresco.transform.client.model.TransformReply; -import org.alfresco.transform.client.model.TransformRequest; -import org.alfresco.transform.config.TransformConfig; import org.codehaus.plexus.util.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -61,47 +96,17 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.StringJoiner; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.alfresco.transform.base.AbstractBaseTest.getTestFile; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_BMP; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_ERROR; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_LIVE; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_LOG; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_READY; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_ROOT; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TEST; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG_LATEST; -import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_VERSION; -import static org.alfresco.transform.common.RequestParamMap.PAGE_REQUEST_PARAM; -import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; -import static org.alfresco.transform.common.RequestParamMap.SOURCE_MIMETYPE; -import static org.alfresco.transform.common.RequestParamMap.TARGET_MIMETYPE; -import static org.hamcrest.Matchers.containsString; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.springframework.http.HttpHeaders.ACCEPT; -import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION; -import static org.springframework.http.HttpHeaders.CONTENT_TYPE; -import static org.springframework.http.HttpStatus.CREATED; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.alfresco.transform.base.fakes.FakeTransformEngineWithTwoCustomTransformers; +import org.alfresco.transform.base.fakes.FakeTransformerPdf2Png; +import org.alfresco.transform.base.fakes.FakeTransformerTxT2Pdf; +import org.alfresco.transform.base.model.FileRefEntity; +import org.alfresco.transform.base.model.FileRefResponse; +import org.alfresco.transform.base.probes.ProbeTransform; +import org.alfresco.transform.base.sfs.SharedFileStoreClient; +import org.alfresco.transform.base.transform.TransformHandler; +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transform.config.TransformConfig; /** * Tests the endpoints of the TransformController. @@ -294,8 +299,19 @@ public void testLiveEndpointReturnsSuccessful() throws Exception { resetProbeForTesting(transformController); mockMvc.perform(MockMvcRequestBuilders.get(ENDPOINT_LIVE)) - .andExpect(status().isOk()) - .andExpect(content().string(containsString("Success - "))); + .andExpect(status().isOk()) + .andExpect(content().string(containsString("Success - "))); + } + + @Test + public void testLiveEndpointReturnsErrorAfterTooManyTransforms() throws Exception + { + resetProbeForTesting(transformController); + ProbeTransform probeTransform = transformController.getProbeTransform(); + IntStream.range(0, 1024 + 1).forEach(i -> probeTransform.incrementTransformerCount()); + mockMvc.perform(MockMvcRequestBuilders.get(ENDPOINT_LIVE)) + .andExpect(status().is(TOO_MANY_REQUESTS.value())) + .andExpect(content().string(containsString("Transformer requested to die. It has performed more than 1024 transformations"))); } @Test diff --git a/engines/example/pom.xml b/engines/example/pom.xml index 8e5035f1d..e9df44069 100644 --- a/engines/example/pom.xml +++ b/engines/example/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml diff --git a/engines/imagemagick/pom.xml b/engines/imagemagick/pom.xml index fd4d2bad6..76e32023a 100644 --- a/engines/imagemagick/pom.xml +++ b/engines/imagemagick/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -53,6 +53,11 @@ org.apache.commons commons-lang3 + + org.testcontainers + testcontainers + test + diff --git a/engines/imagemagick/src/test/java/org/alfresco/transform/imagemagick/ImageMagickLivenessReadinessProbeIT.java b/engines/imagemagick/src/test/java/org/alfresco/transform/imagemagick/ImageMagickLivenessReadinessProbeIT.java new file mode 100644 index 000000000..e0a8b0f5a --- /dev/null +++ b/engines/imagemagick/src/test/java/org/alfresco/transform/imagemagick/ImageMagickLivenessReadinessProbeIT.java @@ -0,0 +1,37 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2023 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.imagemagick; + +import org.alfresco.transform.base.LivenessReadinessProbeTest; + +public class ImageMagickLivenessReadinessProbeIT extends LivenessReadinessProbeTest { + @Override + protected ImagesForTests getImageForTest() { + return new ImagesForTests("alfresco-imagemagick", "image/jpeg", "image/png", "quick.jpg"); + } +} diff --git a/engines/libreoffice/Dockerfile b/engines/libreoffice/Dockerfile index 570424aa8..a626d5377 100644 --- a/engines/libreoffice/Dockerfile +++ b/engines/libreoffice/Dockerfile @@ -7,7 +7,7 @@ FROM alfresco/alfresco-base-java:jre17-rockylinux8-202306121108 ARG LIBREOFFICE_VERSION=7.2.5 ENV LIBREOFFICE_RPM_URL=https://nexus.alfresco.com/nexus/service/local/repositories/thirdparty/content/org/libreoffice/libreoffice-dist/${LIBREOFFICE_VERSION}/libreoffice-dist-${LIBREOFFICE_VERSION}-linux.gz -ENV LIBREOFFICE_ARM64_RPM_URL=https://dl.rockylinux.org/pub/rocky/8/Devel/aarch64/os/Packages/l/libreoffice-6.4.7.2-13.el8.aarch64.rpm +ENV LIBREOFFICE_ARM64_RPM_URL=https://dl.rockylinux.org/pub/rocky/8/Devel/aarch64/os/Packages/l/libreoffice-6.4.7.2-15.el8.aarch64.rpm ENV JAVA_OPTS="" # Set default user information diff --git a/engines/libreoffice/pom.xml b/engines/libreoffice/pom.xml index 03eb87a52..9964c4202 100644 --- a/engines/libreoffice/pom.xml +++ b/engines/libreoffice/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -57,6 +57,11 @@ org.apache.pdfbox pdfbox + + org.testcontainers + testcontainers + test + diff --git a/engines/libreoffice/src/test/java/org/alfresco/transform/libreoffice/LibreOfficeLivenessReadinessProbeIT.java b/engines/libreoffice/src/test/java/org/alfresco/transform/libreoffice/LibreOfficeLivenessReadinessProbeIT.java new file mode 100644 index 000000000..1fe37792c --- /dev/null +++ b/engines/libreoffice/src/test/java/org/alfresco/transform/libreoffice/LibreOfficeLivenessReadinessProbeIT.java @@ -0,0 +1,37 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2023 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.libreoffice; + +import org.alfresco.transform.base.LivenessReadinessProbeTest; + +public class LibreOfficeLivenessReadinessProbeIT extends LivenessReadinessProbeTest { + @Override + protected ImagesForTests getImageForTest() { + return new ImagesForTests("alfresco-libreoffice", "text/plain", "application/pdf", "original.txt"); + } +} diff --git a/engines/misc/pom.xml b/engines/misc/pom.xml index 813e11a71..cc82afbb7 100644 --- a/engines/misc/pom.xml +++ b/engines/misc/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -57,17 +57,6 @@ org.apache.poi poi-ooxml ${dependency.poi.version} - - - org.apache.poi - poi-ooxml-schemas - - - - - org.apache.poi - poi-ooxml-lite - ${dependency.poi-ooxml-lite.version} @@ -101,6 +90,11 @@ org.dom4j dom4j + + org.testcontainers + testcontainers + test + diff --git a/engines/misc/src/test/java/org/alfresco/transform/misc/MiscLivenessReadinessProbeIT.java b/engines/misc/src/test/java/org/alfresco/transform/misc/MiscLivenessReadinessProbeIT.java new file mode 100644 index 000000000..1d772a37f --- /dev/null +++ b/engines/misc/src/test/java/org/alfresco/transform/misc/MiscLivenessReadinessProbeIT.java @@ -0,0 +1,37 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2023 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.misc; + +import org.alfresco.transform.base.LivenessReadinessProbeTest; + +public class MiscLivenessReadinessProbeIT extends LivenessReadinessProbeTest { + @Override + protected ImagesForTests getImageForTest() { + return new ImagesForTests("alfresco-transform-misc", "text/plain", "text/plain", "original.txt"); + } +} diff --git a/engines/pdfrenderer/pom.xml b/engines/pdfrenderer/pom.xml index b67a29ab3..ba6695599 100644 --- a/engines/pdfrenderer/pom.xml +++ b/engines/pdfrenderer/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -49,6 +49,11 @@ org.dom4j dom4j + + org.testcontainers + testcontainers + test + diff --git a/engines/pdfrenderer/src/test/java/org/alfresco/transform/pdfrenderer/PdfRendererLivenessReadinessProbeIT.java b/engines/pdfrenderer/src/test/java/org/alfresco/transform/pdfrenderer/PdfRendererLivenessReadinessProbeIT.java new file mode 100644 index 000000000..5a16b4556 --- /dev/null +++ b/engines/pdfrenderer/src/test/java/org/alfresco/transform/pdfrenderer/PdfRendererLivenessReadinessProbeIT.java @@ -0,0 +1,37 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2023 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.pdfrenderer; + +import org.alfresco.transform.base.LivenessReadinessProbeTest; + +public class PdfRendererLivenessReadinessProbeIT extends LivenessReadinessProbeTest { + @Override + protected ImagesForTests getImageForTest() { + return new ImagesForTests("alfresco-pdf-renderer", "application/pdf", "image/png", "quick.pdf"); + } +} diff --git a/engines/tika/pom.xml b/engines/tika/pom.xml index 056ee5f44..9f85849be 100644 --- a/engines/tika/pom.xml +++ b/engines/tika/pom.xml @@ -7,7 +7,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT ../../pom.xml @@ -62,14 +62,6 @@ tika-parsers-standard-package ${dependency.tika.version} - - org.bouncycastle - bcprov-jdk15on - - - org.bouncycastle - bcmail-jdk15on - xml-apis xml-apis @@ -81,19 +73,7 @@ joda-time joda-time - 2.12.5 - - - - - org.bouncycastle - bcprov-jdk15on - 1.70 - - - org.bouncycastle - bcmail-jdk15on - 1.70 + 2.12.6 @@ -106,17 +86,6 @@ org.apache.poi poi-ooxml ${dependency.poi.version} - - - org.apache.poi - poi-ooxml-schemas - - - - - org.apache.poi - poi-ooxml-lite - ${dependency.poi-ooxml-lite.version} org.apache.poi @@ -145,6 +114,11 @@ mockito-junit-jupiter test + + org.testcontainers + testcontainers + test + diff --git a/engines/tika/src/test/java/org/alfresco/transform/tika/TikaLivenessReadinessProbeIT.java b/engines/tika/src/test/java/org/alfresco/transform/tika/TikaLivenessReadinessProbeIT.java new file mode 100644 index 000000000..dddb9e1e0 --- /dev/null +++ b/engines/tika/src/test/java/org/alfresco/transform/tika/TikaLivenessReadinessProbeIT.java @@ -0,0 +1,38 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2023 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.tika; + +import org.alfresco.transform.base.LivenessReadinessProbeTest; + + +public class TikaLivenessReadinessProbeIT extends LivenessReadinessProbeTest { + @Override + protected ImagesForTests getImageForTest() { + return new ImagesForTests("alfresco-tika", "text/plain", "text/plain", "original.txt"); + } +} diff --git a/model/pom.xml b/model/pom.xml index 32f2ed13f..d62230b8e 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -8,7 +8,7 @@ org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT @@ -30,7 +30,7 @@ org.apache.maven maven-artifact - 3.8.6 + 3.9.6 com.fasterxml.jackson.core diff --git a/pom.xml b/pom.xml index 5faac5deb..696450122 100644 --- a/pom.xml +++ b/pom.xml @@ -3,14 +3,14 @@ 4.0.0 org.alfresco alfresco-transform-core - 5.0.1-A2-SNAPSHOT + 5.0.2-A5-SNAPSHOT Alfresco Transform Core pom org.springframework.boot spring-boot-starter-parent - 3.1.5 + 3.2.2 @@ -20,13 +20,12 @@ latest quay.io - 2.0.26 + 2.0.30 3.0.1.18 ${project.version} - 2.15.2 - 2.4.1 - 5.2.2 - 5.2.4 + 2.16.1 + 2.9.1 + 5.2.5 2.2 false @@ -160,7 +159,7 @@ com.google.guava guava - 32.1.2-jre + 33.0.0-jre @@ -171,7 +170,7 @@ org.apache.commons commons-compress - 1.24.0 + 1.25.0 @@ -184,7 +183,11 @@ snakeyaml ${dependency.snakeyaml.version} - + + org.testcontainers + testcontainers + 1.19.4 + @@ -197,7 +200,6 @@ activemq-client-jakarta 5.18.3 - @@ -335,7 +337,7 @@ io.fabric8 docker-maven-plugin - 0.43.0 + 0.43.4 org.apache.maven.plugins