From c24d1804c6edf05e518ab27b8c6740df362ccf05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yoann=20Rodi=C3=A8re?= Date: Thu, 7 Nov 2024 10:50:21 +0100 Subject: [PATCH] Reduce side of bulks when indexing in dev, tests and prod It seems documents have gotten bigger and are now saturating the OpenSearch memory, which is rather restricted in our current setup, leading to failures. --- src/main/resources/application.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index d613333..3d1b9cf 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -72,10 +72,10 @@ quarkus.hibernate-search-standalone.elasticsearch.version=opensearch:2.18 quarkus.elasticsearch.devservices.image-name=opensearch-custom:${maven.version.opensearch} quarkus.elasticsearch.devservices.java-opts=${PROD_OPENSEARCH_JAVA_OPTS} # Limit parallelism of indexing, because OpenSearch can only handle so many documents in its buffers. -# This leads to at most 8*20=160 documents being indexed in parallel, which should be plenty +# This leads to at most 8*12=96 documents being indexed in parallel, which should be plenty # given how large our documents can be. INDEXING_QUEUE_COUNT=8 -INDEXING_BULK_SIZE=20 +INDEXING_BULK_SIZE=12 quarkus.hibernate-search-standalone.elasticsearch.indexing.queue-count=${INDEXING_QUEUE_COUNT} quarkus.hibernate-search-standalone.elasticsearch.indexing.max-bulk-size=${INDEXING_BULK_SIZE} # We need to apply a custom OpenSearch mapping to exclude very large fields from the _source