Merge pull request #372 from yrodiere/more-queues

Adapt to slow indexing in prod

src/main/helm/values.staging.yaml

@@ -2,7 +2,7 @@ app:
   envs:
     QUARKUS_PROFILE: 'staging'
     # Avoid overloading the rather resource-constrained Search backend instance
-    INDEXING_QUEUE_COUNT: '4'
+    INDEXING_QUEUE_COUNT: '6'
     INDEXING_BULK_SIZE: '10'
   resources:
     limits:

src/main/resources/application.properties

@@ -27,7 +27,7 @@ quarkusio.localized.ja.git-uri=https://github.com/quarkusio/ja.quarkus.io.git
 ########################
 fetching.parallelism=8
 fetching.timeout=10m
-indexing.timeout=5m
+indexing.timeout=10m
 # Index at 00:00 UTC every day
 # The time was selected to minimize impact on users:
 # In winter this would be: Paris 01:00, New Delhi: 05:30, New York: 19:00, Beijing: 08:00, Los Angeles: 16:00
@@ -72,9 +72,9 @@ quarkus.hibernate-search-standalone.elasticsearch.version=${maven.distribution.s
 quarkus.elasticsearch.devservices.image-name=${maven.name.search.backend}:${maven.version.search.backend}
 quarkus.elasticsearch.devservices.java-opts=${PROD_ES_JAVA_OPTS}
 # Limit parallelism of indexing, because the search backend can only handle so many documents in its buffers.
-# This leads to at most 8*20=160 documents being indexed in parallel, which should be plenty
+# This leads to at most 12*20=240 documents being indexed in parallel, which should be plenty
 # given how large our documents can be.
-INDEXING_QUEUE_COUNT=8
+INDEXING_QUEUE_COUNT=12
 INDEXING_BULK_SIZE=20
 quarkus.hibernate-search-standalone.elasticsearch.indexing.queue-count=${INDEXING_QUEUE_COUNT}
 quarkus.hibernate-search-standalone.elasticsearch.indexing.max-bulk-size=${INDEXING_BULK_SIZE}
@@ -213,12 +213,11 @@ quarkus.openshift.env.vars.QUARKUS_PROFILE=prod
 quarkus.openshift.env.vars.INDEXING_QUEUE_COUNT=${INDEXING_QUEUE_COUNT}
 quarkus.openshift.env.vars.INDEXING_BULK_SIZE=${INDEXING_BULK_SIZE}
 # Initial indexing may take a while, especially the quarkus.io Git cloning
-# Tests indicate it can take ~240s in prod,
+# Tests indicate the whole process can take ~5M in prod,
 # so we'll use 3 times as much as the limit, to allow for 3 attempts.
-# 15S * 48 = 720S = 240S * 3
-quarkus.openshift.startup-probe.initial-delay=30S
-quarkus.openshift.startup-probe.period=15S
-quarkus.openshift.startup-probe.failure-threshold=48
+quarkus.openshift.startup-probe.initial-delay=2M
+quarkus.openshift.startup-probe.period=1M
+quarkus.openshift.startup-probe.failure-threshold=15
 # Declare the management port on the service
 quarkus.openshift.ports."management".container-port=9000
 quarkus.openshift.ports."management".host-port=90