Merge pull request #184 from yrodiere/helm

Use quarkus-helm for deployment, set lower resource requirements on staging
quarkusio · Feb 20, 2024 · e9518f4 · e9518f4
2 parents 3b5c6c4 + 6c5217d
commit e9518f4
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 111 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -25,6 +25,10 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    env:
+      QUARKUS_PROFILE: ${{ github.ref == 'refs/heads/production' && 'prod' || 'staging' }}
+      TAG: ${{ github.sha }}
+
     steps:
     - uses: actions/checkout@v2
 
@@ -34,6 +38,11 @@ jobs:
         distribution: temurin
         java-version: 17
 
+    - name: Set up Helm
+      uses: azure/setup-helm@v4
+      with:
+         version: 'v3.13.3'
+
     - name: Log in to OpenShift (Dev)
       if: ${{ github.ref == 'refs/heads/main' }}
       uses: redhat-actions/oc-login@v1
@@ -50,8 +59,31 @@ jobs:
         openshift_token: ${{ secrets.OPENSHIFT_TOKEN_PROD }}
         namespace: ${{ env.OPENSHIFT_NAMESPACE_PROD }}
 
-    - name: Delete problematic image
-      run: oc delete is ubi-quarkus-native-binary-s2i || true
+    - name: Create ImageStreams
+      run: |
+        oc create imagestream search-quarkus-io || true
+        oc create imagestream opensearch-custom || true
+        # https://docs.openshift.com/container-platform/4.14/openshift_images/using-imagestreams-with-kube-resources.html
+        oc set image-lookup search-quarkus-io
+        oc set image-lookup opensearch-custom
+
+    - name: Log in to OpenShift container registry
+      run: docker login -u "$(oc whoami)" -p "$(oc whoami --show-token)" "$(oc registry info)"
+
+    - name: Build container images and Helm charts, push app container image
+      run: |
+        ./mvnw clean package \
+          -Drevision="$TAG" \
+          -Dquarkus.container-image.build=true \
+          -Dquarkus.container-image.push=true \
+          -Dquarkus.container-image.registry="$(oc registry info)" \
+          -Dquarkus.container-image.group="$(oc project --short)"
+
+    - name: Push OpenSearch container image
+      run: |
+        docker push "opensearch-custom:latest" "$(oc registry info)/$(oc project --short)/opensearch-custom:latest"
 
-    - name: Build and deploy
-      run: mvn clean package -Dquarkus.kubernetes.deploy=true -Dquarkus.native.container-build=true -Drevision=${{ github.sha }}
+    - name: Deploy Helm charts
+      run: |
+        helm upgrade --install search-quarkus-io ./target/helm/openshift/search-quarkus-io \
+          -f ./src/main/helm/values.$QUARKUS_PROFILE.yaml
diff --git a/README.adoc b/README.adoc
@@ -128,9 +128,9 @@ If you want to start a (production) container locally with podman, you can build
 
 [source,shell]
 ----
-quarkus build -DskipTests -Dquarkus.container-image.build=true -Dquarkus.container-image.builder=jib
+quarkus build -DskipTests -Dquarkus.container-image.build=true
 # OR
-./mvnw install -DskipTests -Dquarkus.container-image.build=true -Dquarkus.container-image.builder=jib
+./mvnw install -DskipTests -Dquarkus.container-image.build=true
 ----
 
 Then start it this way:
@@ -147,7 +147,7 @@ podman container run -d --name search-backend-0 --pod search.quarkus.io \
     -e "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g" \
     -e "DISABLE_SECURITY_PLUGIN=true" \
     -e "cluster.routing.allocation.disk.threshold_enabled=false" \
-    opensearch-custom-plugin:2.11.0
+    opensearch-custom:2.11.0
 podman container run -d --name search-backend-1 --pod search.quarkus.io \
     --cpus=2 --memory=2g \
     -e "node.name=search-backend-1" \
@@ -156,7 +156,7 @@ podman container run -d --name search-backend-1 --pod search.quarkus.io \
     -e "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g" \
     -e "DISABLE_SECURITY_PLUGIN=true" \
     -e "cluster.routing.allocation.disk.threshold_enabled=false" \
-    opensearch-custom-plugin:2.11.0
+    opensearch-custom:2.11.0
 podman container run -d --name search-backend-2 --pod search.quarkus.io \
     --cpus=2 --memory=2g \
     -e "node.name=search-backend-2" \
@@ -165,7 +165,7 @@ podman container run -d --name search-backend-2 --pod search.quarkus.io \
     -e "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g" \
     -e "DISABLE_SECURITY_PLUGIN=true" \
     -e "cluster.routing.allocation.disk.threshold_enabled=false" \
-    opensearch-custom-plugin:2.11.0
+    opensearch-custom:2.11.0
 # Then the app; this will fetch the actual data on startup (might take a while):
 podman container run -it --rm --name search.quarkus.io --pod search.quarkus.io search-quarkus-io:999-SNAPSHOT
 # OR, if you already have locals clones of *.quarkus.io:

diff --git a/pom.xml b/pom.xml
@@ -23,6 +23,7 @@
   </issueManagement>
   <properties>
     <compiler-plugin.version>3.12.1</compiler-plugin.version>
+    <docker.buildArg.OPENSEARCH_VERSION>${version.opensearch}</docker.buildArg.OPENSEARCH_VERSION>
     <maven.compiler.release>17</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -36,6 +37,7 @@
     <version.docker.plugin>0.44.0</version.docker.plugin>
     <version.formatter.plugin>2.23.0</version.formatter.plugin>
     <version.impsort-maven-plugin>1.9.0</version.impsort-maven-plugin>
+    <version.opensearch>2.11.0</version.opensearch>
   </properties>
   <dependencyManagement>
     <dependencies>
@@ -161,6 +163,11 @@
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-hibernate-validator</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.quarkiverse.helm</groupId>
+      <artifactId>quarkus-helm</artifactId>
+      <version>1.2.1</version>
+    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-junit5</artifactId>
@@ -205,6 +212,7 @@
           <jvmArgs>${test.jvm.args}</jvmArgs>
           <systemProperties>
             <maven.project.testResourceDirectory>${project.basedir}/src/test/resources</maven.project.testResourceDirectory>
+            <maven.version.opensearch>${version.opensearch}</maven.version.opensearch>
           </systemProperties>
         </configuration>
       </plugin>
@@ -229,6 +237,7 @@
             <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager>
             <maven.home>${maven.home}</maven.home>
             <maven.project.testResourceDirectory>${project.basedir}/src/test/resources</maven.project.testResourceDirectory>
+            <maven.version.opensearch>${version.opensearch}</maven.version.opensearch>
           </systemPropertyVariables>
           <statelessTestsetReporter implementation="org.apache.maven.plugin.surefire.extensions.junit5.JUnit5Xml30StatelessReporter">
             <usePhrasedFileName>true</usePhrasedFileName>
@@ -308,11 +317,11 @@
         <configuration>
           <images>
             <image>
-              <name>opensearch-custom-plugin</name>
+              <name>opensearch-custom</name>
               <build>
-                <dockerFile>${project.basedir}/Dockerfile</dockerFile>
+                <dockerFile>${project.basedir}/src/main/docker/opensearch-custom.Dockerfile</dockerFile>
                 <tags>
-                  <tag>2.11.0</tag>
+                  <tag>${version.opensearch}</tag>
                 </tags>
               </build>
             </image>

diff --git a/Dockerfile → src/main/docker/opensearch-custom.Dockerfile b/Dockerfile → src/main/docker/opensearch-custom.Dockerfile
@@ -1,4 +1,8 @@
-FROM opensearchproject/opensearch:2.11.0
+ARG OPENSEARCH_VERSION=
+FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
+
+# Workaround for https://github.com/opensearch-project/opensearch-devops/issues/97
+RUN chmod -R go=u /usr/share/opensearch
 
 RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-kuromoji
 RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-smartcn

diff --git a/src/main/helm/values.prod.yaml b/src/main/helm/values.prod.yaml
@@ -0,0 +1 @@
+# Prod values are the defaults
diff --git a/src/main/helm/values.staging.yaml b/src/main/helm/values.staging.yaml
@@ -0,0 +1,18 @@
+app:
+  resources:
+    limits:
+      cpu: 500m
+      memory: 1Gi
+    requests:
+      cpu: 400m
+      memory: 500Mi
+opensearch:
+  envs:
+    OPENSEARCH_JAVA_OPTS: ' -Xms500m -Xmx500m '
+  resources:
+    limits:
+      cpu: 1000m
+      memory: 1.5Gi
+    requests:
+      cpu: 500m
+      memory: 1Gi
diff --git a/src/main/kubernetes/openshift.yml b/src/main/kubernetes/openshift.yml
@@ -1,73 +1,3 @@
-apiVersion: image.openshift.io/v1
-kind: "ImageStream"
-metadata:
-  name: opensearch
-  annotations:
-    openshift.io/display-name: opensearch
-  labels:
-    app.kubernetes.io/name: search-backend
-    app.kubernetes.io/component: datastore
-    app.kubernetes.io/part-of: search-quarkus-io
-    app.kubernetes.io/managed-by: quarkus
-spec:
-  lookupPolicy:
-    local: true
-  tags:
-    - name: current
-      from:
-        kind: DockerImage
-        name: docker.io/opensearchproject/opensearch:2.11.0
----
-apiVersion: image.openshift.io/v1
-kind: "ImageStream"
-metadata:
-  name: opensearch-fixed
-  annotations:
-    openshift.io/display-name: opensearch-fixed
-  labels:
-    app.kubernetes.io/name: search-backend
-    app.kubernetes.io/component: datastore
-    app.kubernetes.io/part-of: search-quarkus-io
-    app.kubernetes.io/managed-by: quarkus
-spec:
-  lookupPolicy:
-    local: true
----
-apiVersion: build.openshift.io/v1
-kind: BuildConfig
-metadata:
-  name: opensearch-fixed-build
-  labels:
-    app.kubernetes.io/name: search-backend
-    app.kubernetes.io/component: datastore
-    app.kubernetes.io/part-of: search-quarkus-io
-    app.kubernetes.io/managed-by: quarkus
-spec:
-  triggers:
-    - type: ConfigChange
-    - type: ImageChange
-      imageChange:
-        from:
-          kind: ImageStreamTag
-          name: opensearch:current
-  source:
-    # Workaround for https://github.com/opensearch-project/opensearch-devops/issues/97
-    dockerfile: |-
-      FROM opensearch:current
-      RUN chmod -R go=u /usr/share/opensearch
-      RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-kuromoji
-      RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-smartcn
-      RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu
-  strategy:
-    type: Docker
-    dockerStrategy:
-      from:
-        kind: ImageStreamTag
-        name: opensearch:current
-  output:
-    to:
-      kind: ImageStreamTag
-      name: opensearch-fixed:current
 ---
 apiVersion: v1
 kind: Service
@@ -78,7 +8,6 @@ metadata:
     app.kubernetes.io/name: search-backend
     app.kubernetes.io/component: datastore
     app.kubernetes.io/part-of: search-quarkus-io
-    app.kubernetes.io/managed-by: quarkus
 spec:
   ports:
     - name: http
@@ -103,7 +32,6 @@ metadata:
     app.kubernetes.io/name: search-backend
     app.kubernetes.io/component: datastore
     app.kubernetes.io/part-of: search-quarkus-io
-    app.kubernetes.io/managed-by: quarkus
 # See https://www.hafifbilgiler.com/hafif-bilgiler/elasticsearch-installation-on-openshift/
 spec:
   serviceName: search-backend
@@ -118,19 +46,21 @@ spec:
         app.kubernetes.io/name: search-backend
         app.kubernetes.io/component: datastore
         app.kubernetes.io/part-of: search-quarkus-io
-        app.kubernetes.io/managed-by: quarkus
+      annotations:
+        alpha.image.policy.openshift.io/resolve-names: '*'
     spec:
       containers:
         - name: opensearch
-          image: opensearch-fixed:current
+          # The image gets pushed manually as part of the "deploy" workflow.
+          image: opensearch-custom:latest
           imagePullPolicy: Always
           resources:
             limits:
-              cpu: 2000m
-              memory: 3Gi
+              cpu: '{{ .Values.opensearch.resources.limits.cpu }}'
+              memory: '{{ .Values.opensearch.resources.limits.memory }}'
             requests:
-              cpu: 1000m
-              memory: 2Gi
+              cpu: '{{ .Values.opensearch.resources.requests.cpu }}'
+              memory: '{{ .Values.opensearch.resources.requests.memory }}'
           readinessProbe:
             httpGet:
               scheme: HTTP
@@ -164,14 +94,14 @@ spec:
               value: "false"
             # OpenSearch doesn't seem to automatically adapt -Xmx to available memory, for some reason
             - name: OPENSEARCH_JAVA_OPTS
-              value: -Xms1g -Xmx1g
+              value: '{{ .Values.opensearch.envs.OPENSEARCH_JAVA_OPTS }}'
             # This is necessary to avoid OpenSearch trying to install various things on startup,
             # which leads to filesystem operations (chmod/chown) that won't work
             # because only user 1000 has the relevant permissions,
             # and we can't run with user 1000 on OpenShift.
             # See also:
             # - https://github.com/opensearch-project/opensearch-devops/issues/97
-            # - BuildConfig opensearch-fixed-build
+            # - src/main/docker/opensearch-custom.Dockerfile
             - name: DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI
               value: 'true'
             - name: DISABLE_INSTALL_DEMO_CONFIG
@@ -195,20 +125,9 @@ spec:
           app.kubernetes.io/name: search-backend
           app.kubernetes.io/component: datastore
           app.kubernetes.io/part-of: search-quarkus-io
-          app.kubernetes.io/managed-by: quarkus
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: "gp2"
         resources:
           requests:
-            storage: 5Gi
-  triggers:
-    - type: ConfigChange
-    - type: ImageChange
-      imageChangeParams:
-        automatic: true
-        containerNames:
-          - search-backend
-        from:
-          kind: ImageStreamTag
-          name: opensearch-fixed:current
+            storage: 5Gi