diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2de6c958..d013dbff 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,6 +25,10 @@ jobs: runs-on: ubuntu-latest + env: + QUARKUS_PROFILE: ${{ github.ref == 'refs/heads/production' && 'prod' || 'staging' }} + TAG: ${{ github.sha }} + steps: - uses: actions/checkout@v2 @@ -34,6 +38,11 @@ jobs: distribution: temurin java-version: 17 + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: 'v3.13.3' + - name: Log in to OpenShift (Dev) if: ${{ github.ref == 'refs/heads/main' }} uses: redhat-actions/oc-login@v1 @@ -50,8 +59,31 @@ jobs: openshift_token: ${{ secrets.OPENSHIFT_TOKEN_PROD }} namespace: ${{ env.OPENSHIFT_NAMESPACE_PROD }} - - name: Delete problematic image - run: oc delete is ubi-quarkus-native-binary-s2i || true + - name: Create ImageStreams + run: | + oc create imagestream search-quarkus-io || true + oc create imagestream opensearch-custom || true + # https://docs.openshift.com/container-platform/4.14/openshift_images/using-imagestreams-with-kube-resources.html + oc set image-lookup search-quarkus-io + oc set image-lookup opensearch-custom + + - name: Log in to OpenShift container registry + run: docker login -u "$(oc whoami)" -p "$(oc whoami --show-token)" "$(oc registry info)" + + - name: Build container images and Helm charts, push app container image + run: | + ./mvnw clean package \ + -Drevision="$TAG" \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.push=true \ + -Dquarkus.container-image.registry="$(oc registry info)" \ + -Dquarkus.container-image.group="$(oc project --short)" + + - name: Push OpenSearch container image + run: | + docker push "opensearch-custom:latest" "$(oc registry info)/$(oc project --short)/opensearch-custom:latest" - - name: Build and deploy - run: mvn clean package -Dquarkus.kubernetes.deploy=true -Dquarkus.native.container-build=true -Drevision=${{ github.sha }} + - name: Deploy Helm charts + run: | + helm upgrade --install search-quarkus-io ./target/helm/openshift/search-quarkus-io \ + -f ./src/main/helm/values.$QUARKUS_PROFILE.yaml \ No newline at end of file diff --git a/README.adoc b/README.adoc index d12373bf..2720f248 100644 --- a/README.adoc +++ b/README.adoc @@ -128,9 +128,9 @@ If you want to start a (production) container locally with podman, you can build [source,shell] ---- -quarkus build -DskipTests -Dquarkus.container-image.build=true -Dquarkus.container-image.builder=jib +quarkus build -DskipTests -Dquarkus.container-image.build=true # OR -./mvnw install -DskipTests -Dquarkus.container-image.build=true -Dquarkus.container-image.builder=jib +./mvnw install -DskipTests -Dquarkus.container-image.build=true ---- Then start it this way: @@ -147,7 +147,7 @@ podman container run -d --name search-backend-0 --pod search.quarkus.io \ -e "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g" \ -e "DISABLE_SECURITY_PLUGIN=true" \ -e "cluster.routing.allocation.disk.threshold_enabled=false" \ - opensearch-custom-plugin:2.11.0 + opensearch-custom:2.11.0 podman container run -d --name search-backend-1 --pod search.quarkus.io \ --cpus=2 --memory=2g \ -e "node.name=search-backend-1" \ @@ -156,7 +156,7 @@ podman container run -d --name search-backend-1 --pod search.quarkus.io \ -e "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g" \ -e "DISABLE_SECURITY_PLUGIN=true" \ -e "cluster.routing.allocation.disk.threshold_enabled=false" \ - opensearch-custom-plugin:2.11.0 + opensearch-custom:2.11.0 podman container run -d --name search-backend-2 --pod search.quarkus.io \ --cpus=2 --memory=2g \ -e "node.name=search-backend-2" \ @@ -165,7 +165,7 @@ podman container run -d --name search-backend-2 --pod search.quarkus.io \ -e "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g" \ -e "DISABLE_SECURITY_PLUGIN=true" \ -e "cluster.routing.allocation.disk.threshold_enabled=false" \ - opensearch-custom-plugin:2.11.0 + opensearch-custom:2.11.0 # Then the app; this will fetch the actual data on startup (might take a while): podman container run -it --rm --name search.quarkus.io --pod search.quarkus.io search-quarkus-io:999-SNAPSHOT # OR, if you already have locals clones of *.quarkus.io: diff --git a/pom.xml b/pom.xml index a4207d6a..f54ab4d0 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,7 @@ 3.12.1 + ${version.opensearch} 17 UTF-8 UTF-8 @@ -36,6 +37,7 @@ 0.44.0 2.23.0 1.9.0 + 2.11.0 @@ -161,6 +163,11 @@ io.quarkus quarkus-hibernate-validator + + io.quarkiverse.helm + quarkus-helm + 1.2.1 + io.quarkus quarkus-junit5 @@ -205,6 +212,7 @@ ${test.jvm.args} ${project.basedir}/src/test/resources + ${version.opensearch} @@ -229,6 +237,7 @@ org.jboss.logmanager.LogManager ${maven.home} ${project.basedir}/src/test/resources + ${version.opensearch} true @@ -308,11 +317,11 @@ - opensearch-custom-plugin + opensearch-custom - ${project.basedir}/Dockerfile + ${project.basedir}/src/main/docker/opensearch-custom.Dockerfile - 2.11.0 + ${version.opensearch} diff --git a/Dockerfile b/src/main/docker/opensearch-custom.Dockerfile similarity index 54% rename from Dockerfile rename to src/main/docker/opensearch-custom.Dockerfile index 4dd42773..5538c62b 100644 --- a/Dockerfile +++ b/src/main/docker/opensearch-custom.Dockerfile @@ -1,4 +1,8 @@ -FROM opensearchproject/opensearch:2.11.0 +ARG OPENSEARCH_VERSION= +FROM opensearchproject/opensearch:${OPENSEARCH_VERSION} + +# Workaround for https://github.com/opensearch-project/opensearch-devops/issues/97 +RUN chmod -R go=u /usr/share/opensearch RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-kuromoji RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-smartcn diff --git a/src/main/helm/values.prod.yaml b/src/main/helm/values.prod.yaml new file mode 100644 index 00000000..ddc176b2 --- /dev/null +++ b/src/main/helm/values.prod.yaml @@ -0,0 +1 @@ +# Prod values are the defaults \ No newline at end of file diff --git a/src/main/helm/values.staging.yaml b/src/main/helm/values.staging.yaml new file mode 100644 index 00000000..ab43437a --- /dev/null +++ b/src/main/helm/values.staging.yaml @@ -0,0 +1,18 @@ +app: + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 400m + memory: 500Mi +opensearch: + envs: + OPENSEARCH_JAVA_OPTS: ' -Xms500m -Xmx500m ' + resources: + limits: + cpu: 1000m + memory: 1.5Gi + requests: + cpu: 500m + memory: 1Gi \ No newline at end of file diff --git a/src/main/kubernetes/openshift.yml b/src/main/kubernetes/openshift.yml index d907ce77..446cfa44 100644 --- a/src/main/kubernetes/openshift.yml +++ b/src/main/kubernetes/openshift.yml @@ -1,73 +1,3 @@ -apiVersion: image.openshift.io/v1 -kind: "ImageStream" -metadata: - name: opensearch - annotations: - openshift.io/display-name: opensearch - labels: - app.kubernetes.io/name: search-backend - app.kubernetes.io/component: datastore - app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus -spec: - lookupPolicy: - local: true - tags: - - name: current - from: - kind: DockerImage - name: docker.io/opensearchproject/opensearch:2.11.0 ---- -apiVersion: image.openshift.io/v1 -kind: "ImageStream" -metadata: - name: opensearch-fixed - annotations: - openshift.io/display-name: opensearch-fixed - labels: - app.kubernetes.io/name: search-backend - app.kubernetes.io/component: datastore - app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus -spec: - lookupPolicy: - local: true ---- -apiVersion: build.openshift.io/v1 -kind: BuildConfig -metadata: - name: opensearch-fixed-build - labels: - app.kubernetes.io/name: search-backend - app.kubernetes.io/component: datastore - app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus -spec: - triggers: - - type: ConfigChange - - type: ImageChange - imageChange: - from: - kind: ImageStreamTag - name: opensearch:current - source: - # Workaround for https://github.com/opensearch-project/opensearch-devops/issues/97 - dockerfile: |- - FROM opensearch:current - RUN chmod -R go=u /usr/share/opensearch - RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-kuromoji - RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-smartcn - RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu - strategy: - type: Docker - dockerStrategy: - from: - kind: ImageStreamTag - name: opensearch:current - output: - to: - kind: ImageStreamTag - name: opensearch-fixed:current --- apiVersion: v1 kind: Service @@ -78,7 +8,6 @@ metadata: app.kubernetes.io/name: search-backend app.kubernetes.io/component: datastore app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus spec: ports: - name: http @@ -103,7 +32,6 @@ metadata: app.kubernetes.io/name: search-backend app.kubernetes.io/component: datastore app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus # See https://www.hafifbilgiler.com/hafif-bilgiler/elasticsearch-installation-on-openshift/ spec: serviceName: search-backend @@ -118,19 +46,21 @@ spec: app.kubernetes.io/name: search-backend app.kubernetes.io/component: datastore app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus + annotations: + alpha.image.policy.openshift.io/resolve-names: '*' spec: containers: - name: opensearch - image: opensearch-fixed:current + # The image gets pushed manually as part of the "deploy" workflow. + image: opensearch-custom:latest imagePullPolicy: Always resources: limits: - cpu: 2000m - memory: 3Gi + cpu: '{{ .Values.opensearch.resources.limits.cpu }}' + memory: '{{ .Values.opensearch.resources.limits.memory }}' requests: - cpu: 1000m - memory: 2Gi + cpu: '{{ .Values.opensearch.resources.requests.cpu }}' + memory: '{{ .Values.opensearch.resources.requests.memory }}' readinessProbe: httpGet: scheme: HTTP @@ -164,14 +94,14 @@ spec: value: "false" # OpenSearch doesn't seem to automatically adapt -Xmx to available memory, for some reason - name: OPENSEARCH_JAVA_OPTS - value: -Xms1g -Xmx1g + value: '{{ .Values.opensearch.envs.OPENSEARCH_JAVA_OPTS }}' # This is necessary to avoid OpenSearch trying to install various things on startup, # which leads to filesystem operations (chmod/chown) that won't work # because only user 1000 has the relevant permissions, # and we can't run with user 1000 on OpenShift. # See also: # - https://github.com/opensearch-project/opensearch-devops/issues/97 - # - BuildConfig opensearch-fixed-build + # - src/main/docker/opensearch-custom.Dockerfile - name: DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI value: 'true' - name: DISABLE_INSTALL_DEMO_CONFIG @@ -195,20 +125,9 @@ spec: app.kubernetes.io/name: search-backend app.kubernetes.io/component: datastore app.kubernetes.io/part-of: search-quarkus-io - app.kubernetes.io/managed-by: quarkus spec: accessModes: [ "ReadWriteOnce" ] storageClassName: "gp2" resources: requests: - storage: 5Gi - triggers: - - type: ConfigChange - - type: ImageChange - imageChangeParams: - automatic: true - containerNames: - - search-backend - from: - kind: ImageStreamTag - name: opensearch-fixed:current \ No newline at end of file + storage: 5Gi \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f010bdf5..30886422 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -60,8 +60,8 @@ quarkus.datasource.jdbc.max-size=100 ## But we don't need it to be that large after indexing. quarkus.datasource.jdbc.min-size=0 ## Hibernate Search -quarkus.hibernate-search-orm.elasticsearch.version=opensearch:2.11 -quarkus.elasticsearch.devservices.image-name=opensearch-custom-plugin:2.11.0 +quarkus.hibernate-search-orm.elasticsearch.version=opensearch:${maven.version.opensearch} +quarkus.elasticsearch.devservices.image-name=opensearch-custom:${maven.version.opensearch} ## Limit parallelism of indexing, because OpenSearch can only handle so many documents in its buffers. ## This leads to at most 12*20=240 documents being indexed in parallel, which should be plenty ## given how large our documents can be. @@ -130,8 +130,8 @@ mp.openapi.extensions.smallrye.info.contact.url=https://github.com/quarkusio/sea quarkus.swagger-ui.always-include=true quarkus.swagger-ui.title=Quarkus Search API -# Deployment to OpenShift -quarkus.container-image.builder=openshift +# OpenShift - App config +quarkus.container-image.builder=jib quarkus.openshift.part-of=search-quarkus-io # See src/main/kubernetes/openshift.yml for the search-backend StatefulSet definition # Rely on OpenShift's internal DNS to resolve the IP to search-backend nodes @@ -147,11 +147,15 @@ quarkus.openshift.env.vars.home=/tmp quarkus.openshift.annotations."kubernetes.io/tls-acme"=true quarkus.openshift.env.configmaps=search-quarkus-io-config quarkus.openshift.env.secrets=search-quarkus-io-secrets -# Declare resource requirements +# Resource requirements (overridden for staging, see src/main/helm) quarkus.openshift.resources.limits.cpu=1000m quarkus.openshift.resources.requests.cpu=400m quarkus.openshift.resources.limits.memory=2Gi quarkus.openshift.resources.requests.memory=1Gi +quarkus.helm.values."resources.limits.cpu".paths=spec.template.spec.containers.resources.limits.cpu +quarkus.helm.values."resources.requests.cpu".paths=spec.template.spec.containers.resources.requests.cpu +quarkus.helm.values."resources.limits.memory".paths=spec.template.spec.containers.resources.limits.memory +quarkus.helm.values."resources.requests.memory".paths=spec.template.spec.containers.resources.requests.memory # Initial indexing may take a while, especially the quarkus.io Git cloning quarkus.openshift.startup-probe.initial-delay=30S quarkus.openshift.startup-probe.period=15S @@ -161,4 +165,12 @@ quarkus.openshift.ports."management".container-port=9000 quarkus.openshift.ports."management".host-port=90 # Don't use the version in (service) selectors, # otherwise a rollback to an earlier version (due to failing startup) makes the service unavailable -quarkus.openshift.add-version-to-label-selectors=false \ No newline at end of file +quarkus.openshift.add-version-to-label-selectors=false + +# OpenShift - Backend config +# Resource requirements (overridden for staging, see src/main/helm) +quarkus.helm.values."@.opensearch.envs.OPENSEARCH_JAVA_OPTS".value=\ -Xms1g -Xmx1g +quarkus.helm.values."@.opensearch.resources.limits.cpu".value=2000m +quarkus.helm.values."@.opensearch.resources.requests.cpu".value=1000m +quarkus.helm.values."@.opensearch.resources.limits.memory".value=3Gi +quarkus.helm.values."@.opensearch.resources.requests.memory".value=2Gi