Merge pull request #11 from 3scale/manticore

upgrade to manticore and RHEL9

.github/workflows/container-image-buildah.yml

@@ -1,13 +1,30 @@
 name: Container Image
 
 on:
-  workflow_dispatch: {}
-  workflow_call: {}
+  workflow_dispatch:
+    inputs:
+      platforms:
+        description: "comma-separated list of platforms to build for, downstream supported are linux/amd64,linux/s390x,linux/ppc64le; note: clang is broken on s390x (RHEL-15874), also cross-builds take more than 6 hours so we don't do them"
+        default: linux/amd64
+      custom_tag:
+        description: optional custom tag on remote repo you want image to be tagged with
+        default: scratch
+  workflow_call:
+    inputs:
+      platforms:
+        required: false
+        default: linux/amd64
+        type: string
+      custom_tag:
+        required: false
+        default: ''
+        type: string
   schedule:
     # every Wednesday morning
     - cron: 7 7 * * 3
   push:
     branches: [ main ]
+    tags: [ "**" ]
   pull_request:
     types: [opened, reopened, synchronize]
 
@@ -25,11 +42,18 @@ jobs:
   buildah:
     runs-on: ubuntu-latest
     steps:
+      - name: Sanitize Platforms
+        id: platforms
+        run: |
+          platforms="${{ inputs.platforms == '' && 'linux/amd64' || inputs.platforms }}"
+          archs="$( sed -e 's#linux/##g' <<< $platforms )"
+          echo "platforms=$platforms" >> $GITHUB_OUTPUT
+          echo "archs=$archs" >> $GITHUB_OUTPUT
       # Allow multi-target builds
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:
-          platforms: s390x,ppc64le # arm64
+          platforms: ${{ steps.platforms.outputs.archs }}
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -49,7 +73,9 @@ jobs:
           tags: |
             type=schedule
             type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
-            ${{ github.ref_name == 'main' && 'type=raw,value=nightly' || 'type=ref,event=branch' }}
+            type=raw,value=nightly,enable=${{ github.ref_name == 'main' }}
+            type=ref,event=branch,enable=${{ github.ref_name != 'main' && inputs.custom_tag == '' }}
+            ${{ inputs.custom_tag }}
             type=ref,event=tag
             type=ref,event=pr
 
@@ -61,7 +87,7 @@ jobs:
         uses: redhat-actions/buildah-build@v2
         with:
           tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/s390x,linux/ppc64le
+          platforms: ${{ steps.platforms.outputs.platforms }}
           labels: ${{ steps.meta.outputs.labels }}
           layers: false
           oci: true
@@ -89,13 +115,25 @@ jobs:
           # accessing a mapped port from a container did not work so lets
           # create a pod where both - server and client have same localhost
           podman pod create > podid
-          for arch in amd64 s390x ppc64le; do
-            podman run -d --pod-id-file=podid --name=searchd -u 14:0 ${{ steps.build-image.outputs.image-with-tag }}-linux$arch
+          platforms="${{ steps.platforms.outputs.platforms }}"
+
+          test_tag () {
+            podman run -d --pod-id-file=podid --name=searchd -u 14:0 "${{ steps.build-image.outputs.image-with-tag }}$1"
             sleep 3
             podman logs searchd
-            podman run --pod-id-file=podid --rm --entrypoint "/bin/env" mysql:5.7 -- mysql -h 127.0.0.1 -P 9306 -e "SELECT * FROM account limit 1;"
+            podman run --pod-id-file=podid --rm --entrypoint "/bin/env" mysql:8 -- mysql -h 127.0.0.1 -P 9306 -e "SELECT * FROM account limit 1;"
             podman rm -f searchd
-          done
+          }
+
+          if [ x$( sed -E -e 's#[^/]##g' <<< $platforms ) != "x/" ]; then
+            # if we are here, user has selected more than one build platform
+            arch_tags=$( tr ',' ' ' <<< $platforms | tr -d '/' )
+            # removed slashes to produce "linuxamd64 linuxs390x linuxppc64le"
+            for tag in $arch_tags; do test_tag -$tag; done
+          else
+            # if we are here, user has selected a single build platform
+            test_tag
+          fi
 
       - name: Push To Container Registry
         id: push-to-container-registry

.hadolint.yaml

@@ -1,4 +1,5 @@
 ignored:
-  - DL3006
-  - DL3022
-  - DL3033
+  - DL3003 # cd used as part of command, using workdir increases complexity
+  - DL3041 # hardcoding version of rpms is not how this should work
+  - DL3022 # COPY --from can work from another image, not needed to be a layer
+  - DL3033 # hardcoding version of rpms is not how this should work

Containerfile

@@ -1,28 +1,25 @@
-FROM registry.access.redhat.com/ubi7/ubi AS builder
-ENV SEARCHD_RPM=sphinx-2.2.11-1.rhel7.x86_64.rpm \
-    SEARCHD_RPM_SHA=959b04eb3f7fb2314d7a2702b61e9b3e627b66b1a8574dece21c0592be1b90e2 \
-    SEARCHD_SRC=sphinx-2.2.11-release.tar.gz \
-    SEARCHD_SRC_DIR=sphinx-2.2.11-release \
-    SEARCHD_SRC_SHA=6662039f093314f896950519fa781bc87610f926f64b3d349229002f06ac41a9
+FROM quay.io/centos/centos:stream9 AS builder
+ENV SEARCHD_REF=6.2.12 \
+    SEARCHD_REPO=https://github.com/manticoresoftware/manticoresearch.git \
+    CC=clang \
+    CXX=clang++ \
+    BUILD_PATH=/tmp/manticore_uselessly_long_path_to_prevent_rpm_build_issues \
+    BUILD_FLAGS="-DUSE_SYSLOG=0 -DWITH_GALERA=0 -DWITH_RE2=0 -DWITH_STEMMER=0 -DWITH_ICU_FORCE_STATIC=0 -DWITH_SSL=1 -DWITH_ZLIB=1 -DWITH_ODBC=0 -DWITH_EXPAT=0 -DWITH_ICONV=1 -DWITH_POSTGRESQL=0 -DWITH_MYSQL=0 -DBUILD_TESTING=0"
+WORKDIR $BUILD_PATH
+
 SHELL ["/bin/bash", "-x", "-o", "pipefail", "-c"]
+# clang is broken on s390x (RHEL-15874), to use gcc remove CC and CXX variables,
+#   also replace llvm-toolset with make automake gcc gcc-c++ kernel-devel
 # hadolint ignore=DL3003,DL3032,SC2046
-RUN if [ "$(uname -m)" == "x86_64" ]; then \
-      yumdownloader --downloadonly --destdir=/home/rpms mariadb-libs.$(uname -m) postgresql-libs.$(uname -m) unixODBC.$(uname -m) --resolve && \
-      curl -sSL https://sphinxsearch.com/files/$SEARCHD_RPM -o /home/rpms/$SEARCHD_RPM && \
-     sha256sum -c - <<< "$SEARCHD_RPM_SHA /home/rpms/$SEARCHD_RPM" && \
-     mkdir /home/searchd ; \
-    else \
-      yumdownloader --downloadonly --destdir=/home/rpms mariadb-libs.$(uname -m) --resolve && \
-      yum install -y gcc-c++ mysql-devel make  && \
-      curl -sSL http://sphinxsearch.com/files/$SEARCHD_SRC -o /tmp/$SEARCHD_SRC && \
-      sha256sum -c - <<< "$SEARCHD_SRC_SHA /tmp/$SEARCHD_SRC" && \
-      tar xzf /tmp/$SEARCHD_SRC -C /tmp && \
-      cd /tmp/$SEARCHD_SRC_DIR && ./configure && DESTDIR=/home/searchd make install && \
-      rm -rf /tmp/sphinx* ; \
-    fi
-
+RUN yum install -y --setopt=skip_missing_names_on_install=False,tsflags=nodocs llvm-toolset mysql cmake boost-devel openssl-devel zlib-devel libicu-devel bison flex systemd-units rpm-build git && \
+      git clone --depth=1 --branch=$SEARCHD_REF $SEARCHD_REPO . && \
+      # boost lib in RHEL9 comes dynamic only so enable its use \
+      sed -i -e 's/Boost_USE_STATIC_LIBS ON/Boost_USE_STATIC_LIBS OFF/' src/CMakeLists.txt && \
+      mkdir build && cd build && \
+      cmake $BUILD_FLAGS .. && \
+      cmake --build . --target package --config RelWithDebInfo
 
-FROM registry.access.redhat.com/ubi7-minimal
+FROM quay.io/centos/centos:stream9-minimal
 
 LABEL org.opencontainers.image.authors="https://issues.redhat.com/browse/THREESCALE" \
       org.opencontainers.image.title="3scale searchd" \
@@ -37,14 +34,20 @@ LABEL org.opencontainers.image.authors="https://issues.redhat.com/browse/THREESC
       # org.opencontainers.image.created=""
 
 ARG PORTA_IMAGE=quay.io/3scale/porta:nightly
-COPY --from=builder /home/rpms /tmp/rpms
-COPY --from=builder /home/searchd /
-COPY --from=$PORTA_IMAGE /opt/system/config/standalone.sphinx.conf "/etc/sphinx/system.sphinx.conf"
-RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release && \
-    rpm -iv --excludedocs /tmp/rpms/* && \
-    rm -rf /tmp/rpms && \
-    mkdir -p /var/lib/searchd /var/run/sphinx && \
-    chmod 770 /var/lib/searchd /var/run/sphinx
+COPY --from=builder /tmp/manticore_uselessly_long_path_to_prevent_rpm_build_issues/build/*.rpm /tmp/rpms/
+COPY --from=$PORTA_IMAGE /opt/system/config/standalone.sphinx.conf "/etc/manticoresearch/manticore.conf"
+ENV MANTICORE_RPMS="manticore-converter* manticore-common* manticore-server-core* manticore-server*"
+RUN microdnf install -y --nodocs mysql openssl boost-context boost-filesystem zlib libicu && \
+    cd /tmp/rpms && ls -l && \
+    rpm -iv --excludedocs $MANTICORE_RPMS && \
+    cd - && rm -rf /tmp/rpms && \
+    microdnf clean all && \
+    # TODO: once in production, update porta to generate config with the correct path \
+    sed -i -e 's#/var/run/sphinx/#/var/run/manticore/#' /etc/manticoresearch/manticore.conf && \
+    mkdir /var/lib/searchd && \
+    chmod g+w /var/lib/searchd /var/run/manticore /var/log/manticore && \
+    chgrp 0 /var/lib/searchd /var/run/manticore /var/log/manticore
 
-ENTRYPOINT ["/bin/env", "searchd", "--pidfile", "--config", "/etc/sphinx/system.sphinx.conf", "--nodetach"]
+WORKDIR /var/lib/manticore
+ENTRYPOINT ["/bin/env", "searchd", "--pidfile", "--config", "/etc/manticoresearch/manticore.conf", "--nodetach"]
 EXPOSE 9306/tcp