logstash.yaml

# Logstash is a bit of a behemoth because of the way the plugin system works.
# Usually, patched plugins would be dropped on the filesystem as APK's and the
# application could just pick them up. Logstash's plugin system requires the
# plugins to be added by the logstash-plugin cli tool since it uses a generated
# Gemfile.
#
# In order to patch plugins we can do a few things
#
# 1. If the plugin is a default plugin, patch it out of the plugins-metadata.json
#    file so the upstream plugin is not installed. Add the plugin apk to the build
#    environment and use logstash-plugin to install the .gem file before packaging
#    the dist into a tarball (and eventually apk). All of this should be done in
#    the main pipeline so the changes are picked up by all subpackages.
#
# 2. If the plugin is not a default plugin, do the same as above, but in a
#    subpackage to be installed separately.
package:
  name: logstash
  version: 8.12.2
  epoch: 0
  description: Logstash - transport and process your logs, events, or other data
  copyright:
    - license: Apache-2.0
  dependencies:
    provides:
      - ${{package.name}}-base=${{package.full-version}}
    runtime:
      - bash # some helper scripts use bash and busybox utilities
      - busybox
      - openjdk-17-jre
      - openjdk-17-default-jvm

environment:
  contents:
    packages:
      - bash
      - busybox
      - ca-certificates-bundle
      - curl
      - gradle
      - jq
      - jruby-9.4
      # logstash plugins to load at build-time
      - logstash-filter-xml
      - logstash-integration-jdbc
      - logstash-output-opensearch
      # end logstash plugins to load at build-time
      - openjdk-17
      - openjdk-17-default-jvm
  environment:
    OSS: "true"
    LOGSTASH_SOURCE: "1"
    LANG: en_US.UTF-8
    JAVA_HOME: /usr/lib/jvm/default-jvm
    LS_JAVA_HOME: /usr/lib/jvm/default-jvm
    LS_DESTINATION: usr/share/logstash
    MAVEN_OPTS: "-Dmaven.repo.local=/var/cache/melange"
    GRADLE_USER_HOME: "/var/cache/melange"
    GRADLE_OPTS: "-Dorg.gradle.caching=true"

pipeline:
  - uses: git-checkout
    with:
      repository: https://github.com/elastic/logstash
      tag: v${{package.version}}
      expected-commit: ee9dfd33260ed956568b47be75497c7bb7165e34

  - name: Patch sources
    runs: |
      # Disable the logstash-integration-jdbc plugin download as we build and
      # package it separately
      jq 'del(.["logstash-integration-jdbc"])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json
      mv /tmp/plugins-metadata.json rakelib/

      # Disable the logstash-filter-xml plugin download as we build and
      # package it separately
      jq 'del(.["logstash-filter-xml"])' rakelib/plugins-metadata.json > /tmp/plugins-metadata.json
      mv /tmp/plugins-metadata.json rakelib/

      # Disable -Werror which means that warnings are not treated as errors
      # during compilation.  Required for Java 21.
      sed -i '/options.compilerArgs.add("-Werror")/s/^/\/\//' build.gradle

      # A hack-fix to tackle with `Could not find method enabled() for
      # arguments [true] on Report xml` error starting after 8.11.x.
      sed -i '/enabled/d' logstash-core/build.gradle

  - name: Patch out the windows and mac tarball creation
    uses: patch
    with:
      patches: create_archive_pack.patch

  - name: Install all the default plugins
    runs: |
      ./gradlew installDefaultGems

  - name: Re-inject default plugins that have been patched
    runs: |
      for gem in \
        "logstash-integration-jdbc" \
        "logstash-filter-xml";
      do
        echo "Searching for gem search=$gem-*.gem"
        gem_path=$(find /usr/share/jruby -type f -name "$gem-*.gem")
        if [ -z "$gem_path" ]; then
          echo "Could not find gem gem_path=$gem_path gem=$gem"
          exit 1
        fi
        echo "Installing gem gem_path=$gem_path"
        bin/logstash-plugin install "$gem_path"
      done

  - name: Build tarball distribution
    runs: |
      ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux

  - name: Package distribution
    runs: |
      tarball=$(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz")
      if [ -z $tarball ]; then
        echo "Could not find logstash-oss tarball, nothing to unpack"
        exit 1
      fi

      mkdir -p ${{targets.contextdir}}/${LS_DESTINATION}
      tar --strip-components 1 \
        -C ${{targets.contextdir}}/${LS_DESTINATION} \
        -xf $tarball

      mkdir -p ${{targets.contextdir}}/usr/bin/$name
      for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do
        name=$(basename $i)
        ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name
      done

  - name: Patch CVEs
    runs: |
      # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855
      rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib

subpackages:
  - name: ${{package.name}}-compat
    description: Compatibility with the upstream image
    dependencies:
      replaces:
        - ${{package.name}}
        - ${{package.name}}-with-output-opensearch
    pipeline:
      - runs: |
          src=docker/data/logstash
          dst="${{targets.contextdir}}/${LS_DESTINATION}"
          mkdir -p "${dst}/config" \
                   "${dst}/pipeline" \
                   "${{targets.contextdir}}/usr/bin"

          install -m644 "${src}/config/pipelines.yml"          "${dst}/config/pipelines.yml"
          install -m644 "${src}/config/logstash-oss.yml"       "${dst}/config/logstash-oss.yml"
          install -m644 "${src}/config/logstash-full.yml"      "${dst}/config/logstash-full.yml"
          install -m644 "${src}/config/log4j2.file.properties" "${dst}/config/log4j2.file.properties"
          install -m644 "${src}/config/log4j2.properties"      "${dst}/config/log4j2.properties"
          install -m644 "${src}/pipeline/default.conf"         "${dst}/pipeline/default.conf"
          install -m755 "${src}/bin/docker-entrypoint"         "${{targets.contextdir}}/usr/bin/docker-entrypoint"

  - name: ${{package.name}}-env2yaml
    description: Merge environment variables into logstash.yml
    pipeline:
      - working-directory: docker/data/logstash/env2yaml
        pipeline:
          - uses: go/build
            with:
              packages: .
              output: env2yaml
              ldflags: -s -w
              subpackage: "true"
      - uses: strip

  # Due to the way logstash implements their plugin system, this is a full
  # logstash package with the opensearch plugin bundled. It is based off the
  # same artifacts in the main package, meaning it will pick up the changes to
  # the default gems that were made in the main package it just adds the
  # opensearch plugin.
  - name: ${{package.name}}-with-output-opensearch
    description: Build logstash with logstash-output-opensearch plugin
    dependencies:
      replaces:
        - ${{package.name}}
      runtime:
        - bash # some helper scripts use bash and busybox utilities
        - busybox
        - openjdk-17-jre
        - openjdk-17-default-jvm
    pipeline:
      - name: Re-inject default plugins that have been patched
        runs: |
          for gem in \
            "logstash-output-opensearch";
          do
            echo "Searching for gem search=$gem-*.gem"
            gem_path=$(find /usr/share/jruby -type f -name "$gem-*.gem")
            if [ -z "$gem_path" ]; then
              echo "Could not find gem gem_path=$gem_path gem=$gem"
              exit 1
            fi
            echo "Installing gem gem_path=$gem_path"
            bin/logstash-plugin install "$gem_path"
          done
      - name: Build tarball distribution
        runs: |
          ./gradlew assembleOssTarDistribution -Pjdk_bundle_os=linux
      - name: Package distribution
        runs: |
          mkdir -p ${{targets.contextdir}}/${LS_DESTINATION}
          tar --strip-components 1 \
            -C ${{targets.contextdir}}/${LS_DESTINATION} \
            -xf $(find . -type f -name "logstash-oss-${{package.version}}-SNAPSHOT-no-jdk.tar.gz")

          mkdir -p ${{targets.contextdir}}/usr/bin/$name
          for i in ${{targets.contextdir}}/${LS_DESTINATION}/bin/*; do
            name=$(basename $i)
            ln -sf /${LS_DESTINATION}/bin/$name ${{targets.contextdir}}/usr/bin/$name
          done
      - name: Patch CVEs
        runs: |
          # Hack-fix to reduce CVE count: https://github.com/elastic/logstash/pull/3855
          rm -rf ${{targets.contextdir}}/${LS_DESTINATION}/vendor/bundle/jruby/*/gems/ruby-maven-libs-*/maven-home/lib
    test:
      environment:
        contents:
          packages:
            - wolfi-base
        environment:
          LS_JAVA_HOME: /usr/lib/jvm/default-jvm
      pipeline:
        - name: Ensure output opensearch plugin was installed
          runs: |
            logstash-plugin list | grep logstash-output-opensearch
        - name: Ensure logstash itself works with basic function
          runs: |
            echo "hello" | \
              logstash -e 'input { stdin { } } output { stdout {} }' | \
              grep 'message.*hello'

test:
  pipeline:
    - name: Ensure default plugins were actually installed
      runs: |
        logstash-plugin list | grep logstash-integration-jdbc
        logstash-plugin list | grep logstash-filter-xml
    - name: Ensure logstash itself works with basic function
      runs: |
        echo "hello" | \
          logstash -e 'input { stdin { } } output { stdout {} }' | \
          grep 'message.*hello'
    - name: Test logstash-filter-xml
      runs: |
        # Needs to be an absolute path or logstash will ignore
        logstash -f $PWD/test/logstash-filter-xml/logstash.conf

update:
  enabled: true
  github:
    identifier: elastic/logstash
    strip-prefix: v
    use-tag: true